From dfc4e6ad9ad2183b461986bcd7eb9b659579b331 Mon Sep 17 00:00:00 2001 From: dry923 Date: Fri, 19 Mar 2021 10:23:38 -0400 Subject: [PATCH 001/249] Adding debug workload option for snafu related jobs and enabling it for CI jobs --- README.md | 26 +++++++++++++++++++ roles/cyclictest/templates/cyclictestjob.yaml | 7 ++++- roles/fio_distributed/templates/client.yaml | 7 ++++- roles/flent/templates/workload.yml.j2 | 6 ++++- roles/fs-drift/templates/workload_job.yml.j2 | 5 +++- .../templates/db_mssql_workload.yml.j2 | 6 ++++- .../templates/db_mysql_workload.yml.j2 | 6 ++++- .../templates/db_postgres_workload.yml.j2 | 6 ++++- roles/oslat/templates/oslatjob.yaml | 7 ++++- roles/pgbench/templates/workload.yml.j2 | 6 ++++- roles/scale_openshift/templates/scale.yml | 6 ++++- roles/smallfile/templates/workload_job.yml.j2 | 6 ++++- .../templates/stressng_workload.yml.j2 | 7 ++++- .../templates/stressng_workload_vm.yml.j2 | 6 ++++- roles/testpmd/templates/trex.yml.j2 | 9 ++++--- roles/uperf/templates/configmap_script.yml.j2 | 6 ++++- roles/uperf/templates/workload.yml.j2 | 9 ++++--- roles/vegeta/templates/vegeta.yml.j2 | 6 ++++- roles/ycsb/templates/ycsb_load.yaml | 7 ++++- roles/ycsb/templates/ycsb_run.yaml | 7 ++++- tests/test_crs/valid_fiod.yaml | 1 + tests/test_crs/valid_fiod_bsrange.yaml | 1 + tests/test_crs/valid_fiod_hostpath.yaml | 1 + tests/test_crs/valid_flent.yaml | 1 + tests/test_crs/valid_flent_resources.yaml | 1 + tests/test_crs/valid_fs_drift.yaml | 1 + tests/test_crs/valid_fs_drift_hostpath.yaml | 1 + tests/test_crs/valid_hammerdb.yaml | 1 + tests/test_crs/valid_pgbench.yaml | 1 + tests/test_crs/valid_scale_down.yaml | 2 +- tests/test_crs/valid_scale_up.yaml | 2 +- tests/test_crs/valid_smallfile.yaml | 1 + tests/test_crs/valid_smallfile_hostpath.yaml | 1 + tests/test_crs/valid_stressng.yaml | 2 +- tests/test_crs/valid_uperf.yaml | 1 + tests/test_crs/valid_uperf_networkpolicy.yaml | 1 + tests/test_crs/valid_uperf_resources.yaml | 1 + tests/test_crs/valid_uperf_serviceip.yaml | 1 + tests/test_crs/valid_vegeta.yaml | 1 + tests/test_crs/valid_vegeta_hostnetwork.yaml | 1 + tests/test_crs/valid_ycsb-mongo.yaml | 1 + 41 files changed, 149 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 22435c14f..a8246c0fc 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,32 @@ spec: image: my.location/foo:latest ``` +## Optional debug out for benchmark-wrapper workloads +Workloads that are triggered through [benchmark-wrapper](https://github.com/cloud-bulldozer/benchmark-wrapper) +can optionally pass the debug flag through the workload CR. + +NOTE: This is not a required arguement. If omitted it will default to the default logging level of +the benchmark-wrapper + +For Example: + +``` +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: example-benchmark + namespace: my-ripsaw +spec: + elasticsearch: + url: "http://my-es.foo.bar:80" + metadata_collection: true + cleanup: false + workload: + name: snafu_workload + args: + debug: true +``` + ## User Provided UUID All benchmarks in the benchmark-operator utilize a UUID for tracking and indexing purposes. This UUID is, by default, generated when the workload is first started. However, if desired, a user provided UUID can diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index 1652f68a4..c6bf0b398 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -17,7 +17,12 @@ spec: - name: cyclictest image: {{ workload_args.image | default('quay.io/cloud-bulldozer/cyclictest:latest') }} command: ["/bin/sh", "-c"] - args: ["run_snafu --tool cyclictest -p /tmp/cyclictest.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}}"] + args: + - run_snafu --tool cyclictest -p /tmp/cyclictest.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; imagePullPolicy: Always {% if workload_args.nodeselector is defined %} nodeSelector: diff --git a/roles/fio_distributed/templates/client.yaml b/roles/fio_distributed/templates/client.yaml index 551098952..ecef56843 100644 --- a/roles/fio_distributed/templates/client.yaml +++ b/roles/fio_distributed/templates/client.yaml @@ -72,7 +72,12 @@ spec: {% for numjobs in workload_args.numjobs %} {% for i in loopvar %} {% for job in workload_args.jobs %} - cat /tmp/fio/fiojob-{{job}}-{{i}}-{{numjobs}}; mkdir -p /tmp/fiod-{{uuid}}/fiojob-{{job}}-{{i}}-{{numjobs}}; run_snafu -t fio -H /tmp/host/hosts -j /tmp/fio/fiojob-{{job}}-{{i}}-{{numjobs}} -s {{workload_args.samples}} -d /tmp/fiod-{{ uuid }}/fiojob-{{job}}-{{i}}-{{numjobs}}; + cat /tmp/fio/fiojob-{{job}}-{{i}}-{{numjobs}}; mkdir -p /tmp/fiod-{{uuid}}/fiojob-{{job}}-{{i}}-{{numjobs}}; + run_snafu -t fio -H /tmp/host/hosts -j /tmp/fio/fiojob-{{job}}-{{i}}-{{numjobs}} -s {{workload_args.samples}} -d /tmp/fiod-{{ uuid }}/fiojob-{{job}}-{{i}}-{{numjobs}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; {% if workload_args.fio_json_to_log is defined and workload_args.fio_json_to_log is sameas true %} for fio_sample in $(seq 1 {{workload_args.samples}}); do echo START_FIO_JSON_OUTPUT_fiod-{{uuid}}_fiojob-{{job}}-{{i}}-{{numjobs}}_SAMPLE_$fio_sample; diff --git a/roles/flent/templates/workload.yml.j2 b/roles/flent/templates/workload.yml.j2 index a6022efd5..421318d7d 100644 --- a/roles/flent/templates/workload.yml.j2 +++ b/roles/flent/templates/workload.yml.j2 @@ -69,7 +69,11 @@ spec: while true; do if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start) =~ 'true' ]]; then {% for test in workload_args.test_types %} - run_snafu --tool flent --ftest {{test}} -u {{ uuid }} -r $h --user {{test_user | default("ripsaw")}}; + run_snafu --tool flent --ftest {{test}} -u {{ uuid }} -r $h --user {{test_user | default("ripsaw")}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; {% endfor %} else continue; diff --git a/roles/fs-drift/templates/workload_job.yml.j2 b/roles/fs-drift/templates/workload_job.yml.j2 index 4b1cc0ff2..41f706088 100644 --- a/roles/fs-drift/templates/workload_job.yml.j2 +++ b/roles/fs-drift/templates/workload_job.yml.j2 @@ -78,12 +78,15 @@ spec: cat /tmp/fs-drift/params; run_snafu --tool fs-drift +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} --top {{fs_drift_path}}/fs_drift_test_data --dir ${TMPDIR}/RESULTS {% if workload_args.samples is defined %} --samples {{workload_args.samples}} {% endif %} - --yaml-input-file /tmp/fs-drift/params ; + --yaml-input-file /tmp/fs-drift/params; if [ $? = 0 ] ; then echo RUN STATUS DONE ; else echo FAIL ; exit 1 ; fi volumeMounts: - name: config-volume diff --git a/roles/hammerdb/templates/db_mssql_workload.yml.j2 b/roles/hammerdb/templates/db_mssql_workload.yml.j2 index 84aeb5bab..e5513f0c0 100644 --- a/roles/hammerdb/templates/db_mssql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mssql_workload.yml.j2 @@ -77,7 +77,11 @@ spec: export db_mssql_durability={{workload_args.db_mssql_durability}}; export db_mssql_checkpoint={{workload_args.db_mssql_checkpoint}}; cd /hammer; - run_snafu --tool hammerdb -u {{ uuid }}" + run_snafu --tool hammerdb -u {{ uuid }} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ;" volumeMounts: - name: hammerdb-workload-volume mountPath: "/workload" diff --git a/roles/hammerdb/templates/db_mysql_workload.yml.j2 b/roles/hammerdb/templates/db_mysql_workload.yml.j2 index 4888e36eb..90e93146e 100644 --- a/roles/hammerdb/templates/db_mysql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mysql_workload.yml.j2 @@ -71,7 +71,11 @@ spec: export db_mysql_storage_engine={{workload_args.db_mysql_storage_engine}}; export db_mysql_partition={{workload_args.db_mysql_partition}}; cd /hammer; - run_snafu --tool hammerdb -u {{ uuid }}" + run_snafu --tool hammerdb -u {{ uuid }} +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ;" volumeMounts: - name: hammerdb-workload-volume mountPath: "/workload" diff --git a/roles/hammerdb/templates/db_postgres_workload.yml.j2 b/roles/hammerdb/templates/db_postgres_workload.yml.j2 index 4f207e718..7569a7be9 100644 --- a/roles/hammerdb/templates/db_postgres_workload.yml.j2 +++ b/roles/hammerdb/templates/db_postgres_workload.yml.j2 @@ -75,7 +75,11 @@ spec: export db_postgresql_oracompat={{workload_args.db_postgresql_oracompat}}; export db_postgresql_storedprocs={{workload_args.db_postgresql_storedprocs}}; cd /hammer; - run_snafu --tool hammerdb -u {{ uuid }}" + run_snafu --tool hammerdb -u {{ uuid }} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ;" volumeMounts: - name: hammerdb-workload-volume mountPath: "/workload" diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index 7441527d3..9ed2636e5 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -17,7 +17,12 @@ spec: - name: oslat image: {{ workload_args.image | default('quay.io/cloud-bulldozer/oslat:latest') }} command: ["/bin/sh", "-c"] - args: ["run_snafu --tool oslat -p /tmp/oslat.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}}"] + args: + - run_snafu --tool oslat -p /tmp/oslat.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; imagePullPolicy: Always {% if workload_args.nodeselector is defined %} nodeSelector: diff --git a/roles/pgbench/templates/workload.yml.j2 b/roles/pgbench/templates/workload.yml.j2 index 7d804b165..30854d300 100644 --- a/roles/pgbench/templates/workload.yml.j2 +++ b/roles/pgbench/templates/workload.yml.j2 @@ -73,7 +73,11 @@ spec: export sample_start_timestamp=$(date +%s.%3N); echo \"Begin test sample $i of {{ workload_args.samples }}...\"; export pgbench_opts=\"$pgbench_auth -c {{ clients }} -j {{ workload_args.threads }} {% if workload_args.transactions is defined and workload_args.transactions|int > 0 %} -t {{ workload_args.transactions }} {% elif run_time|int > 0 %} -T {{ run_time }} {% endif %} -s {{ workload_args.scaling_factor }} {{ cmd_flags }} {{ item.1.db_name }}\"; - run_snafu --tool pgbench -r $i; + run_snafu --tool pgbench -r $i \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; done; {% endfor %} else diff --git a/roles/scale_openshift/templates/scale.yml b/roles/scale_openshift/templates/scale.yml index e7e8eb750..64f06460b 100755 --- a/roles/scale_openshift/templates/scale.yml +++ b/roles/scale_openshift/templates/scale.yml @@ -64,7 +64,11 @@ spec: {% endif %} command: ["/bin/sh", "-c"] args: - - "run_snafu --tool scale --scale {{workload_args.scale}} -u {{uuid}} --user {{test_user|default("ripsaw")}} --incluster true --poll_interval {{workload_args.poll_interval|default(5)}}; + - "run_snafu --tool scale --scale {{workload_args.scale}} -u {{uuid}} --user {{test_user|default("ripsaw")}} --incluster true --poll_interval {{workload_args.poll_interval|default(5)}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; sleep {{post_sleep|default(0)}}" serviceAccountName: {{workload_args.serviceaccount|default("default")}} restartPolicy: Never diff --git a/roles/smallfile/templates/workload_job.yml.j2 b/roles/smallfile/templates/workload_job.yml.j2 index 5a6c06efb..96880328b 100644 --- a/roles/smallfile/templates/workload_job.yml.j2 +++ b/roles/smallfile/templates/workload_job.yml.j2 @@ -92,7 +92,11 @@ spec: --operations $arr --top {{smallfile_path}}/smallfile_test_data --dir /var/tmp/RESULTS - --yaml-input-file /tmp/smallfile/smallfilejob ; + --yaml-input-file /tmp/smallfile/smallfilejob +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + ; if [ $? = 0 ] ; then echo RUN STATUS DONE ; else exit 1 ; fi volumeMounts: - name: config-volume diff --git a/roles/stressng/templates/stressng_workload.yml.j2 b/roles/stressng/templates/stressng_workload.yml.j2 index dfb9eb309..040f2deb0 100644 --- a/roles/stressng/templates/stressng_workload.yml.j2 +++ b/roles/stressng/templates/stressng_workload.yml.j2 @@ -74,7 +74,12 @@ spec: value: "{{ prometheus.prom_url | default() }}" {% endif %} command: ["/bin/sh", "-c"] - args: ["run_snafu --tool stressng -j /workload/jobfile -u {{ uuid }}"] + args: + - run_snafu --tool stressng -j /workload/jobfile -u {{ uuid }} +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + ; volumeMounts: - name: stressng-workload-volume mountPath: "/workload" diff --git a/roles/stressng/templates/stressng_workload_vm.yml.j2 b/roles/stressng/templates/stressng_workload_vm.yml.j2 index 7545aa84d..e00394749 100644 --- a/roles/stressng/templates/stressng_workload_vm.yml.j2 +++ b/roles/stressng/templates/stressng_workload_vm.yml.j2 @@ -66,7 +66,11 @@ spec: {% endif %} - dnf install -y stress-ng redis git - pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper - - run_snafu --tool stressng -j /tmp/stressng-test/jobfile -u {{ uuid }} + - run_snafu --tool stressng -j /tmp/stressng-test/jobfile -u {{ uuid }} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; name: cloudinitdisk - configMap: name: "{{ meta.name }}-workload-{{ trunc_uuid }}" diff --git a/roles/testpmd/templates/trex.yml.j2 b/roles/testpmd/templates/trex.yml.j2 index a7c15f29f..d6ca79275 100644 --- a/roles/testpmd/templates/trex.yml.j2 +++ b/roles/testpmd/templates/trex.yml.j2 @@ -47,10 +47,13 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: trex - command: ["/bin/sh"] + command: ["/bin/sh", "-c"] args: - - "-c" - - "run_snafu --tool trex --uuid {{ trunc_uuid }} --user {{ test_user | default('snafu') }}" + - run_snafu --tool trex --uuid {{ trunc_uuid }} --user {{ test_user | default('snafu') }} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; image: "{{ workload_args.image_trex | default('quay.io/cloud-bulldozer/trex:latest') }}" imagePullPolicy: "{{ workload_args.image_pull_policy | default('Always') }}" securityContext: diff --git a/roles/uperf/templates/configmap_script.yml.j2 b/roles/uperf/templates/configmap_script.yml.j2 index 278f51da4..bb5e44749 100644 --- a/roles/uperf/templates/configmap_script.yml.j2 +++ b/roles/uperf/templates/configmap_script.yml.j2 @@ -42,7 +42,11 @@ data: {% endif %} {% for nthr in workload_args.nthrs %} cat /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; - run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}}; + run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; {% endfor %} {% endfor %} {% endfor %} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 493b4b7a9..20274ee0d 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -112,11 +112,14 @@ spec: {% endif %} {% for nthr in workload_args.nthrs %} cat /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; + run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ {% if workload_args.run_id is defined %} - run_snafu --tool uperf --run-id {{workload_args.run_id}} -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}}; -{% else %} - run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}}; + --run-id {{workload_args.run_id}} \ +{% endif %} +{% if workload_args.debug is defined and workload_args.debug %} + -v \ {% endif %} + ; {% endfor %} {% endfor %} {% endfor %} diff --git a/roles/vegeta/templates/vegeta.yml.j2 b/roles/vegeta/templates/vegeta.yml.j2 index 199a8f748..49f7130eb 100644 --- a/roles/vegeta/templates/vegeta.yml.j2 +++ b/roles/vegeta/templates/vegeta.yml.j2 @@ -75,7 +75,11 @@ spec: done sleep 0.1 redis-cli -h {{ bo.resources[0].status.podIP }} set vegeta-{{ item }}-{{ trunc_uuid }} notready - run_snafu --tool vegeta --targets /tmp/vegeta/{{ t.name|replace(" ","") }} -u ${uuid} -d {{ t.duration }} --user ${test_user} -w {{ t.workers|default(1) }} -s {{ t.samples|default(1) }} {{ "--keepalive" if t.keepalive|default(false) }} + run_snafu --tool vegeta --targets /tmp/vegeta/{{ t.name|replace(" ","") }} -u ${uuid} -d {{ t.duration }} --user ${test_user} -w {{ t.workers|default(1) }} -s {{ t.samples|default(1) }} {{ "--keepalive" if t.keepalive|default(false) }} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; {% endfor %} volumeMounts: - name: targets-volume diff --git a/roles/ycsb/templates/ycsb_load.yaml b/roles/ycsb/templates/ycsb_load.yaml index 9ef0407b5..54093a481 100644 --- a/roles/ycsb/templates/ycsb_load.yaml +++ b/roles/ycsb/templates/ycsb_load.yaml @@ -54,7 +54,12 @@ spec: {% endif %} imagePullPolicy: Always command: ["/bin/sh", "-c"] - args: ['run_snafu --tool ycsb -r 1 -l -d {{ workload_args.driver }} -u {{ uuid }} --user {{ test_user | default("ripsaw") }} -w {{ ycsb_workload_load }} -x "{{ workload_args.options_load }}"'] + args: + - run_snafu --tool ycsb +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + -r 1 -l -d {{ workload_args.driver }} -u {{ uuid }} --user {{ test_user | default("ripsaw") }} -w {{ ycsb_workload_load }} -x "{{ workload_args.options_load }}"; volumeMounts: - name: config-volume mountPath: "/tmp/ycsb" diff --git a/roles/ycsb/templates/ycsb_run.yaml b/roles/ycsb/templates/ycsb_run.yaml index 9711d0af3..7f094e44c 100644 --- a/roles/ycsb/templates/ycsb_run.yaml +++ b/roles/ycsb/templates/ycsb_run.yaml @@ -54,7 +54,12 @@ spec: {% endif %} imagePullPolicy: Always command: ["/bin/sh", "-c"] - args: ['run_snafu --tool ycsb -r 1 -d {{ workload_args.driver }} -w {{ ycsb_workload }} -u {{ uuid }} --user {{ test_user | default("ripsaw") }} -x "{{ workload_args.options_run }}"'] + args: + - run_snafu --tool ycsb +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + -r 1 -d {{ workload_args.driver }} -w {{ ycsb_workload }} -u {{ uuid }} --user {{ test_user | default("ripsaw") }} -x "{{ workload_args.options_run }}"; volumeMounts: - name: config-volume mountPath: "/tmp/ycsb" diff --git a/tests/test_crs/valid_fiod.yaml b/tests/test_crs/valid_fiod.yaml index 1bb26a04b..ca8c1b39a 100644 --- a/tests/test_crs/valid_fiod.yaml +++ b/tests/test_crs/valid_fiod.yaml @@ -33,6 +33,7 @@ spec: filesize: 10MiB log_sample_rate: 2000 storagesize: 16Mi + debug: true ####################################### # EXPERT AREA - MODIFY WITH CAUTION # ####################################### diff --git a/tests/test_crs/valid_fiod_bsrange.yaml b/tests/test_crs/valid_fiod_bsrange.yaml index a580eccf5..c86b09773 100644 --- a/tests/test_crs/valid_fiod_bsrange.yaml +++ b/tests/test_crs/valid_fiod_bsrange.yaml @@ -33,6 +33,7 @@ spec: filesize: 10MiB log_sample_rate: 2000 storagesize: 16Mi + debug: true ####################################### # EXPERT AREA - MODIFY WITH CAUTION # ####################################### diff --git a/tests/test_crs/valid_fiod_hostpath.yaml b/tests/test_crs/valid_fiod_hostpath.yaml index ec7d9573d..9b826bbf0 100644 --- a/tests/test_crs/valid_fiod_hostpath.yaml +++ b/tests/test_crs/valid_fiod_hostpath.yaml @@ -36,6 +36,7 @@ spec: write_ramp_time: 1 filesize: 10MiB log_sample_rate: 2000 + debug: true ####################################### # EXPERT AREA - MODIFY WITH CAUTION # ####################################### diff --git a/tests/test_crs/valid_flent.yaml b/tests/test_crs/valid_flent.yaml index 3fff6fd72..73f9d882a 100644 --- a/tests/test_crs/valid_flent.yaml +++ b/tests/test_crs/valid_flent.yaml @@ -22,3 +22,4 @@ spec: test_types: - tcp_download runtime: 2 + debug: true diff --git a/tests/test_crs/valid_flent_resources.yaml b/tests/test_crs/valid_flent_resources.yaml index e0ec9ba8f..75b75f4b2 100644 --- a/tests/test_crs/valid_flent_resources.yaml +++ b/tests/test_crs/valid_flent_resources.yaml @@ -30,3 +30,4 @@ spec: test_types: - tcp_download runtime: 2 + debug: true diff --git a/tests/test_crs/valid_fs_drift.yaml b/tests/test_crs/valid_fs_drift.yaml index 84d237fcc..f949c665a 100644 --- a/tests/test_crs/valid_fs_drift.yaml +++ b/tests/test_crs/valid_fs_drift.yaml @@ -21,3 +21,4 @@ spec: max_file_size_kb: 4 max_files: 1000 duration: 15 + debug: true diff --git a/tests/test_crs/valid_fs_drift_hostpath.yaml b/tests/test_crs/valid_fs_drift_hostpath.yaml index 650cbce58..bc1e338f0 100644 --- a/tests/test_crs/valid_fs_drift_hostpath.yaml +++ b/tests/test_crs/valid_fs_drift_hostpath.yaml @@ -22,3 +22,4 @@ spec: max_file_size_kb: 4 max_files: 1000 duration: 15 + debug: true diff --git a/tests/test_crs/valid_hammerdb.yaml b/tests/test_crs/valid_hammerdb.yaml index 1c975cee6..74c1bf74e 100644 --- a/tests/test_crs/valid_hammerdb.yaml +++ b/tests/test_crs/valid_hammerdb.yaml @@ -61,3 +61,4 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" + debug: true diff --git a/tests/test_crs/valid_pgbench.yaml b/tests/test_crs/valid_pgbench.yaml index 6862b19d7..b0d9b72f6 100644 --- a/tests/test_crs/valid_pgbench.yaml +++ b/tests/test_crs/valid_pgbench.yaml @@ -28,3 +28,4 @@ spec: user: ci password: ci db_name: cidb + debug: true diff --git a/tests/test_crs/valid_scale_down.yaml b/tests/test_crs/valid_scale_down.yaml index fb24e9666..ade94fe09 100644 --- a/tests/test_crs/valid_scale_down.yaml +++ b/tests/test_crs/valid_scale_down.yaml @@ -13,4 +13,4 @@ spec: scale: 0 serviceaccount: scaler poll_interval: 2 - + debug: true diff --git a/tests/test_crs/valid_scale_up.yaml b/tests/test_crs/valid_scale_up.yaml index f3ce103a4..a54cc658a 100644 --- a/tests/test_crs/valid_scale_up.yaml +++ b/tests/test_crs/valid_scale_up.yaml @@ -13,4 +13,4 @@ spec: scale: 1 serviceaccount: scaler poll_interval: 2 - + debug: true diff --git a/tests/test_crs/valid_smallfile.yaml b/tests/test_crs/valid_smallfile.yaml index f15d3bbea..ab9cec08f 100644 --- a/tests/test_crs/valid_smallfile.yaml +++ b/tests/test_crs/valid_smallfile.yaml @@ -21,3 +21,4 @@ spec: threads: 1 file_size: 0 files: 100000 + debug: true diff --git a/tests/test_crs/valid_smallfile_hostpath.yaml b/tests/test_crs/valid_smallfile_hostpath.yaml index 51604ab4b..f57942152 100644 --- a/tests/test_crs/valid_smallfile_hostpath.yaml +++ b/tests/test_crs/valid_smallfile_hostpath.yaml @@ -22,3 +22,4 @@ spec: threads: 1 file_size: 0 files: 100000 + debug: true diff --git a/tests/test_crs/valid_stressng.yaml b/tests/test_crs/valid_stressng.yaml index be0fc3468..d30f5afc1 100644 --- a/tests/test_crs/valid_stressng.yaml +++ b/tests/test_crs/valid_stressng.yaml @@ -25,4 +25,4 @@ spec: vm_bytes: "128M" # mem stressor options mem_stressors: "1" - + debug: true diff --git a/tests/test_crs/valid_uperf.yaml b/tests/test_crs/valid_uperf.yaml index fcc5061f2..3dedadad8 100644 --- a/tests/test_crs/valid_uperf.yaml +++ b/tests/test_crs/valid_uperf.yaml @@ -36,3 +36,4 @@ spec: - 1 - 2 runtime: 2 + debug: true diff --git a/tests/test_crs/valid_uperf_networkpolicy.yaml b/tests/test_crs/valid_uperf_networkpolicy.yaml index a3a95317e..a236b2a21 100644 --- a/tests/test_crs/valid_uperf_networkpolicy.yaml +++ b/tests/test_crs/valid_uperf_networkpolicy.yaml @@ -37,3 +37,4 @@ spec: - 1 - 2 runtime: 2 + debug: true diff --git a/tests/test_crs/valid_uperf_resources.yaml b/tests/test_crs/valid_uperf_resources.yaml index 435d80e95..70341ee76 100644 --- a/tests/test_crs/valid_uperf_resources.yaml +++ b/tests/test_crs/valid_uperf_resources.yaml @@ -43,3 +43,4 @@ spec: - 1 - 2 runtime: 2 + debug: true diff --git a/tests/test_crs/valid_uperf_serviceip.yaml b/tests/test_crs/valid_uperf_serviceip.yaml index e03d5ea39..9e5e5e707 100644 --- a/tests/test_crs/valid_uperf_serviceip.yaml +++ b/tests/test_crs/valid_uperf_serviceip.yaml @@ -35,3 +35,4 @@ spec: - 1 - 2 runtime: 2 + debug: true diff --git a/tests/test_crs/valid_vegeta.yaml b/tests/test_crs/valid_vegeta.yaml index ff69405dd..a40213e34 100644 --- a/tests/test_crs/valid_vegeta.yaml +++ b/tests/test_crs/valid_vegeta.yaml @@ -28,3 +28,4 @@ spec: - GET https://1.1.1.1 workers: 2 duration: 5 + debug: true diff --git a/tests/test_crs/valid_vegeta_hostnetwork.yaml b/tests/test_crs/valid_vegeta_hostnetwork.yaml index 50f4fe0f2..287aafd92 100644 --- a/tests/test_crs/valid_vegeta_hostnetwork.yaml +++ b/tests/test_crs/valid_vegeta_hostnetwork.yaml @@ -24,3 +24,4 @@ spec: workers: 2 duration: 5 keepalive: true + debug: true diff --git a/tests/test_crs/valid_ycsb-mongo.yaml b/tests/test_crs/valid_ycsb-mongo.yaml index 4fd9f4b38..3953bc790 100644 --- a/tests/test_crs/valid_ycsb-mongo.yaml +++ b/tests/test_crs/valid_ycsb-mongo.yaml @@ -21,3 +21,4 @@ spec: - workloada options_load: '-p mongodb.url="mongodb://mongo-0.mongo/ycsb?"' #passed as is to ycsb when loading database options_run: '-p mongodb.url="mongodb://mongo-0.mongo/ycsb?" -threads 10 -target 100' + debug: true From d4c075d7a5a67dfdd12ca30112266d4051b90382 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Thu, 17 Dec 2020 11:09:43 -0500 Subject: [PATCH 002/249] basic pod enumeration over node list over denstiy argument --- roles/uperf/tasks/main.yml | 38 +++++++++++++++++++++++++++-- roles/uperf/templates/server.yml.j2 | 12 +++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index 5e5fafefb..7d54966fb 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -45,6 +45,23 @@ register: serviceip when: workload_args.serviceip is defined and workload_args.serviceip +### V2 vs V1 +- name: V2 Record num server (V1) pods + set_fact: + num_server_pods: "{{ workload_args.pair | default('1'|int) }}" + when: workload_args.max_node is not defined + +- name: V2 Setup eligible node (static) list - Tobe replaced by real node list builder + set_fact: + server_node_list: [ worker0 ] + +- name: V2 Record num server pods + set_fact: + num_server_pods: "{{ server_node_list|length * workload_args.density | default('1'|int) }}" + when: workload_args.max_node is defined + +### + - block: - name: Create service for server pods @@ -58,6 +75,19 @@ definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" register: servers with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + when: workload_args.max_node is not defined + +############ + + - name: V2 Start Server(s) - total = eligible nodes * density + k8s: + definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" + with_nested: + - "{{ server_node_list }}" + - "{{ range(0, workload_args.density | default('1'|int)) | list }}" + when: workload_args.max_node is defined + +############ - name: Wait for pods to be running.... k8s_facts: @@ -78,7 +108,8 @@ state: "Starting Servers" when: resource_state.resources[0].status.state == "Building" and resource_kind == "pod" - + +########### VM block - block: - name: Start Server(s) @@ -107,6 +138,8 @@ when: resource_state.resources[0].status.state == "Building" and resource_kind == "vm" +########### + - block: - name: Get server pods @@ -126,7 +159,8 @@ namespace: "{{ operator_namespace }}" status: state: "Starting Clients" - when: "workload_args.pair|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + when: "num_server_pods|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + #when: "workload_args.pair|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index d46a66799..2d91d2421 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -2,7 +2,13 @@ kind: Job apiVersion: batch/v1 metadata: + +{% if workload_args.max_node is not defined %} name: 'uperf-server-{{ item }}-{{ trunc_uuid }}' +{% else %} + name: 'uperf-server-{{ item[0] }}-{{ item[1] }}-{{ trunc_uuid }}' +{% endif %} + namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -10,7 +16,13 @@ spec: template: metadata: labels: + +{% if workload_args.max_node is not defined %} app: uperf-bench-server-{{item}}-{{ trunc_uuid }} +{% else %} + app: uperf-bench-server-{{ item[0] }}-{{ item[1] }}-{{ trunc_uuid }} +{% endif %} + type: uperf-bench-server-{{ trunc_uuid }} {% if workload_args.multus.enabled is sameas true %} annotations: From 4169f64bd6da3d6195f380774a17af50c2defccf Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Fri, 18 Dec 2020 14:19:01 +0000 Subject: [PATCH 003/249] density fully working -with debug --- roles/uperf/tasks/main.yml | 124 +++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 54 deletions(-) diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index 7d54966fb..eb9e73310 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -1,6 +1,6 @@ --- -- name: Get current state +- name: A1 Get current state k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -18,7 +18,7 @@ complete: false when: resource_state.resources[0].status.state is not defined -- name: Get current state - If it has changed +- name: A3 Get current state - If it has changed k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -26,7 +26,7 @@ namespace: '{{ operator_namespace }}' register: resource_state -- name: Capture operator information +- name: A4 Capture operator information k8s_facts: kind: Pod api_version: v1 @@ -35,7 +35,7 @@ - name = benchmark-operator register: bo -- name: Capture ServiceIP +- name: A5 Capture ServiceIP k8s_facts: kind: Service api_version: v1 @@ -46,31 +46,35 @@ when: workload_args.serviceip is defined and workload_args.serviceip ### V2 vs V1 -- name: V2 Record num server (V1) pods +- name: A6 Record num server (V1) pods set_fact: - num_server_pods: "{{ workload_args.pair | default('1'|int) }}" + num_server_pods: "{{ workload_args.pair | default('1')|int }}" when: workload_args.max_node is not defined -- name: V2 Setup eligible node (static) list - Tobe replaced by real node list builder +- name: A7 V2 Setup eligible node (static) list - Tobe replaced by real node list builder set_fact: - server_node_list: [ worker0 ] + server_node_list: [ worker0, worker2 ] -- name: V2 Record num server pods +- name: A8 V2 Record num server pods set_fact: - num_server_pods: "{{ server_node_list|length * workload_args.density | default('1'|int) }}" + num_server_pods: "{{ server_node_list|length * workload_args.density | default('1')|int }}" when: workload_args.max_node is defined +- name: A8_1 show num_server_pods + debug: + var: num_server_pods + ### - block: - - name: Create service for server pods + - name: P8 Create service for server pods k8s: definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}" with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} when: workload_args.serviceip is defined and workload_args.serviceip - - name: Start Server(s) + - name: P9 Start Server(s) k8s: definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" register: servers @@ -79,7 +83,7 @@ ############ - - name: V2 Start Server(s) - total = eligible nodes * density + - name: P10 V2 Start Server(s) - total = eligible nodes * density k8s: definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" with_nested: @@ -89,7 +93,7 @@ ############ - - name: Wait for pods to be running.... + - name: P11 Wait for pods to be running.... k8s_facts: kind: Pod api_version: v1 @@ -98,7 +102,7 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_pods - - name: Update resource state + - name: P12 Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -109,16 +113,16 @@ when: resource_state.resources[0].status.state == "Building" and resource_kind == "pod" -########### VM block +########### - block: - - name: Start Server(s) + - name: V11 Start Server(s) k8s: definition: "{{ lookup('template', 'server_vm.yml.j2') | from_yaml }}" register: servers with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} - - name: Wait for vms to be running.... + - name: V12 Wait for vms to be running.... k8s_facts: kind: VirtualMachineInstance api_version: kubevirt.io/v1alpha3 @@ -127,7 +131,7 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_vms - - name: Update resource state + - name: V13 Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -140,9 +144,10 @@ ########### +########### - block: - - name: Get server pods + - name: P13 Get server pods k8s_facts: kind: Pod api_version: v1 @@ -151,7 +156,7 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_pods - - name: Update resource state + - name: P14 Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -165,9 +170,11 @@ when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" +######## +######## - block: - - name: Wait for vms to be running.... + - name: V14 Wait for vms to be running.... k8s_facts: kind: VirtualMachineInstance api_version: kubevirt.io/v1alpha3 @@ -176,7 +183,7 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_vms - - name: Update resource state + - name: V15 Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -184,18 +191,22 @@ namespace: "{{ operator_namespace }}" status: state: "Starting Clients" - when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + #when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + when: "num_server_pods|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - - name: blocking client from running uperf + - name: V16 blocking client from running uperf command: "redis-cli set start false" with_items: "{{ server_vms.resources }}" - when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + #when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + when: "num_server_pods|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" -- block: +#### + +- block: #HN while state is "start client" - - name: Get pod info + - name: A17 Get pod info k8s_facts: kind: Pod api_version: v1 @@ -204,28 +215,28 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_pods - - name: Generate uperf xml files + - name: A18 Generate uperf xml files k8s: definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" - block: - - name: Start Client(s) + - name: P19 Start Client(s) k8s: definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" with_items: "{{ server_pods.resources }}" when: workload_args.serviceip is defined and not workload_args.serviceip|default('false') and server_pods.resources|length > 0 - - name: Start Client(s) - ServiceIP + - name: P20 Start Client(s) - ServiceIP k8s: definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" with_items: "{{ serviceip.resources }}" when: workload_args.serviceip is defined and workload_args.serviceip and serviceip.resources|length > 0 when: resource_kind == "pod" - + - block: - - name: Wait for vms to be running.... + - name: V19 Wait for vms to be running.... k8s_facts: kind: VirtualMachineInstance api_version: kubevirt.io/v1alpha3 @@ -234,12 +245,12 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_vms - - name: Generate uperf test files + - name: V20 Generate uperf test files k8s: definition: "{{ lookup('template', 'configmap_script.yml.j2') | from_yaml }}" with_items: "{{ server_vms.resources }}" - - name: Start Client(s) + - name: V21 Start Client(s) k8s: definition: "{{ lookup('template', 'workload_vm.yml.j2') | from_yaml }}" with_items: "{{ server_vms.resources }}" @@ -259,8 +270,8 @@ - block: - - block: - - name: Get client pod status + - block: #P block + - name: P22 Get client pod status k8s_facts: kind: Pod api_version: v1 @@ -269,7 +280,7 @@ - app = uperf-bench-client-{{ trunc_uuid }} register: client_pods - - name: Update resource state + - name: P23 Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -277,20 +288,19 @@ namespace: "{{ operator_namespace }}" status: state: Clients Running - when: "workload_args.pair|default('1')|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" + when: "num_server_pods|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (client_pods | json_query('resources[].status.podIP')|length)" + #when: "workload_args.pair|default('1')|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" - when: resource_kind == "pod" - - - block: + - block: #V block - - name: set complete to false + - name: V22 set complete to false command: "redis-cli set complete false" - - name: Get count of clients ready + - name: V23 Get count of clients ready command: "redis-cli get clients-{{ trunc_uuid }}" register: clients_ready_count - - name: Update resource state + - name: V24 Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -300,16 +310,21 @@ state: Clients Running when: "workload_args.pair|default('1')|int == clients_ready_count.stdout|int" + - name: V14 debug state + debug: + msg: "HN ater V24 {{ resource_state.resources[0].status.state }}" + when: resource_kind == "vm" + when: resource_state.resources[0].status.state == "Waiting for Clients" -- block: +- block: #ALL state = Clients running - - name: Signal workload + - name: A25 Signal workload command: "redis-cli set start true" - - name: Update resource state + - name: A26 Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -322,7 +337,7 @@ - block: - block: - - name: Waiting for pods to complete.... + - name: P27 Waiting for pods to complete.... k8s_facts: kind: pod api_version: v1 @@ -339,12 +354,13 @@ status: state: Cleanup complete: false - when: "workload_args.pair|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + #when: "workload_args.pair|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: "num_server_pods|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" when: resource_kind == "pod" - block: - - name: get complete + - name: V28 get complete command: "redis-cli get complete" register: complete_status @@ -364,7 +380,7 @@ - block: - block: - - name: Get Server Pods + - name: P29 Get Server Pods k8s_facts: kind: Pod api_version: v1 @@ -373,7 +389,7 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_pods - - name: Pod names - to clean + - name: P30 Pod names - to clean set_fact: clean_pods: | [ @@ -382,7 +398,7 @@ {% endfor %} ] - - name: Cleanup run + - name: P31 Cleanup run k8s: kind: pod api_version: v1 From 5891efb38b7f603c8bcfc108e14ccfab786990c3 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Tue, 22 Dec 2020 01:31:06 +0000 Subject: [PATCH 004/249] client affinity and client/server pairinng are working correctly --- roles/uperf/tasks/main.yml | 87 ++++++++++++++++++++++----- roles/uperf/templates/server.yml.j2 | 9 +++ roles/uperf/templates/workload.yml.j2 | 21 ++++--- roles/uperf/vars/main.yml | 1 + 4 files changed, 95 insertions(+), 23 deletions(-) diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index eb9e73310..daeebb41e 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -45,27 +45,30 @@ register: serviceip when: workload_args.serviceip is defined and workload_args.serviceip -### V2 vs V1 -- name: A6 Record num server (V1) pods +# +# "pin" mode exists prior to "scale" mode. If "pin: true", we will +# do the old way using pin_server and pin_client +# +- name: A6 Record num server (V1) pods using workload_args.pair - TBD set_fact: num_server_pods: "{{ workload_args.pair | default('1')|int }}" when: workload_args.max_node is not defined +#### + - name: A7 V2 Setup eligible node (static) list - Tobe replaced by real node list builder set_fact: - server_node_list: [ worker0, worker2 ] + worker_node_list: "{{workload_args.node_list}}" -- name: A8 V2 Record num server pods +- name: A8 V2 Record num server pods using new worker_node_list set_fact: - num_server_pods: "{{ server_node_list|length * workload_args.density | default('1')|int }}" + num_server_pods: "{{ worker_node_list|length * workload_args.density | default('1')|int }}" when: workload_args.max_node is defined - name: A8_1 show num_server_pods debug: var: num_server_pods -### - - block: - name: P8 Create service for server pods @@ -87,7 +90,7 @@ k8s: definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" with_nested: - - "{{ server_node_list }}" + - "{{ worker_node_list }}" - "{{ range(0, workload_args.density | default('1'|int)) | list }}" when: workload_args.max_node is defined @@ -114,6 +117,8 @@ when: resource_state.resources[0].status.state == "Building" and resource_kind == "pod" ########### +# VM remains scale agnostic for now +########### - block: - name: V11 Start Server(s) @@ -144,7 +149,7 @@ ########### -########### +########### mode - block: - name: P13 Get server pods @@ -171,6 +176,7 @@ when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" ######## +# ######## - block: @@ -204,7 +210,7 @@ #### -- block: #HN while state is "start client" +- block: #HN while state is "start client" for pod - name: A17 Get pod info k8s_facts: @@ -219,19 +225,68 @@ k8s: definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" - - block: - - name: P19 Start Client(s) + - block: # Starting Clients" + - set_fact: + cpod_affi_list: [] + + - block: + - name: HN colocate TBD + debug: + msg: "HN colocate TBD" + when: workload_args.colocate is defined and workload_args.colocate|bool == True + + #### generate affinity list + - block: + + - name: Pass 1 - Build client list for node[1:] + set_fact: + cpod_affi_list: "{{ cpod_affi_list + [ item[0]] }}" + with_nested: + - "{{ worker_node_list[1:] }}" + - "{{ range(0, workload_args.density | default('1'|int)) | list }}" + + - name: HNC_0 debug cpod_list + debug: + var: cpod_affi_list + + - name: Pass 2 - Append client list with node[0] to the end + set_fact: + cpod_affi_list: "{{cpod_affi_list + [ item[0] ] }}" + with_nested: + - "{{ worker_node_list[0] }}" + - "{{ range(0, workload_args.density | default('1'|int)) | list }}" + + + - name: HNC_1 debug cpod_list + debug: + var: cpod_affi_list + + + when: workload_args.colocate is not defined or workload_args.colocate|bool == False + #### End generate cpod affinity list + + - name: P19 Start Client(s) w/o serviceIP k8s: definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - with_items: "{{ server_pods.resources }}" + with_together: + - "{{ server_pods.resources }}" + - "{{ cpod_affi_list }}" when: workload_args.serviceip is defined and not workload_args.serviceip|default('false') and server_pods.resources|length > 0 - - name: P20 Start Client(s) - ServiceIP + - name: P20 Start Client(s) with serviceIP k8s: definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - with_items: "{{ serviceip.resources }}" + with_together: + - "{{ serviceip.resources }}" + - "{{ cpod_affi_list }}" + when: workload_args.serviceip is defined and workload_args.serviceip and serviceip.resources|length > 0 + - name: compare worker000 versus pin_server + debug: + msg: "HN equal" + when: worker_node_list[0] == workload_args.pin_server + when: resource_kind == "pod" - block: @@ -270,7 +325,7 @@ - block: - - block: #P block + - block: # Pod block - name: P22 Get client pod status k8s_facts: kind: Pod diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 2d91d2421..4fafff7a1 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -50,6 +50,15 @@ spec: nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_server }}' {% endif %} + +# +# V2 pin server pod to node +# +{% if workload_args.max_node is defined %} + nodeSelector: + kubernetes.io/hostname: '{{ item[0] }}' +{% endif %} + {% if workload_args.serviceip is sameas true %} securityContext: sysctls: diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 20274ee0d..a0a443623 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -3,9 +3,9 @@ kind: Job apiVersion: batch/v1 metadata: {% if workload_args.serviceip is sameas true %} - name: 'uperf-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item[0].spec.clusterIP}}-{{ trunc_uuid }}' {% else %} - name: 'uperf-client-{{item.status.podIP}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item[0].status.podIP}}-{{ trunc_uuid }}' {% endif %} namespace: '{{ operator_namespace }}' spec: @@ -13,7 +13,7 @@ spec: metadata: labels: app: uperf-bench-client-{{ trunc_uuid }} - clientfor: {{ item.metadata.labels.app }} + clientfor: {{ item[0].metadata.labels.app }} type: uperf-bench-client-{{ trunc_uuid }} {% if workload_args.multus.enabled is sameas true %} annotations: @@ -37,7 +37,7 @@ spec: - key: app operator: In values: - - {{ item.metadata.labels.app }} + - {{ item[0].metadata.labels.app }} topologyKey: kubernetes.io/hostname containers: - name: benchmark @@ -83,13 +83,13 @@ spec: args: {% if workload_args.serviceip is sameas true %} - "export serviceip=true; - export h={{item.spec.clusterIP}}; + export h={{item[0].spec.clusterIP}}; {% else %} {% if workload_args.multus.client is defined %} - "export multus_client={{workload_args.multus.client}}; - export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; + export h={{ (item[0]['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; {% else %} - - "export h={{item.status.podIP}}; + - "export h={{item[0].status.podIP}}; {% endif %} {% endif %} {% if workload_args.networkpolicy is defined %} @@ -142,4 +142,11 @@ spec: nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_client }}' {% endif %} + +{% if workload_args.max_node is defined %} + nodeSelector: + kubernetes.io/hostname: '{{ item[1] }}' +{% endif %} + + {% include "metadata.yml.j2" %} diff --git a/roles/uperf/vars/main.yml b/roles/uperf/vars/main.yml index 82fc9c23f..6c879b5c3 100644 --- a/roles/uperf/vars/main.yml +++ b/roles/uperf/vars/main.yml @@ -1,3 +1,4 @@ --- # vars file for bench cleanup: true +worker_node_list: [] From f11b1e2e8d2d27d59f41e5fbe12497bb17d58728 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Wed, 23 Dec 2020 19:39:04 +0000 Subject: [PATCH 005/249] Annotate server with node-idx and use it to derive client affinity --- roles/uperf/tasks/main.yml | 82 +++++++-------------------- roles/uperf/templates/server.yml.j2 | 11 ++-- roles/uperf/templates/workload.yml.j2 | 31 ++++++---- 3 files changed, 46 insertions(+), 78 deletions(-) diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index daeebb41e..51f1d9894 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -56,19 +56,21 @@ #### -- name: A7 V2 Setup eligible node (static) list - Tobe replaced by real node list builder +- name: A7 V2 scale run - Setup eligible node (static) list - Tobe replaced by real node list builder + set_fact: + worker_node_list: "{{workload_args.node_list[0]}}" + when: workload_args.max_node is not defined + +- name: A7 V1 non-scale Setup eligible node (static) list - Tobe replaced by real node list builder set_fact: worker_node_list: "{{workload_args.node_list}}" + when: workload_args.max_node is defined - name: A8 V2 Record num server pods using new worker_node_list set_fact: num_server_pods: "{{ worker_node_list|length * workload_args.density | default('1')|int }}" when: workload_args.max_node is defined -- name: A8_1 show num_server_pods - debug: - var: num_server_pods - - block: - name: P8 Create service for server pods @@ -90,9 +92,12 @@ k8s: definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" with_nested: - - "{{ worker_node_list }}" + - "{{ range(0, worker_node_list|length| default('0'|int)) | list }}" - "{{ range(0, workload_args.density | default('1'|int)) | list }}" - when: workload_args.max_node is defined + # + # Each server annotates a "node_idx" which will allow its peer client + # to derive its affinity according the 'colocate' variable + # ############ @@ -226,67 +231,25 @@ definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" - block: # Starting Clients" - - set_fact: - cpod_affi_list: [] - - - block: - - name: HN colocate TBD - debug: - msg: "HN colocate TBD" - when: workload_args.colocate is defined and workload_args.colocate|bool == True - - #### generate affinity list - - block: - - - name: Pass 1 - Build client list for node[1:] - set_fact: - cpod_affi_list: "{{ cpod_affi_list + [ item[0]] }}" - with_nested: - - "{{ worker_node_list[1:] }}" - - "{{ range(0, workload_args.density | default('1'|int)) | list }}" - - - name: HNC_0 debug cpod_list - debug: - var: cpod_affi_list - - - name: Pass 2 - Append client list with node[0] to the end - set_fact: - cpod_affi_list: "{{cpod_affi_list + [ item[0] ] }}" - with_nested: - - "{{ worker_node_list[0] }}" - - "{{ range(0, workload_args.density | default('1'|int)) | list }}" - - - - name: HNC_1 debug cpod_list - debug: - var: cpod_affi_list - - - when: workload_args.colocate is not defined or workload_args.colocate|bool == False - #### End generate cpod affinity list - - name: P19 Start Client(s) w/o serviceIP k8s: definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - with_together: + with_items: - "{{ server_pods.resources }}" - - "{{ cpod_affi_list }}" when: workload_args.serviceip is defined and not workload_args.serviceip|default('false') and server_pods.resources|length > 0 + # + # Each server annotates a "node_idx". Each peer client will + # derive its affinity according the 'colocate' variable. + # + - name: P20 Start Client(s) with serviceIP k8s: definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - with_together: + with_items: - "{{ serviceip.resources }}" - - "{{ cpod_affi_list }}" - when: workload_args.serviceip is defined and workload_args.serviceip and serviceip.resources|length > 0 - - name: compare worker000 versus pin_server - debug: - msg: "HN equal" - when: worker_node_list[0] == workload_args.pin_server - when: resource_kind == "pod" - block: @@ -365,13 +328,6 @@ state: Clients Running when: "workload_args.pair|default('1')|int == clients_ready_count.stdout|int" - - name: V14 debug state - debug: - msg: "HN ater V24 {{ resource_state.resources[0].status.state }}" - - when: resource_kind == "vm" - - when: resource_state.resources[0].status.state == "Waiting for Clients" - block: #ALL state = Clients running diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 4fafff7a1..8a67fbf14 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -6,7 +6,7 @@ metadata: {% if workload_args.max_node is not defined %} name: 'uperf-server-{{ item }}-{{ trunc_uuid }}' {% else %} - name: 'uperf-server-{{ item[0] }}-{{ item[1] }}-{{ trunc_uuid }}' + name: 'uperf-server-{{worker_node_list[ item[0]] }}-{{ item[1] }}-{{ trunc_uuid }}' {% endif %} namespace: "{{ operator_namespace }}" @@ -20,14 +20,16 @@ spec: {% if workload_args.max_node is not defined %} app: uperf-bench-server-{{item}}-{{ trunc_uuid }} {% else %} - app: uperf-bench-server-{{ item[0] }}-{{ item[1] }}-{{ trunc_uuid }} + #app: uperf-bench-server-{{ item[0] }}-{{ item[1] }}-{{ trunc_uuid }} + app: uperf-bench-server-{{ worker_node_list[item[0]] }}-{{ item[1] }}-{{ trunc_uuid }} {% endif %} type: uperf-bench-server-{{ trunc_uuid }} -{% if workload_args.multus.enabled is sameas true %} annotations: +{% if workload_args.multus.enabled is sameas true %} k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} {% endif %} + node_idx: '{{ item[0] }}' spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -56,7 +58,8 @@ spec: # {% if workload_args.max_node is defined %} nodeSelector: - kubernetes.io/hostname: '{{ item[0] }}' + #kubernetes.io/hostname: '{{ item[0] }}' + kubernetes.io/hostname: '{{ worker_node_list[item[0]] }}' {% endif %} {% if workload_args.serviceip is sameas true %} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index a0a443623..199cda7aa 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -3,9 +3,9 @@ kind: Job apiVersion: batch/v1 metadata: {% if workload_args.serviceip is sameas true %} - name: 'uperf-client-{{item[0].spec.clusterIP}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}' {% else %} - name: 'uperf-client-{{item[0].status.podIP}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.status.podIP}}-{{ trunc_uuid }}' {% endif %} namespace: '{{ operator_namespace }}' spec: @@ -13,7 +13,7 @@ spec: metadata: labels: app: uperf-bench-client-{{ trunc_uuid }} - clientfor: {{ item[0].metadata.labels.app }} + clientfor: {{ item.metadata.labels.app }} type: uperf-bench-client-{{ trunc_uuid }} {% if workload_args.multus.enabled is sameas true %} annotations: @@ -37,7 +37,7 @@ spec: - key: app operator: In values: - - {{ item[0].metadata.labels.app }} + - {{ item.metadata.labels.app }} topologyKey: kubernetes.io/hostname containers: - name: benchmark @@ -83,13 +83,13 @@ spec: args: {% if workload_args.serviceip is sameas true %} - "export serviceip=true; - export h={{item[0].spec.clusterIP}}; + export h={{item.spec.clusterIP}}; {% else %} {% if workload_args.multus.client is defined %} - "export multus_client={{workload_args.multus.client}}; - export h={{ (item[0]['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; + export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; {% else %} - - "export h={{item[0].status.podIP}}; + - "export h={{item.status.podIP}}; {% endif %} {% endif %} {% if workload_args.networkpolicy is defined %} @@ -138,14 +138,23 @@ spec: configMap: name: uperf-test-{{ trunc_uuid }} restartPolicy: OnFailure -{% if workload_args.pin is sameas true %} +{% if workload_args.max_node is defined %} +{% if workload_args.colocate is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + # client node same as server node + kubernetes.io/hostname: "{{ worker_node_list[item['metadata']['annotations']['node_idx'] | from_json] }}" +{% else %} + nodeSelector: + # skew client node one position left in the woker_node_list + kubernetes.io/hostname: "{{ worker_node_list[ (1+(item['metadata']['annotations']['node_idx'] | from_json)) % (worker_node_list|length)] }}" {% endif %} -{% if workload_args.max_node is defined %} +{% else %} +{% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ item[1] }}' + kubernetes.io/hostname: '{{ workload_args.pin_client }}' +{% endif %} + {% endif %} From 59830ae67f9192f704318606a7b255e0b95d941f Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Wed, 6 Jan 2021 02:50:18 +0000 Subject: [PATCH 006/249] restructure main, add stm to main and client to support iterations --- roles/uperf/tasks/cleanup.yml | 45 ++ roles/uperf/tasks/main.yml | 445 ++----------------- roles/uperf/tasks/next_set.yml | 59 +++ roles/uperf/tasks/run_a_set.yml | 27 ++ roles/uperf/tasks/send_client_run_signal.yml | 18 + roles/uperf/tasks/setup.yml | 71 +++ roles/uperf/tasks/start_client.yml | 76 ++++ roles/uperf/tasks/start_server.yml | 104 +++++ roles/uperf/tasks/wait_client_done.yml | 45 ++ roles/uperf/tasks/wait_client_ready.yml | 55 +++ roles/uperf/tasks/wait_server_ready.yml | 55 +++ roles/uperf/tasks/wait_set_done.yml | 31 ++ roles/uperf/templates/workload.yml.j2 | 20 +- roles/uperf/vars/main.yml | 3 + 14 files changed, 645 insertions(+), 409 deletions(-) create mode 100644 roles/uperf/tasks/cleanup.yml create mode 100644 roles/uperf/tasks/next_set.yml create mode 100644 roles/uperf/tasks/run_a_set.yml create mode 100644 roles/uperf/tasks/send_client_run_signal.yml create mode 100644 roles/uperf/tasks/setup.yml create mode 100644 roles/uperf/tasks/start_client.yml create mode 100644 roles/uperf/tasks/start_server.yml create mode 100644 roles/uperf/tasks/wait_client_done.yml create mode 100644 roles/uperf/tasks/wait_client_ready.yml create mode 100644 roles/uperf/tasks/wait_server_ready.yml create mode 100644 roles/uperf/tasks/wait_set_done.yml diff --git a/roles/uperf/tasks/cleanup.yml b/roles/uperf/tasks/cleanup.yml new file mode 100644 index 000000000..ae67c2a6a --- /dev/null +++ b/roles/uperf/tasks/cleanup.yml @@ -0,0 +1,45 @@ +--- + +- block: + + - block: + - name: P29 Get Server Pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: P30 Pod names - to clean + set_fact: + clean_pods: | + [ + {% for item in server_pods.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + + - name: P31 Cleanup run + k8s: + kind: pod + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_pods }}" + when: cleanup + when: resource_kind == "pod" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + + when: resource_state.resources[0].status.state == "Cleanup" + diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index 51f1d9894..f9fa3d045 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -1,438 +1,67 @@ --- -- name: A1 Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state +- include_tasks: setup.yml -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: A3 Get current state - If it has changed - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- name: A4 Capture operator information - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - name = benchmark-operator - register: bo - -- name: A5 Capture ServiceIP - k8s_facts: - kind: Service - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: serviceip - when: workload_args.serviceip is defined and workload_args.serviceip - -# -# "pin" mode exists prior to "scale" mode. If "pin: true", we will -# do the old way using pin_server and pin_client -# -- name: A6 Record num server (V1) pods using workload_args.pair - TBD - set_fact: - num_server_pods: "{{ workload_args.pair | default('1')|int }}" - when: workload_args.max_node is not defined - -#### - -- name: A7 V2 scale run - Setup eligible node (static) list - Tobe replaced by real node list builder - set_fact: - worker_node_list: "{{workload_args.node_list[0]}}" - when: workload_args.max_node is not defined - -- name: A7 V1 non-scale Setup eligible node (static) list - Tobe replaced by real node list builder - set_fact: - worker_node_list: "{{workload_args.node_list}}" - when: workload_args.max_node is defined - -- name: A8 V2 Record num server pods using new worker_node_list - set_fact: - num_server_pods: "{{ worker_node_list|length * workload_args.density | default('1')|int }}" - when: workload_args.max_node is defined +- include_tasks: start_server.yml + when: resource_state.resources[0].status.state == "Building" - block: - - name: P8 Create service for server pods - k8s: - definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}" - with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} - when: workload_args.serviceip is defined and workload_args.serviceip - - - name: P9 Start Server(s) - k8s: - definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" - register: servers - with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} - when: workload_args.max_node is not defined - -############ + - include_tasks: wait_server_ready.yml + when: resource_state.resources[0].status.state == "Starting Servers" - - name: P10 V2 Start Server(s) - total = eligible nodes * density - k8s: - definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" - with_nested: - - "{{ range(0, worker_node_list|length| default('0'|int)) | list }}" - - "{{ range(0, workload_args.density | default('1'|int)) | list }}" - # - # Each server annotates a "node_idx" which will allow its peer client - # to derive its affinity according the 'colocate' variable - # + - include_tasks: start_client.yml + when: resource_state.resources[0].status.state == "Starting Clients" -############ + - include_tasks: wait_client_ready.yml + when: resource_state.resources[0].status.state == "Waiting for Clients" - - name: P11 Wait for pods to be running.... - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_pods - - - name: P12 Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Servers" - - when: resource_state.resources[0].status.state == "Building" and resource_kind == "pod" - -########### -# VM remains scale agnostic for now -########### -- block: + - include_tasks: run_a_set.yml + when: resource_state.resources[0].status.state == "Clients Running" - - name: V11 Start Server(s) - k8s: - definition: "{{ lookup('template', 'server_vm.yml.j2') | from_yaml }}" - register: servers - with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + - include_tasks: wait_set_done.yml + when: resource_state.resources[0].status.state == "Set Running" - - name: V12 Wait for vms to be running.... - k8s_facts: - kind: VirtualMachineInstance - api_version: kubevirt.io/v1alpha3 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_vms - - name: V13 Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Servers" + - include_tasks: next_set.yml + when: resource_state.resources[0].status.state == "Run Next Set" + # will loop back to "Client Running" state, or FALLTHRU to "Running" state below and finish - when: resource_state.resources[0].status.state == "Building" and resource_kind == "vm" +# - include_tasks: send_client_run_signal.yml +# when: resource_state.resources[0].status.state == "Clients Running" -########### -########### mode -- block: + - include_tasks: wait_client_done.yml + when: resource_state.resources[0].status.state == "Running" - - name: P13 Get server pods - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_pods + - include_tasks: cleanup.yml + when: resource_state.resources[0].status.state == "Cleanup" - - name: P14 Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" - when: "num_server_pods|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" - #when: "workload_args.pair|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + when: resource_kind == "pod" - when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" -######## -# -######## - block: - - name: V14 Wait for vms to be running.... - k8s_facts: - kind: VirtualMachineInstance - api_version: kubevirt.io/v1alpha3 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_vms - - - name: V15 Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" - #when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - when: "num_server_pods|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - - - name: V16 blocking client from running uperf - command: "redis-cli set start false" - with_items: "{{ server_vms.resources }}" - #when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - when: "num_server_pods|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - - when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" - -#### - -- block: #HN while state is "start client" for pod - - - name: A17 Get pod info - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_pods - - - name: A18 Generate uperf xml files - k8s: - definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" - - - block: # Starting Clients" - - name: P19 Start Client(s) w/o serviceIP - k8s: - definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - with_items: - - "{{ server_pods.resources }}" - when: workload_args.serviceip is defined and not workload_args.serviceip|default('false') and server_pods.resources|length > 0 - - # - # Each server annotates a "node_idx". Each peer client will - # derive its affinity according the 'colocate' variable. - # - - - name: P20 Start Client(s) with serviceIP - k8s: - definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - with_items: - - "{{ serviceip.resources }}" - when: workload_args.serviceip is defined and workload_args.serviceip and serviceip.resources|length > 0 - - when: resource_kind == "pod" - - - block: - - - name: V19 Wait for vms to be running.... - k8s_facts: - kind: VirtualMachineInstance - api_version: kubevirt.io/v1alpha3 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_vms + - include_tasks: wait_server_ready.yml + when: resource_state.resources[0].status.state == "Starting Servers" - - name: V20 Generate uperf test files - k8s: - definition: "{{ lookup('template', 'configmap_script.yml.j2') | from_yaml }}" - with_items: "{{ server_vms.resources }}" + - include_tasks: start_client.yml + when: resource_state.resources[0].status.state == "Starting Clients" - - name: V21 Start Client(s) - k8s: - definition: "{{ lookup('template', 'workload_vm.yml.j2') | from_yaml }}" - with_items: "{{ server_vms.resources }}" - when: server_vms.resources|length > 0 - - when: resource_kind == "vm" - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Waiting for Clients - - when: resource_state.resources[0].status.state == "Starting Clients" - -- block: - - - block: # Pod block - - name: P22 Get client pod status - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = uperf-bench-client-{{ trunc_uuid }} - register: client_pods - - - name: P23 Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Clients Running - when: "num_server_pods|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (client_pods | json_query('resources[].status.podIP')|length)" - #when: "workload_args.pair|default('1')|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" - - - block: #V block - - - name: V22 set complete to false - command: "redis-cli set complete false" - - - name: V23 Get count of clients ready - command: "redis-cli get clients-{{ trunc_uuid }}" - register: clients_ready_count - - - name: V24 Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Clients Running - when: "workload_args.pair|default('1')|int == clients_ready_count.stdout|int" - - when: resource_state.resources[0].status.state == "Waiting for Clients" - -- block: #ALL state = Clients running - - - name: A25 Signal workload - command: "redis-cli set start true" - - - name: A26 Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running" - - when: resource_state.resources[0].status.state == "Clients Running" - -- block: - - block: - - name: P27 Waiting for pods to complete.... - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = uperf-bench-client-{{ trunc_uuid }} - register: client_pods - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false - #when: "workload_args.pair|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" - when: "num_server_pods|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" - when: resource_kind == "pod" - - - block: - - - name: V28 get complete - command: "redis-cli get complete" - register: complete_status - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false - when: complete_status.stdout == "true" - when: resource_kind == "vm" - - when: resource_state.resources[0].status.state == "Running" - -- block: + - include_tasks: wait_client_ready.yml + when: resource_state.resources[0].status.state == "Waiting for Clients" - - block: - - name: P29 Get Server Pods - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_pods + - include_tasks: send_client_run_signal.yml + #when: resource_state.resources[0].status.state == "Clients Running" + when: resource_state.resources[0].status.state == "Clients" - - name: P30 Pod names - to clean - set_fact: - clean_pods: | - [ - {% for item in server_pods.resources %} - "{{ item['metadata']['name'] }}", - {% endfor %} - ] + - include_tasks: wait_client_done.yml + when: resource_state.resources[0].status.state == "Running" - - name: P31 Cleanup run - k8s: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - state: absent - name: "{{ item }}" - with_items: "{{ clean_pods }}" - when: cleanup - when: resource_kind == "pod" + - include_tasks: cleanup.yml + when: resource_state.resources[0].status.state == "Cleanup" - - name: delete redis keys - command: "redis-cli del {{ item }}" - loop: - - "{{ trunc_uuid }}" - - "clients-{{ trunc_uuid }}" + when: resource_kind == "vm" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: resource_state.resources[0].status.state == "Cleanup" diff --git a/roles/uperf/tasks/next_set.yml b/roles/uperf/tasks/next_set.yml new file mode 100644 index 000000000..7d7e4e544 --- /dev/null +++ b/roles/uperf/tasks/next_set.yml @@ -0,0 +1,59 @@ +--- + +- block: + - name: debug + command: "redis-cli set task next_set" + + - name: read last group_node_count + command: "redis-cli get group_node_count" + register: redis_out + + - name: Compute next run group size + set_fact: + group_node_count: "{{redis_out.stdout|int + 1}}" + + - block: + # + # We have passed max_node - All done + # + - name: Unpause pods to complete + command: "redis-cli set start done" + + - name: Change state to proceed to exit + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Set Running + + when: "group_node_count|int > workload_args.max_node|int" + + - block: + # + # More round(s) to run. + # + + - name: Send redis restart signal + command: "redis-cli set start restart" + + - name: Reset redis num_completion + command: "redis-cli set num_completion 0" + + - name: Set next run group_node_count + command: "redis-cli set group_node_count {{group_node_count}}" + + - name: Change state to run next round + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + + when: "group_node_count|int <= workload_args.max_node|int" + + when: resource_state.resources[0].status.state == "Run Next Set" + diff --git a/roles/uperf/tasks/run_a_set.yml b/roles/uperf/tasks/run_a_set.yml new file mode 100644 index 000000000..1b3114598 --- /dev/null +++ b/roles/uperf/tasks/run_a_set.yml @@ -0,0 +1,27 @@ +--- + +- block: + # + # Entry Condition: + # 1. A previous task has set 'group_node_count' in redis + # 2. All cliest are polling for 'start' to run its workoad + # Output: Clients with idx <= group_node_count will run + # + + - name: Signal group to run + command: "redis-cli set start true " + + - name: Update state to "Set Running" + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Set Running + + - name: debug 2 + command: "redis-cli set state Set_Running" + + when: resource_state.resources[0].status.state == "Clients Running" + diff --git a/roles/uperf/tasks/send_client_run_signal.yml b/roles/uperf/tasks/send_client_run_signal.yml new file mode 100644 index 000000000..043d4c5cd --- /dev/null +++ b/roles/uperf/tasks/send_client_run_signal.yml @@ -0,0 +1,18 @@ +--- + +- block: + + - name: A25 Signal workload + command: "redis-cli set start true" + + - name: A26 Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Running" + + when: resource_state.resources[0].status.state == "Clients Running" + diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml new file mode 100644 index 000000000..64b326b75 --- /dev/null +++ b/roles/uperf/tasks/setup.yml @@ -0,0 +1,71 @@ +--- + +- name: A1 Get current state + k8s_facts: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: '{{ meta.name }}' + namespace: '{{ operator_namespace }}' + register: resource_state + +- operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Building + complete: false + when: resource_state.resources[0].status.state is not defined + +- name: A3 Get current state - If it has changed + k8s_facts: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: '{{ meta.name }}' + namespace: '{{ operator_namespace }}' + register: resource_state + +- name: A4 Capture operator information + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - name = benchmark-operator + register: bo + +- name: A5 Capture ServiceIP + k8s_facts: + kind: Service + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: serviceip + when: workload_args.serviceip is defined and workload_args.serviceip + +# +# "pin" mode exists prior to "scale" mode. If "pin: true", we will +# do the old way using pin_server and pin_client +# +- name: A6 Record num server (V1) pods using workload_args.pair - TBD + set_fact: + num_server_pods: "{{ workload_args.pair | default('1')|int }}" + when: workload_args.max_node is not defined + +- name: A7 V2 scale run - Setup eligible node (static) list - Tobe replaced by real node list builder + set_fact: + worker_node_list: "{{workload_args.node_list[0]}}" + when: workload_args.max_node is not defined + +- name: A7 V1 non-scale Setup eligible node (static) list - Tobe replaced by real node list builder + set_fact: + worker_node_list: "{{workload_args.node_list}}" + when: workload_args.max_node is defined + +- name: A8 V2 Record num server pods using new worker_node_list + set_fact: + num_server_pods: "{{ worker_node_list|length * workload_args.density | default('1')|int }}" + when: workload_args.max_node is defined + diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml new file mode 100644 index 000000000..220ecde4f --- /dev/null +++ b/roles/uperf/tasks/start_client.yml @@ -0,0 +1,76 @@ +--- + +- block: + +### kind + - name: A17 Get pod info + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: A18 Generate uperf xml files + k8s: + definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" + + - block: # Starting Clients" + - name: P19 Start Client(s) w/o serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + with_items: + - "{{ server_pods.resources }}" + when: workload_args.serviceip is defined and not workload_args.serviceip|default('false') and server_pods.resources|length > 0 + + # + # Each server annotates a "node_idx". Each peer client will + # derive its affinity according the 'colocate' variable. + # + + - name: P20 Start Client(s) with serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + with_items: + - "{{ serviceip.resources }}" + when: workload_args.serviceip is defined and workload_args.serviceip and serviceip.resources|length > 0 + + when: resource_kind == "pod" + +### kind + - block: + + - name: V19 Wait for vms to be running.... + k8s_facts: + kind: VirtualMachineInstance + api_version: kubevirt.io/v1alpha3 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: server_vms + + + - name: V20 Generate uperf test files + k8s: + definition: "{{ lookup('template', 'configmap_script.yml.j2') | from_yaml }}" + with_items: "{{ server_vms.resources }}" + + - name: V21 Start Client(s) + k8s: + definition: "{{ lookup('template', 'workload_vm.yml.j2') | from_yaml }}" + with_items: "{{ server_vms.resources }}" + when: server_vms.resources|length > 0 + + when: resource_kind == "vm" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Waiting for Clients + + when: resource_state.resources[0].status.state == "Starting Clients" + diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml new file mode 100644 index 000000000..e54b5b02b --- /dev/null +++ b/roles/uperf/tasks/start_server.yml @@ -0,0 +1,104 @@ +--- + + #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + # Start servers + #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +- block: + - name: init group node count + set_fact: + group_node_count: "{{ workload_args.min_node | default('2')|int }}" + # + # TBD HN range check, colocate and other checks + # + + - name: init redis + command: "redis-cli set group_mark 0" + - name: init redis + command: "redis-cli set start 0" + - name: init redis + command: "redis-cli set num_completion 0" + - name: init redis + command: "redis-cli set group_node_count {{ group_node_count }}" + + - name: P8 Create service for server pods + k8s: + definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + when: workload_args.serviceip is defined and workload_args.serviceip + + - name: P9 Start Server(s) + k8s: + definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" + register: servers + with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + when: workload_args.max_node is not defined + +############ + + - name: P10 V2 Start Server(s) - total = eligible nodes * density + k8s: + definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" + with_nested: + - "{{ range(0, worker_node_list|length| default('0'|int)) | list }}" + - "{{ range(0, workload_args.density | default('1'|int)) | list }}" + # + # Each server annotates a "node_idx" which will allow its peer client + # to derive its affinity according the 'colocate' variable + # + +############ + + - name: P11 Wait for pods to be running.... + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: P12 Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Servers" + + when: resource_state.resources[0].status.state == "Building" and resource_kind == "pod" + +########### +# VM remains scale agnostic for now +########### +- block: + + - name: V11 Start Server(s) + k8s: + definition: "{{ lookup('template', 'server_vm.yml.j2') | from_yaml }}" + register: servers + with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + + - name: V12 Wait for vms to be running.... + k8s_facts: + kind: VirtualMachineInstance + api_version: kubevirt.io/v1alpha3 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: server_vms + + - name: V13 Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Servers" + + when: resource_state.resources[0].status.state == "Building" and resource_kind == "vm" + +########### + + diff --git a/roles/uperf/tasks/wait_client_done.yml b/roles/uperf/tasks/wait_client_done.yml new file mode 100644 index 000000000..1ff18c530 --- /dev/null +++ b/roles/uperf/tasks/wait_client_done.yml @@ -0,0 +1,45 @@ +--- + +- block: +#### kind + - block: + - name: P27 Waiting for pods to complete.... + k8s_facts: + kind: pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-bench-client-{{ trunc_uuid }} + register: client_pods + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Cleanup + complete: false + when: "num_server_pods|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: resource_kind == "pod" + +#### kind + - block: + + - name: V28 get complete + command: "redis-cli get complete" + register: complete_status + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Cleanup + complete: false + when: complete_status.stdout == "true" + when: resource_kind == "vm" + + when: resource_state.resources[0].status.state == "Running" + diff --git a/roles/uperf/tasks/wait_client_ready.yml b/roles/uperf/tasks/wait_client_ready.yml new file mode 100644 index 000000000..e691270da --- /dev/null +++ b/roles/uperf/tasks/wait_client_ready.yml @@ -0,0 +1,55 @@ +--- +- block: + +##### kind + - block: # Pod block + - name: P22 Get client pod status + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-bench-client-{{ trunc_uuid }} + register: client_pods + + - name: P23 Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + when: "num_server_pods|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (client_pods | json_query('resources[].status.podIP')|length)" + + when: resource_kind == "pod" + +##### kind + - block: + + - name: V22 set complete to false + command: "redis-cli set complete false" + + - name: V23 Get client vm status + k8s_facts: + kind: VirtualMachineInstance + api_version: kubevirt.io/v1alpha3 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-bench-client-{{ trunc_uuid }} + register: client_vms + + - name: V24 Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + when: "workload_args.pair|default('1')|int == client_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + + when: resource_kind == "vm" + + when: resource_state.resources[0].status.state == "Waiting for Clients" + diff --git a/roles/uperf/tasks/wait_server_ready.yml b/roles/uperf/tasks/wait_server_ready.yml new file mode 100644 index 000000000..5ec01046b --- /dev/null +++ b/roles/uperf/tasks/wait_server_ready.yml @@ -0,0 +1,55 @@ +--- +########### kind +- block: + + - name: P13 Get server pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: P14 Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Clients" + when: "num_server_pods|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + + when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" + +######## kind +- block: + + - name: V14 Wait for vms to be running.... + k8s_facts: + kind: VirtualMachineInstance + api_version: kubevirt.io/v1alpha3 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: server_vms + + - name: V15 Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Clients" + when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + + - name: V16 blocking client from running uperf + command: "redis-cli set {{ trunc_uuid }} false" + with_items: "{{ server_vms.resources }}" + when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + + when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" and workload_args.pair|default('1')|int|int == 1 + + diff --git a/roles/uperf/tasks/wait_set_done.yml b/roles/uperf/tasks/wait_set_done.yml new file mode 100644 index 000000000..e608c4070 --- /dev/null +++ b/roles/uperf/tasks/wait_set_done.yml @@ -0,0 +1,31 @@ +--- + +- block: + - block: + + - name: debug + command: "redis-cli set task wait_set_done" + + - name: read pod completion count + command: "redis-cli get num_completion" + register: num_completion + + - name: read group_node_count + command: "redis-cli get group_node_count" + register: group_node_count + + - name: debug + command: "redis-cli set read_num_completion {{ num_completion.stdout }}" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Run Next Set + when: "num_completion.stdout|int == group_node_count.stdout|int * workload_args.density|default('1')|int" + when: resource_kind == "pod" + + when: resource_state.resources[0].status.state == "Set Running" + diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 199cda7aa..3ee2afe6d 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -99,7 +99,8 @@ spec: export ips=$(hostname -I); export num_pairs={{workload_args.pair}}; while true; do - if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start) =~ 'true' ]]; then + state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); + if [[ $state =~ 'true' ]]; then {% for test in workload_args.test_types %} {% for proto in workload_args.protos %} {% for size in workload_args.sizes %} @@ -124,6 +125,23 @@ spec: {% endfor %} {% endfor %} {% endfor %} + redis-cli -h {{bo.resources[0].status.podIP}} incr num_completion; + while true; do + state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); + if [[ $state =~ 'restart' ]]; then + break; + elif [[ $state =~ 'done' ]]; then + break; + else + continue; + fi; + done; + if [[ $state =~ 'restart' ]]; then + continue; + fi; + + elif [[ $state =~ 'done' ]]; then + break; else continue; fi; diff --git a/roles/uperf/vars/main.yml b/roles/uperf/vars/main.yml index 6c879b5c3..4a156a383 100644 --- a/roles/uperf/vars/main.yml +++ b/roles/uperf/vars/main.yml @@ -2,3 +2,6 @@ # vars file for bench cleanup: true worker_node_list: [] +group_node_count: 0 +max_node_count: 0 + From 8deb295c75718975460cd4adbd7905103ec4b51c Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Wed, 6 Jan 2021 17:08:01 +0000 Subject: [PATCH 007/249] Make iteration from min_node to max_node working --- roles/uperf/tasks/start_server.yml | 2 +- roles/uperf/templates/workload.yml.j2 | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml index e54b5b02b..04f10d0df 100644 --- a/roles/uperf/tasks/start_server.yml +++ b/roles/uperf/tasks/start_server.yml @@ -6,7 +6,7 @@ - block: - name: init group node count set_fact: - group_node_count: "{{ workload_args.min_node | default('2')|int }}" + group_node_count: "{{ workload_args.min_node | default('1')|int }}" # # TBD HN range check, colocate and other checks # diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 3ee2afe6d..dfc973604 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -96,11 +96,19 @@ spec: export networkpolicy={{workload_args.networkpolicy}}; {% endif %} export hostnet={{workload_args.hostnetwork}}; + mynode={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; export ips=$(hostname -I); export num_pairs={{workload_args.pair}}; + node_limit=0; while true; do state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); if [[ $state =~ 'true' ]]; then + node_limit=$(redis-cli -h {{bo.resources[0].status.podIP}} get group_node_count); + if [[ $node_limit -le $mynode ]]; then + continue; + fi; + + {% for test in workload_args.test_types %} {% for proto in workload_args.protos %} {% for size in workload_args.sizes %} From 4374a0736f5465ef6cf622a9f75bb364b3241f39 Mon Sep 17 00:00:00 2001 From: Logan Blyth Date: Thu, 7 Jan 2021 12:51:26 -0500 Subject: [PATCH 008/249] retrieve list of nodes with role worker --- deploy/25_role.yaml | 13 +++++++++++++ deploy/35_role_binding.yaml | 13 +++++++++++++ roles/uperf/tasks/setup.yml | 31 ++++++++++++++++++++++--------- 3 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 deploy/25_role.yaml create mode 100644 deploy/35_role_binding.yaml diff --git a/deploy/25_role.yaml b/deploy/25_role.yaml new file mode 100644 index 000000000..62b2971ab --- /dev/null +++ b/deploy/25_role.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: benchmark-operator +rules: +- apiGroups: + - "*" + resources: + - "*" + verbs: + - '*' + diff --git a/deploy/35_role_binding.yaml b/deploy/35_role_binding.yaml new file mode 100644 index 000000000..cd0b952a9 --- /dev/null +++ b/deploy/35_role_binding.yaml @@ -0,0 +1,13 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: benchmark-operator +subjects: +- kind: ServiceAccount + name: benchmark-operator + namespace: my-ripsaw +roleRef: + kind: ClusterRole + name: benchmark-operator + apiGroup: rbac.authorization.k8s.io + diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 64b326b75..8b2c47b14 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -35,6 +35,19 @@ - name = benchmark-operator register: bo +- name: List Nodes Labeled as Workers + k8s_info: + api_version: v1 + kind: Node + label_selectors: + - "node-role.kubernetes.io/worker=" + register: node_list + no_log: True + +- name: Isolate Worker Role Hostnames + register: + worker_node_list: "{{ node_list | json_query('resources[].metadata.name') | list }}" + - name: A5 Capture ServiceIP k8s_facts: kind: Service @@ -54,15 +67,15 @@ num_server_pods: "{{ workload_args.pair | default('1')|int }}" when: workload_args.max_node is not defined -- name: A7 V2 scale run - Setup eligible node (static) list - Tobe replaced by real node list builder - set_fact: - worker_node_list: "{{workload_args.node_list[0]}}" - when: workload_args.max_node is not defined - -- name: A7 V1 non-scale Setup eligible node (static) list - Tobe replaced by real node list builder - set_fact: - worker_node_list: "{{workload_args.node_list}}" - when: workload_args.max_node is defined + #- name: A7 V2 scale run - Setup eligible node (static) list - Tobe replaced by real node list builder + # set_fact: + # worker_node_list: "{{workload_args.node_list[0]}}" + # when: workload_args.max_node is not defined + # + #- name: A7 V1 non-scale Setup eligible node (static) list - Tobe replaced by real node list builder + # set_fact: + # worker_node_list: "{{workload_args.node_list}}" + # when: workload_args.max_node is defined - name: A8 V2 Record num server pods using new worker_node_list set_fact: From 1e00142c1273596719504fd27e3d315169f20a65 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Thu, 7 Jan 2021 19:34:11 +0000 Subject: [PATCH 009/249] integrate with worker_node_list builder --- roles/uperf/tasks/setup.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 8b2c47b14..ee3400721 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -45,9 +45,13 @@ no_log: True - name: Isolate Worker Role Hostnames - register: + set_fact: worker_node_list: "{{ node_list | json_query('resources[].metadata.name') | list }}" +- name: Exlude unhealthy nodes i.e. low diskspace + set_fact: + worker_node_list: "{{ worker_node_list | reject('search', workload_args.exluded_node) | list }}" + - name: A5 Capture ServiceIP k8s_facts: kind: Service From 9ecf5039def60c6383590386495999e067977148 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Thu, 7 Jan 2021 20:56:25 +0000 Subject: [PATCH 010/249] fix when no exclude_node is defined --- roles/uperf/tasks/setup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index ee3400721..50087eeb2 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -51,6 +51,7 @@ - name: Exlude unhealthy nodes i.e. low diskspace set_fact: worker_node_list: "{{ worker_node_list | reject('search', workload_args.exluded_node) | list }}" + when: workload_args.exluded_node is defined - name: A5 Capture ServiceIP k8s_facts: From b1679c8e16cc089ba63673300de0fdf839b32f6e Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Thu, 7 Jan 2021 22:25:42 +0000 Subject: [PATCH 011/249] make the excluded_node being a list i.e excluded_node: [node1 node2 ...] --- roles/uperf/tasks/setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 50087eeb2..2c849e8a3 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -50,8 +50,8 @@ - name: Exlude unhealthy nodes i.e. low diskspace set_fact: - worker_node_list: "{{ worker_node_list | reject('search', workload_args.exluded_node) | list }}" - when: workload_args.exluded_node is defined + worker_node_list: "{{ worker_node_list | difference(workload_args.excluded_node[0]) }}" + when: workload_args.excluded_node is defined - name: A5 Capture ServiceIP k8s_facts: From e5e1532f832ca2b170a4c9090cd33b96ab63e1c5 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Thu, 14 Jan 2021 14:12:23 +0000 Subject: [PATCH 012/249] Super Model-3, "model S" seems working. Model-S supports "node_range: [n, m]" and "density_range: [x, y]" to specify enumeration by both dimension. --- roles/uperf/tasks/init.yml | 31 ++++++++++ roles/uperf/tasks/main.yml | 8 +-- roles/uperf/tasks/next_set.yml | 80 +++++++++++++++++++++---- roles/uperf/tasks/run_a_set.yml | 4 +- roles/uperf/tasks/setup.yml | 36 ++++++++++- roles/uperf/tasks/start_client.yml | 2 + roles/uperf/tasks/start_server.yml | 23 +++---- roles/uperf/tasks/wait_client_done.yml | 3 + roles/uperf/tasks/wait_client_ready.yml | 3 + roles/uperf/tasks/wait_server_ready.yml | 7 +++ roles/uperf/tasks/wait_set_done.yml | 13 ++-- roles/uperf/templates/server.yml.j2 | 1 + roles/uperf/templates/workload.yml.j2 | 10 +++- roles/uperf/vars/main.yml | 9 +++ 14 files changed, 187 insertions(+), 43 deletions(-) create mode 100644 roles/uperf/tasks/init.yml diff --git a/roles/uperf/tasks/init.yml b/roles/uperf/tasks/init.yml new file mode 100644 index 000000000..bbb839b47 --- /dev/null +++ b/roles/uperf/tasks/init.yml @@ -0,0 +1,31 @@ +--- + + +- name: Clear start flag + command: "redis-cli set start 0" + +- name: Clear num_completion + command: "redis-cli set num_completion 0" + +# Node +- name: Init node_hi_idx + command: "redis-cli set node_hi_idx {{ node_hi_idx }}" + +- name: Init node_low_idx + command: "redis-cli set node_low_idx {{ node_low_idx }}" + + # Pod +- name: Init pod_hi_idx + command: "redis-cli set pod_hi_idx {{ pod_hi_idx }}" + +- name: Init pod_low_idx + command: "redis-cli set pod_low_idx {{ pod_low_idx }}" + + # Starting Node and Pod +- name: Init node_idx + command: "redis-cli set node_idx {{ node_low_idx }}" + +- name: Init pod_idx + command: "redis-cli set pod_idx {{ pod_low_idx }}" + + diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index f9fa3d045..0c03f6921 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -16,6 +16,8 @@ - include_tasks: wait_client_ready.yml when: resource_state.resources[0].status.state == "Waiting for Clients" + # LOOP BEGIN + # - include_tasks: run_a_set.yml when: resource_state.resources[0].status.state == "Clients Running" @@ -26,10 +28,8 @@ - include_tasks: next_set.yml when: resource_state.resources[0].status.state == "Run Next Set" # will loop back to "Client Running" state, or FALLTHRU to "Running" state below and finish - -# - include_tasks: send_client_run_signal.yml -# when: resource_state.resources[0].status.state == "Clients Running" - + # + # LOOP END - include_tasks: wait_client_done.yml when: resource_state.resources[0].status.state == "Running" diff --git a/roles/uperf/tasks/next_set.yml b/roles/uperf/tasks/next_set.yml index 7d7e4e544..c20162ade 100644 --- a/roles/uperf/tasks/next_set.yml +++ b/roles/uperf/tasks/next_set.yml @@ -1,20 +1,73 @@ --- - +# +# This module logically implements RE-ENTRANT double for loops +# with_items: +# range (node_low_idx, node_hi_idx) +# range (pod_low_idx, pod_hi_idx) +# Each iteration executes one item, and each re-entrance +# continues where it left off. +# - block: - name: debug command: "redis-cli set task next_set" + - name: + set_fact: + all_run_done: False - - name: read last group_node_count - command: "redis-cli get group_node_count" - register: redis_out + - name: read previous pod_idx + command: "redis-cli get pod_idx" + register: redis_pod_idx - - name: Compute next run group size + - name: Increment pod_idx set_fact: - group_node_count: "{{redis_out.stdout|int + 1}}" + pod_idx: "{{redis_pod_idx.stdout|int + 1}}" + + - name: read pod_hi_idx + command: "redis-cli get pod_hi_idx" + register: redis_pod_hi_idx + + - name: Read prev node_idx + command: "redis-cli get node_idx" + register: redis_node_idx + + - name: node_idx + set_fact: + node_idx: "{{redis_node_idx.stdout|int}}" + + - block: + # + # This block starts a new node loop + # + - name: Increment node_idx + set_fact: + node_idx: "{{node_idx|int + 1}}" + + - name: Read node_hi_idx + command: "redis-cli get node_hi_idx" + register: redis_node_hi_idx + + - name: Check node loop for ending condition + set_fact: + all_run_done: True + when: "node_idx|int > redis_node_hi_idx.stdout|int" + + # + # Reset pod_idx AFTER node_idx tasks above, else cond change + # causes it to skip node_idx tasks + # + - name: read pod_low_idx + command: "redis-cli get pod_low_idx" + register: redis_pod_low_idx + + - name: Reset pod_idx to pod_low_idx + set_fact: + pod_idx: "{{redis_pod_low_idx.stdout|int}}" + + when: "pod_idx|int > redis_pod_hi_idx.stdout|int" - block: # - # We have passed max_node - All done + # All done # - name: Unpause pods to complete command: "redis-cli set start done" @@ -28,21 +81,24 @@ status: state: Set Running - when: "group_node_count|int > workload_args.max_node|int" + when: all_run_done == True - block: # # More round(s) to run. # - - name: Send redis restart signal command: "redis-cli set start restart" - name: Reset redis num_completion command: "redis-cli set num_completion 0" - - name: Set next run group_node_count - command: "redis-cli set group_node_count {{group_node_count}}" + # New node_idx value on new node loop ONLY. But we are lazy, just write every time. + - name: Set next run node_idx + command: "redis-cli set node_idx {{node_idx}}" + + - name: Set next run pod_idx + command: "redis-cli set pod_idx {{pod_idx}}" - name: Change state to run next round operator_sdk.util.k8s_status: @@ -53,7 +109,7 @@ status: state: Clients Running - when: "group_node_count|int <= workload_args.max_node|int" + when: all_run_done == False when: resource_state.resources[0].status.state == "Run Next Set" diff --git a/roles/uperf/tasks/run_a_set.yml b/roles/uperf/tasks/run_a_set.yml index 1b3114598..c14dc6c1e 100644 --- a/roles/uperf/tasks/run_a_set.yml +++ b/roles/uperf/tasks/run_a_set.yml @@ -3,9 +3,9 @@ - block: # # Entry Condition: - # 1. A previous task has set 'group_node_count' in redis + # 1. A previous task has set 'node_idx' in redis # 2. All cliest are polling for 'start' to run its workoad - # Output: Clients with idx <= group_node_count will run + # Output: Clients with idx <= node_idx will run # - name: Signal group to run diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 2c849e8a3..0db28dece 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -48,7 +48,7 @@ set_fact: worker_node_list: "{{ node_list | json_query('resources[].metadata.name') | list }}" -- name: Exlude unhealthy nodes i.e. low diskspace +- name: Exclude unhealthy nodes i.e. low diskspace set_fact: worker_node_list: "{{ worker_node_list | difference(workload_args.excluded_node[0]) }}" when: workload_args.excluded_node is defined @@ -63,6 +63,38 @@ register: serviceip when: workload_args.serviceip is defined and workload_args.serviceip +# +# Compute node and pod limits using CR params while taking into account +# of the actual number of nodes available in the system +# +- name: init node idx + set_fact: + node_idx: "{{ workload_args.min_node | default('0')|int }}" + # + # TBD HN range check, colocate and other checks + # + +- name: init pod low idx + set_fact: + pod_low_idx: "{{ workload_args.density_range[0]|int | default('1')|int - 1 }}" + +- name: init pod hi idx + set_fact: + pod_hi_idx: "{{ workload_args.density_range[1]|int | default('1')|int - 1 }}" + +- name: init node low idx + set_fact: + node_low_idx: "{{ workload_args.node_range[0]|int | default('1')|int - 1 }}" + +- name: init node hi idx + set_fact: + node_hi_idx: "{{ workload_args.node_range[1]|int | default('1')|int - 1 }}" + +- name: Adjust node_hi_idx if cluster has less nodes + set_fact: + node_hi_idx: "{{ worker_node_list|length| default('0')|int -1 }}" + when: "node_hi_idx|int >= worker_node_list|length| default('0')|int " + # # "pin" mode exists prior to "scale" mode. If "pin: true", we will # do the old way using pin_server and pin_client @@ -84,6 +116,6 @@ - name: A8 V2 Record num server pods using new worker_node_list set_fact: - num_server_pods: "{{ worker_node_list|length * workload_args.density | default('1')|int }}" + num_server_pods: "{{ (node_hi_idx|int+1) * (pod_hi_idx|int+1) }}" when: workload_args.max_node is defined diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml index 220ecde4f..e40bc068e 100644 --- a/roles/uperf/tasks/start_client.yml +++ b/roles/uperf/tasks/start_client.yml @@ -40,6 +40,8 @@ ### kind - block: + - name: debug + command: "redis-cli set state Starting_Clients" - name: V19 Wait for vms to be running.... k8s_facts: diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml index 04f10d0df..1397ae27f 100644 --- a/roles/uperf/tasks/start_server.yml +++ b/roles/uperf/tasks/start_server.yml @@ -4,21 +4,9 @@ # Start servers #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - block: - - name: init group node count - set_fact: - group_node_count: "{{ workload_args.min_node | default('1')|int }}" - # - # TBD HN range check, colocate and other checks - # - - name: init redis - command: "redis-cli set group_mark 0" - - name: init redis - command: "redis-cli set start 0" - - name: init redis - command: "redis-cli set num_completion 0" - - name: init redis - command: "redis-cli set group_node_count {{ group_node_count }}" + - include_tasks: init.yml + - name: P8 Create service for server pods k8s: @@ -39,8 +27,11 @@ k8s: definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" with_nested: - - "{{ range(0, worker_node_list|length| default('0'|int)) | list }}" - - "{{ range(0, workload_args.density | default('1'|int)) | list }}" + - "{{ range(0, node_hi_idx|int +1) | list }}" + - "{{ range(0, pod_hi_idx|int +1) | list }}" + + #- "{{ range(0, worker_node_list|length| default('0'|int)) | list }}" + #- "{{ range(0, workload_args.density | default('1'|int)) | list }}" # # Each server annotates a "node_idx" which will allow its peer client # to derive its affinity according the 'colocate' variable diff --git a/roles/uperf/tasks/wait_client_done.yml b/roles/uperf/tasks/wait_client_done.yml index 1ff18c530..47f2b3b82 100644 --- a/roles/uperf/tasks/wait_client_done.yml +++ b/roles/uperf/tasks/wait_client_done.yml @@ -1,6 +1,9 @@ --- - block: + - name: debug + command: "redis-cli set task Running" + #### kind - block: - name: P27 Waiting for pods to complete.... diff --git a/roles/uperf/tasks/wait_client_ready.yml b/roles/uperf/tasks/wait_client_ready.yml index e691270da..8668a7e95 100644 --- a/roles/uperf/tasks/wait_client_ready.yml +++ b/roles/uperf/tasks/wait_client_ready.yml @@ -3,6 +3,9 @@ ##### kind - block: # Pod block + - name: debug + command: "redis-cli set state Waiting_for_Clients" + - name: P22 Get client pod status k8s_facts: kind: Pod diff --git a/roles/uperf/tasks/wait_server_ready.yml b/roles/uperf/tasks/wait_server_ready.yml index 5ec01046b..f86601f8c 100644 --- a/roles/uperf/tasks/wait_server_ready.yml +++ b/roles/uperf/tasks/wait_server_ready.yml @@ -2,6 +2,9 @@ ########### kind - block: + - name: debug + command: "redis-cli set state Starting_Servers" + - name: P13 Get server pods k8s_facts: kind: Pod @@ -11,6 +14,10 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_pods + - name: debug + debug: + msg: "{{ num_server_pods }}" + - name: P14 Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 diff --git a/roles/uperf/tasks/wait_set_done.yml b/roles/uperf/tasks/wait_set_done.yml index e608c4070..26dae01f9 100644 --- a/roles/uperf/tasks/wait_set_done.yml +++ b/roles/uperf/tasks/wait_set_done.yml @@ -10,9 +10,13 @@ command: "redis-cli get num_completion" register: num_completion - - name: read group_node_count - command: "redis-cli get group_node_count" - register: group_node_count + - name: read node_idx + command: "redis-cli get node_idx" + register: node_idx + + - name: read pod_idx + command: "redis-cli get pod_idx" + register: pod_idx - name: debug command: "redis-cli set read_num_completion {{ num_completion.stdout }}" @@ -24,7 +28,8 @@ namespace: "{{ operator_namespace }}" status: state: Run Next Set - when: "num_completion.stdout|int == group_node_count.stdout|int * workload_args.density|default('1')|int" + when: "num_completion.stdout|int == ((node_idx.stdout|int+1) * (pod_idx.stdout|int +1))" + #when: "num_completion.stdout|int == node_idx.stdout|int * workload_args.density|default('1')|int" when: resource_kind == "pod" when: resource_state.resources[0].status.state == "Set Running" diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 8a67fbf14..9580eadfd 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -30,6 +30,7 @@ spec: k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} {% endif %} node_idx: '{{ item[0] }}' + pod_idx: '{{ item[1] }}' spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index dfc973604..ba8fdecad 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -96,18 +96,22 @@ spec: export networkpolicy={{workload_args.networkpolicy}}; {% endif %} export hostnet={{workload_args.hostnetwork}}; - mynode={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; + my_node_idx={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; + my_pod_idx={{ (item['metadata']['annotations']['pod_idx'] | from_json) }}; export ips=$(hostname -I); export num_pairs={{workload_args.pair}}; node_limit=0; + pod_limit=0; while true; do state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); if [[ $state =~ 'true' ]]; then - node_limit=$(redis-cli -h {{bo.resources[0].status.podIP}} get group_node_count); - if [[ $node_limit -le $mynode ]]; then + node_limit=$(redis-cli -h {{bo.resources[0].status.podIP}} get node_idx); + pod_limit=$(redis-cli -h {{bo.resources[0].status.podIP}} get pod_idx); + if [[ $my_node_idx -gt $node_limit || $my_pod_idx -gt $pod_limit ]]; then continue; fi; + /bin/echo 'UPERF scale config: num_node=' $node_limit+1' density= {{ pod_idx }}'; {% for test in workload_args.test_types %} {% for proto in workload_args.protos %} diff --git a/roles/uperf/vars/main.yml b/roles/uperf/vars/main.yml index 4a156a383..4490915c1 100644 --- a/roles/uperf/vars/main.yml +++ b/roles/uperf/vars/main.yml @@ -5,3 +5,12 @@ worker_node_list: [] group_node_count: 0 max_node_count: 0 +pod_low_idx: 1 +pod_hi_idx: 1 +node_low_idx: 1 +node_hi_idx: 1 + +node_idx: 0 +pod_idx: 0 + +all_run_done: false From 1bedf64c109028ff44a50fdf186717d7d50d2676 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Fri, 15 Jan 2021 12:54:20 +0000 Subject: [PATCH 013/249] Move pod_hi/low_idx and node_hi/low_idx out of redis and into benchmark context. Fix last commit which introduced node_range and density_range but inadvertedly still used min_node and max_node for condition check i.e "when: max_node is defined" --- .../crds/ripsaw_v1alpha1_ripsaw_crd.yaml | 13 +++++++ roles/uperf/tasks/init.yml | 35 +++++++++-------- roles/uperf/tasks/next_set.yml | 38 +++++-------------- roles/uperf/tasks/run_a_set.yml | 4 +- roles/uperf/tasks/setup.yml | 15 ++------ roles/uperf/tasks/start_server.yml | 4 +- roles/uperf/tasks/wait_client_done.yml | 2 - roles/uperf/tasks/wait_set_done.yml | 16 +------- roles/uperf/templates/server.yml.j2 | 6 +-- roles/uperf/templates/workload.yml.j2 | 2 +- 10 files changed, 52 insertions(+), 83 deletions(-) diff --git a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml b/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml index 417f46c0b..2aa13a7a3 100644 --- a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml +++ b/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml @@ -123,6 +123,19 @@ spec: type: string cerberus: type: string + pod_hi_idx: + type: string + pod_low_idx: + type: string + node_hi_idx: + type: string + node_low_idx: + type: string + pod_idx: + type: string + node_idx: + type: string + additionalPrinterColumns: - name: Type type: string diff --git a/roles/uperf/tasks/init.yml b/roles/uperf/tasks/init.yml index bbb839b47..186efade3 100644 --- a/roles/uperf/tasks/init.yml +++ b/roles/uperf/tasks/init.yml @@ -7,25 +7,24 @@ - name: Clear num_completion command: "redis-cli set num_completion 0" -# Node -- name: Init node_hi_idx - command: "redis-cli set node_hi_idx {{ node_hi_idx }}" - -- name: Init node_low_idx - command: "redis-cli set node_low_idx {{ node_low_idx }}" - - # Pod -- name: Init pod_hi_idx - command: "redis-cli set pod_hi_idx {{ pod_hi_idx }}" - -- name: Init pod_low_idx - command: "redis-cli set pod_low_idx {{ pod_low_idx }}" - - # Starting Node and Pod -- name: Init node_idx +- name: Init node and pod indices in benchmark context + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + pod_hi_idx: "{{pod_hi_idx}}" + pod_low_idx: "{{pod_low_idx}}" + node_hi_idx: "{{node_hi_idx}}" + node_low_idx: "{{node_low_idx}}" + node_idx: "{{node_low_idx}}" + pod_idx: "{{pod_low_idx}}" + +# Set redis starting Node and Pod +- name: Set starting node_idx command: "redis-cli set node_idx {{ node_low_idx }}" -- name: Init pod_idx +- name: Set redis starting pod_idx command: "redis-cli set pod_idx {{ pod_low_idx }}" - diff --git a/roles/uperf/tasks/next_set.yml b/roles/uperf/tasks/next_set.yml index c20162ade..e7977235c 100644 --- a/roles/uperf/tasks/next_set.yml +++ b/roles/uperf/tasks/next_set.yml @@ -8,31 +8,17 @@ # continues where it left off. # - block: - - name: debug - command: "redis-cli set task next_set" - name: set_fact: all_run_done: False - - name: read previous pod_idx - command: "redis-cli get pod_idx" - register: redis_pod_idx - - name: Increment pod_idx set_fact: - pod_idx: "{{redis_pod_idx.stdout|int + 1}}" - - - name: read pod_hi_idx - command: "redis-cli get pod_hi_idx" - register: redis_pod_hi_idx + pod_idx: "{{resource_state.resources[0].status.pod_idx|int +1 }}" - - name: Read prev node_idx - command: "redis-cli get node_idx" - register: redis_node_idx - - - name: node_idx + - name: Read previous node_idx set_fact: - node_idx: "{{redis_node_idx.stdout|int}}" + node_idx: "{{resource_state.resources[0].status.node_idx|int}}" - block: # @@ -42,28 +28,20 @@ set_fact: node_idx: "{{node_idx|int + 1}}" - - name: Read node_hi_idx - command: "redis-cli get node_hi_idx" - register: redis_node_hi_idx - - name: Check node loop for ending condition set_fact: all_run_done: True - when: "node_idx|int > redis_node_hi_idx.stdout|int" + when: "node_idx|int > resource_state.resources[0].status.node_hi_idx|int" # # Reset pod_idx AFTER node_idx tasks above, else cond change # causes it to skip node_idx tasks # - - name: read pod_low_idx - command: "redis-cli get pod_low_idx" - register: redis_pod_low_idx - - name: Reset pod_idx to pod_low_idx set_fact: - pod_idx: "{{redis_pod_low_idx.stdout|int}}" + pod_idx: "{{resource_state.resources[0].status.pod_low_idx}}" - when: "pod_idx|int > redis_pod_hi_idx.stdout|int" + when: "pod_idx|int > resource_state.resources[0].status.pod_hi_idx|int" - block: # @@ -93,7 +71,7 @@ - name: Reset redis num_completion command: "redis-cli set num_completion 0" - # New node_idx value on new node loop ONLY. But we are lazy, just write every time. + # New node_idx value on new node loop ONLY. But we simply write every time. - name: Set next run node_idx command: "redis-cli set node_idx {{node_idx}}" @@ -108,6 +86,8 @@ namespace: "{{ operator_namespace }}" status: state: Clients Running + pod_idx: "{{pod_idx}}" + node_idx: "{{node_idx}}" when: all_run_done == False diff --git a/roles/uperf/tasks/run_a_set.yml b/roles/uperf/tasks/run_a_set.yml index c14dc6c1e..47f415c4a 100644 --- a/roles/uperf/tasks/run_a_set.yml +++ b/roles/uperf/tasks/run_a_set.yml @@ -3,9 +3,9 @@ - block: # # Entry Condition: - # 1. A previous task has set 'node_idx' in redis + # 1. A previous task has set 'node_idx' and 'pod_idx' in redis # 2. All cliest are polling for 'start' to run its workoad - # Output: Clients with idx <= node_idx will run + # Output: Clients with node_idx <= redis node_idx && pod_idx <= redis pod_ix # - name: Signal group to run diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 0db28dece..3ae1c11ae 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -67,13 +67,6 @@ # Compute node and pod limits using CR params while taking into account # of the actual number of nodes available in the system # -- name: init node idx - set_fact: - node_idx: "{{ workload_args.min_node | default('0')|int }}" - # - # TBD HN range check, colocate and other checks - # - - name: init pod low idx set_fact: pod_low_idx: "{{ workload_args.density_range[0]|int | default('1')|int - 1 }}" @@ -102,20 +95,20 @@ - name: A6 Record num server (V1) pods using workload_args.pair - TBD set_fact: num_server_pods: "{{ workload_args.pair | default('1')|int }}" - when: workload_args.max_node is not defined + when: workload_args.node_range is not defined #- name: A7 V2 scale run - Setup eligible node (static) list - Tobe replaced by real node list builder # set_fact: # worker_node_list: "{{workload_args.node_list[0]}}" - # when: workload_args.max_node is not defined + # when: workload_args.node_range is not defined # #- name: A7 V1 non-scale Setup eligible node (static) list - Tobe replaced by real node list builder # set_fact: # worker_node_list: "{{workload_args.node_list}}" - # when: workload_args.max_node is defined + # when: workload_args.node_range is defined - name: A8 V2 Record num server pods using new worker_node_list set_fact: num_server_pods: "{{ (node_hi_idx|int+1) * (pod_hi_idx|int+1) }}" - when: workload_args.max_node is defined + when: workload_args.node_range is defined diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml index 1397ae27f..f73e2a657 100644 --- a/roles/uperf/tasks/start_server.yml +++ b/roles/uperf/tasks/start_server.yml @@ -19,7 +19,7 @@ definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" register: servers with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} - when: workload_args.max_node is not defined + when: workload_args.node_range is not defined ############ @@ -30,8 +30,6 @@ - "{{ range(0, node_hi_idx|int +1) | list }}" - "{{ range(0, pod_hi_idx|int +1) | list }}" - #- "{{ range(0, worker_node_list|length| default('0'|int)) | list }}" - #- "{{ range(0, workload_args.density | default('1'|int)) | list }}" # # Each server annotates a "node_idx" which will allow its peer client # to derive its affinity according the 'colocate' variable diff --git a/roles/uperf/tasks/wait_client_done.yml b/roles/uperf/tasks/wait_client_done.yml index 47f2b3b82..08c66c0f7 100644 --- a/roles/uperf/tasks/wait_client_done.yml +++ b/roles/uperf/tasks/wait_client_done.yml @@ -1,8 +1,6 @@ --- - block: - - name: debug - command: "redis-cli set task Running" #### kind - block: diff --git a/roles/uperf/tasks/wait_set_done.yml b/roles/uperf/tasks/wait_set_done.yml index 26dae01f9..5320a495d 100644 --- a/roles/uperf/tasks/wait_set_done.yml +++ b/roles/uperf/tasks/wait_set_done.yml @@ -4,23 +4,12 @@ - block: - name: debug - command: "redis-cli set task wait_set_done" + command: "redis-cli set state Set_Running" - name: read pod completion count command: "redis-cli get num_completion" register: num_completion - - name: read node_idx - command: "redis-cli get node_idx" - register: node_idx - - - name: read pod_idx - command: "redis-cli get pod_idx" - register: pod_idx - - - name: debug - command: "redis-cli set read_num_completion {{ num_completion.stdout }}" - - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -28,8 +17,7 @@ namespace: "{{ operator_namespace }}" status: state: Run Next Set - when: "num_completion.stdout|int == ((node_idx.stdout|int+1) * (pod_idx.stdout|int +1))" - #when: "num_completion.stdout|int == node_idx.stdout|int * workload_args.density|default('1')|int" + when: "num_completion.stdout|int == ((resource_state.resources[0].status.node_idx|int +1) * (resource_state.resources[0].status.pod_idx|int +1))" when: resource_kind == "pod" when: resource_state.resources[0].status.state == "Set Running" diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 9580eadfd..cc4f7ccf8 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -3,7 +3,7 @@ kind: Job apiVersion: batch/v1 metadata: -{% if workload_args.max_node is not defined %} +{% if workload_args.node_range is not defined %} name: 'uperf-server-{{ item }}-{{ trunc_uuid }}' {% else %} name: 'uperf-server-{{worker_node_list[ item[0]] }}-{{ item[1] }}-{{ trunc_uuid }}' @@ -17,7 +17,7 @@ spec: metadata: labels: -{% if workload_args.max_node is not defined %} +{% if workload_args.node_range is not defined %} app: uperf-bench-server-{{item}}-{{ trunc_uuid }} {% else %} #app: uperf-bench-server-{{ item[0] }}-{{ item[1] }}-{{ trunc_uuid }} @@ -57,7 +57,7 @@ spec: # # V2 pin server pod to node # -{% if workload_args.max_node is defined %} +{% if workload_args.node_range is defined %} nodeSelector: #kubernetes.io/hostname: '{{ item[0] }}' kubernetes.io/hostname: '{{ worker_node_list[item[0]] }}' diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index ba8fdecad..c12006138 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -168,7 +168,7 @@ spec: configMap: name: uperf-test-{{ trunc_uuid }} restartPolicy: OnFailure -{% if workload_args.max_node is defined %} +{% if workload_args.node_range is defined %} {% if workload_args.colocate is sameas true %} nodeSelector: # client node same as server node From ae1dc5bde178caf8cb41f8823fc3c09efadbdee9 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Fri, 15 Jan 2021 16:19:32 +0000 Subject: [PATCH 014/249] Export node_count and pod_count variabled to benchmarkwrapper for data correlation. While at it beefup a few place with "when: xxx is defined" for robustness --- resources/crds/ripsaw_v1alpha1_uperf_cr.yaml | 9 +++++++++ roles/uperf/tasks/setup.yml | 4 ++++ roles/uperf/tasks/start_server.yml | 2 +- roles/uperf/templates/workload.yml.j2 | 10 +++++++--- roles/uperf/vars/main.yml | 8 ++++---- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml b/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml index bf2606875..c2b80540d 100644 --- a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml +++ b/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml @@ -32,3 +32,12 @@ spec: nthrs: - 1 runtime: 30 + + # The following variables are for scale uperf. + # The scale mode will be activated with 'node_range' defined. + + #colocate: True + #density_range: [1, 2] + #node_range: [1, 2] + #excluded_node: [ worker001 ] + diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 3ae1c11ae..0f20bfbb4 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -70,18 +70,22 @@ - name: init pod low idx set_fact: pod_low_idx: "{{ workload_args.density_range[0]|int | default('1')|int - 1 }}" + when: workload_args.density_range is defined - name: init pod hi idx set_fact: pod_hi_idx: "{{ workload_args.density_range[1]|int | default('1')|int - 1 }}" + when: workload_args.density_range is defined - name: init node low idx set_fact: node_low_idx: "{{ workload_args.node_range[0]|int | default('1')|int - 1 }}" + when: workload_args.node_range is defined - name: init node hi idx set_fact: node_hi_idx: "{{ workload_args.node_range[1]|int | default('1')|int - 1 }}" + when: workload_args.node_range is defined - name: Adjust node_hi_idx if cluster has less nodes set_fact: diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml index f73e2a657..a67651af2 100644 --- a/roles/uperf/tasks/start_server.yml +++ b/roles/uperf/tasks/start_server.yml @@ -29,7 +29,7 @@ with_nested: - "{{ range(0, node_hi_idx|int +1) | list }}" - "{{ range(0, pod_hi_idx|int +1) | list }}" - + when: workload_args.node_range is defined # # Each server annotates a "node_idx" which will allow its peer client # to derive its affinity according the 'colocate' variable diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index c12006138..19fd9b804 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -96,10 +96,12 @@ spec: export networkpolicy={{workload_args.networkpolicy}}; {% endif %} export hostnet={{workload_args.hostnetwork}}; - my_node_idx={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; - my_pod_idx={{ (item['metadata']['annotations']['pod_idx'] | from_json) }}; + export my_node_idx={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; + export my_pod_idx={{ (item['metadata']['annotations']['pod_idx'] | from_json) }}; export ips=$(hostname -I); export num_pairs={{workload_args.pair}}; + export node_count=0 + export pod_count=0 node_limit=0; pod_limit=0; while true; do @@ -111,7 +113,9 @@ spec: continue; fi; - /bin/echo 'UPERF scale config: num_node=' $node_limit+1' density= {{ pod_idx }}'; + /bin/echo 'UPERF-run-context: num_node=' $((node_limit+1)) 'density=' $((pod_limit+1)) 'my_node_idx=' $my_node_idx 'my_pod_idx=' $my_pod_idx; + node_count=$((node_limit+1)); + pod_count=$((pod_limit+1)); {% for test in workload_args.test_types %} {% for proto in workload_args.protos %} diff --git a/roles/uperf/vars/main.yml b/roles/uperf/vars/main.yml index 4490915c1..09cd865f9 100644 --- a/roles/uperf/vars/main.yml +++ b/roles/uperf/vars/main.yml @@ -5,10 +5,10 @@ worker_node_list: [] group_node_count: 0 max_node_count: 0 -pod_low_idx: 1 -pod_hi_idx: 1 -node_low_idx: 1 -node_hi_idx: 1 +pod_low_idx: 0 +pod_hi_idx: 0 +node_low_idx: 0 +node_hi_idx: 0 node_idx: 0 pod_idx: 0 From 40d652e9381d64fc68121b29d9cc90aa08be375e Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Tue, 19 Jan 2021 04:16:36 +0000 Subject: [PATCH 015/249] Add 'step_size' CR parameter. Valid step_size values are: addN or log2. N can be any decimal number. --- resources/crds/ripsaw_v1alpha1_uperf_cr.yaml | 16 +++++++++--- roles/uperf/tasks/next_set.yml | 27 ++++++++++++++------ roles/uperf/tasks/setup.yml | 2 +- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml b/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml index c2b80540d..b4ab9cc3f 100644 --- a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml +++ b/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml @@ -36,8 +36,16 @@ spec: # The following variables are for scale uperf. # The scale mode will be activated with 'node_range' defined. - #colocate: True - #density_range: [1, 2] - #node_range: [1, 2] - #excluded_node: [ worker001 ] + # colocate: True + # density_range: [1, 32] + # node_range: [1, 10] + # step_size: log2 + # Valid step_size values are: addN or log2 + # N can be any decimal number + # Enumeration examples: + # add1: 1,2,3,4 ,,, + # add2: 1,3,5,7 ... + # add10: 1,11,21,31 ... + # log2: 1,2,4,8,16,32 ,,, + # excluded_node: [ worker001 ] diff --git a/roles/uperf/tasks/next_set.yml b/roles/uperf/tasks/next_set.yml index e7977235c..c80868346 100644 --- a/roles/uperf/tasks/next_set.yml +++ b/roles/uperf/tasks/next_set.yml @@ -1,6 +1,6 @@ --- # -# This module logically implements RE-ENTRANT double for loops +# This module logically implements RE-ENTRANT nested for loops # with_items: # range (node_low_idx, node_hi_idx) # range (pod_low_idx, pod_hi_idx) @@ -8,25 +8,36 @@ # continues where it left off. # - block: - - name: + - name: Read previous node_idx and pod_idx set_fact: all_run_done: False + inc: "{{workload_args.step_size|default('add1')}}" + amount: 0 + pod_idx: "{{resource_state.resources[0].status.pod_idx|int}}" + node_idx: "{{resource_state.resources[0].status.node_idx|int}}" - - name: Increment pod_idx + - name: Extract add amount set_fact: - pod_idx: "{{resource_state.resources[0].status.pod_idx|int +1 }}" + amount: "{{ inc | regex_replace('[^0-9]', '') }}" + inc: add + when: "'add' in inc" - - name: Read previous node_idx + - name: Increment pod_idx set_fact: - node_idx: "{{resource_state.resources[0].status.node_idx|int}}" - + pod_idx: "{%-if inc=='add' -%}{{pod_idx|int+amount|int}} + {%-elif inc=='log2' -%}{{(pod_idx|int*2)+1}} + {%-else -%}{{pod_idx|int+1}} + {% endif %}" - block: # # This block starts a new node loop # - name: Increment node_idx set_fact: - node_idx: "{{node_idx|int + 1}}" + node_idx: "{%- if inc=='add' -%}{{node_idx|int+amount|int}} + {%- elif inc=='log2' -%}{{(node_idx|int *2)+1}} + {%- else -%}{{node_idx|int+1}} + {% endif %}" - name: Check node loop for ending condition set_fact: diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 0f20bfbb4..8ab10fb1d 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -51,7 +51,7 @@ - name: Exclude unhealthy nodes i.e. low diskspace set_fact: worker_node_list: "{{ worker_node_list | difference(workload_args.excluded_node[0]) }}" - when: workload_args.excluded_node is defined + when: workload_args.excluded_node is defined and workload_args.excluded_node|length > 0 - name: A5 Capture ServiceIP k8s_facts: From 6a94f691401ede97fc7d0c4b04b8cd18b11d0385 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Tue, 19 Jan 2021 18:45:16 +0000 Subject: [PATCH 016/249] Fix a timing window when redis is really busy. The idle client pods enter main loop (due to start-True) then acquire pod_idx and node_idx. They miss out the restart state, and run prematurely --- roles/uperf/tasks/wait_set_done.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/roles/uperf/tasks/wait_set_done.yml b/roles/uperf/tasks/wait_set_done.yml index 5320a495d..ba0a2d6f0 100644 --- a/roles/uperf/tasks/wait_set_done.yml +++ b/roles/uperf/tasks/wait_set_done.yml @@ -10,14 +10,20 @@ command: "redis-cli get num_completion" register: num_completion - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Run Next Set + - block: + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Run Next Set + + - name: Change redis state to force idle client pods to outside their main loop + command: "redis-cli set start 0" + when: "num_completion.stdout|int == ((resource_state.resources[0].status.node_idx|int +1) * (resource_state.resources[0].status.pod_idx|int +1))" + when: resource_kind == "pod" when: resource_state.resources[0].status.state == "Set Running" From 622769b1e677caae867dc3f268c928d14eabeff9 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Thu, 21 Jan 2021 01:08:18 +0000 Subject: [PATCH 017/249] Consolidate 3 redis vars 'start', 'node_idx', 'pod_idx' into one to reduce redis load. --- roles/uperf/tasks/init.yml | 6 ------ roles/uperf/tasks/next_set.yml | 9 +-------- roles/uperf/tasks/run_a_set.yml | 2 +- roles/uperf/tasks/wait_set_done.yml | 6 +----- roles/uperf/templates/workload.yml.j2 | 13 ++++++++++--- 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/roles/uperf/tasks/init.yml b/roles/uperf/tasks/init.yml index 186efade3..164c67a43 100644 --- a/roles/uperf/tasks/init.yml +++ b/roles/uperf/tasks/init.yml @@ -21,10 +21,4 @@ node_idx: "{{node_low_idx}}" pod_idx: "{{pod_low_idx}}" -# Set redis starting Node and Pod -- name: Set starting node_idx - command: "redis-cli set node_idx {{ node_low_idx }}" - -- name: Set redis starting pod_idx - command: "redis-cli set pod_idx {{ pod_low_idx }}" diff --git a/roles/uperf/tasks/next_set.yml b/roles/uperf/tasks/next_set.yml index c80868346..3ce613df2 100644 --- a/roles/uperf/tasks/next_set.yml +++ b/roles/uperf/tasks/next_set.yml @@ -68,7 +68,7 @@ name: "{{ meta.name }}" namespace: "{{ operator_namespace }}" status: - state: Set Running + state: Running when: all_run_done == True @@ -82,13 +82,6 @@ - name: Reset redis num_completion command: "redis-cli set num_completion 0" - # New node_idx value on new node loop ONLY. But we simply write every time. - - name: Set next run node_idx - command: "redis-cli set node_idx {{node_idx}}" - - - name: Set next run pod_idx - command: "redis-cli set pod_idx {{pod_idx}}" - - name: Change state to run next round operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 diff --git a/roles/uperf/tasks/run_a_set.yml b/roles/uperf/tasks/run_a_set.yml index 47f415c4a..dcf8f3a59 100644 --- a/roles/uperf/tasks/run_a_set.yml +++ b/roles/uperf/tasks/run_a_set.yml @@ -9,7 +9,7 @@ # - name: Signal group to run - command: "redis-cli set start true " + command: "redis-cli set start true-{{resource_state.resources[0].status.node_idx|int}}-{{resource_state.resources[0].status.pod_idx|int}}" - name: Update state to "Set Running" operator_sdk.util.k8s_status: diff --git a/roles/uperf/tasks/wait_set_done.yml b/roles/uperf/tasks/wait_set_done.yml index ba0a2d6f0..5461cbe44 100644 --- a/roles/uperf/tasks/wait_set_done.yml +++ b/roles/uperf/tasks/wait_set_done.yml @@ -10,17 +10,13 @@ command: "redis-cli get num_completion" register: num_completion - - block: - - operator_sdk.util.k8s_status: + - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark name: "{{ meta.name }}" namespace: "{{ operator_namespace }}" status: state: Run Next Set - - - name: Change redis state to force idle client pods to outside their main loop - command: "redis-cli set start 0" when: "num_completion.stdout|int == ((resource_state.resources[0].status.node_idx|int +1) * (resource_state.resources[0].status.pod_idx|int +1))" diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 19fd9b804..839a52783 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -104,11 +104,14 @@ spec: export pod_count=0 node_limit=0; pod_limit=0; + STR=''; while true; do - state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); + STR=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); + state=$(echo $STR | cut -f1 -d-); if [[ $state =~ 'true' ]]; then - node_limit=$(redis-cli -h {{bo.resources[0].status.podIP}} get node_idx); - pod_limit=$(redis-cli -h {{bo.resources[0].status.podIP}} get pod_idx); + node_limit=$(echo $STR | cut -f2 -d-); + pod_limit=$(echo $STR | cut -f3 -d-); + echo 'state=' $state 'node=' $node_limit 'pod=' $pod_limit; if [[ $my_node_idx -gt $node_limit || $my_pod_idx -gt $pod_limit ]]; then continue; fi; @@ -163,7 +166,11 @@ spec: fi; break; done; +{% if workload_args.node_range is not defined %} redis-cli -h {{bo.resources[0].status.podIP}} set start false" +{% else %} + " +{% endif %} volumeMounts: - name: config-volume mountPath: "/tmp/uperf-test" From 4c027d058ceec2deb57b590bb6f3e08f098fe857 Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy <70236227+mukrishn@users.noreply.github.com> Date: Fri, 22 Jan 2021 11:19:49 -0500 Subject: [PATCH 018/249] single request to N pods (#1) --- roles/uperf/tasks/start_client.yml | 8 +- roles/uperf/tasks/start_server.yml | 9 +- roles/uperf/templates/server.yml.j2 | 109 ++++++---- roles/uperf/templates/workload.yml.j2 | 292 +++++++++++++------------- 4 files changed, 221 insertions(+), 197 deletions(-) diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml index e40bc068e..1018e1f84 100644 --- a/roles/uperf/tasks/start_client.yml +++ b/roles/uperf/tasks/start_client.yml @@ -20,8 +20,8 @@ - name: P19 Start Client(s) w/o serviceIP k8s: definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - with_items: - - "{{ server_pods.resources }}" + vars: + resource_item: "{{ server_pods.resources }}" when: workload_args.serviceip is defined and not workload_args.serviceip|default('false') and server_pods.resources|length > 0 # @@ -32,8 +32,8 @@ - name: P20 Start Client(s) with serviceIP k8s: definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - with_items: - - "{{ serviceip.resources }}" + vars: + resource_item: "{{ serviceip.resources }}" when: workload_args.serviceip is defined and workload_args.serviceip and serviceip.resources|length > 0 when: resource_kind == "pod" diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml index a67651af2..8cf6d6f55 100644 --- a/roles/uperf/tasks/start_server.yml +++ b/roles/uperf/tasks/start_server.yml @@ -18,7 +18,8 @@ k8s: definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" register: servers - with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + vars: + pod_sequence: "{{ workload_args.pair | default('1')|int }}" when: workload_args.node_range is not defined ############ @@ -26,9 +27,9 @@ - name: P10 V2 Start Server(s) - total = eligible nodes * density k8s: definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" - with_nested: - - "{{ range(0, node_hi_idx|int +1) | list }}" - - "{{ range(0, pod_hi_idx|int +1) | list }}" + vars: + pod_sequence: "{{ pod_hi_idx|int +1 }}" + node_sequence: "{{ node_hi_idx|int +1 }}" when: workload_args.node_range is defined # # Each server annotates a "node_idx" which will allow its peer client diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index cc4f7ccf8..9b803a184 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -1,72 +1,89 @@ --- -kind: Job -apiVersion: batch/v1 -metadata: - +apiVersion: v1 +kind: List +metadata: {} +items: +{% macro job_template(item, node_idx_item='') %} + - kind: Job + apiVersion: batch/v1 + metadata: {% if workload_args.node_range is not defined %} - name: 'uperf-server-{{ item }}-{{ trunc_uuid }}' + name: 'uperf-server-{{ item }}-{{ trunc_uuid }}' {% else %} - name: 'uperf-server-{{worker_node_list[ item[0]] }}-{{ item[1] }}-{{ trunc_uuid }}' + name: 'uperf-server-{{worker_node_list[ node_idx_item ] }}-{{ item }}-{{ trunc_uuid }}' {% endif %} - - namespace: "{{ operator_namespace }}" -spec: - ttlSecondsAfterFinished: 600 - backoffLimit: 0 - template: - metadata: - labels: - + namespace: "{{ operator_namespace }}" + spec: + ttlSecondsAfterFinished: 600 + backoffLimit: 0 + template: + metadata: + labels: {% if workload_args.node_range is not defined %} - app: uperf-bench-server-{{item}}-{{ trunc_uuid }} + app: uperf-bench-server-{{item}}-{{ trunc_uuid }} {% else %} - #app: uperf-bench-server-{{ item[0] }}-{{ item[1] }}-{{ trunc_uuid }} - app: uperf-bench-server-{{ worker_node_list[item[0]] }}-{{ item[1] }}-{{ trunc_uuid }} + #app: uperf-bench-server-{{ node_idx_item }}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] }}-{{ item }}-{{ trunc_uuid }} {% endif %} - - type: uperf-bench-server-{{ trunc_uuid }} - annotations: + type: uperf-bench-server-{{ trunc_uuid }} + annotations: {% if workload_args.multus.enabled is sameas true %} - k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} {% endif %} - node_idx: '{{ item[0] }}' - pod_idx: '{{ item[1] }}' - spec: +{% if workload_args.node_range is defined %} + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' +{% endif %} + spec: {% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" + runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} {% if workload_args.hostnetwork is sameas true %} - hostNetwork: true - serviceAccountName: benchmark-operator + hostNetwork: true + serviceAccountName: benchmark-operator {% endif %} - containers: - - name: benchmark - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} {% if workload_args.server_resources is defined %} - resources: {{ workload_args.server_resources | to_json }} + resources: {{ workload_args.server_resources | to_json }} {% endif %} - imagePullPolicy: Always - command: ["/bin/sh","-c"] - args: ["uperf -s -v -P 20000"] - restartPolicy: OnFailure + imagePullPolicy: Always + command: ["/bin/sh","-c"] + args: ["uperf -s -v -P 20000"] + restartPolicy: OnFailure {% if workload_args.pin is sameas true %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_server }}' + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_server }}' {% endif %} # # V2 pin server pod to node # {% if workload_args.node_range is defined %} - nodeSelector: - #kubernetes.io/hostname: '{{ item[0] }}' - kubernetes.io/hostname: '{{ worker_node_list[item[0]] }}' + nodeSelector: + #kubernetes.io/hostname: '{{ node_idx_item }}' + kubernetes.io/hostname: '{{ worker_node_list[node_idx_item] }}' {% endif %} {% if workload_args.serviceip is sameas true %} - securityContext: - sysctls: - - name: net.ipv4.ip_local_port_range - value: 20000 20011 + securityContext: + sysctls: + - name: net.ipv4.ip_local_port_range + value: 20000 20011 +{% endif %} +{% macro metadata() %}{% include "metadata.yml.j2" %}{% endmacro %} + {{ metadata()|indent }} +{% endmacro %} +{% if workload_args.node_range is not defined %} +{% for item in range(pod_sequence|int) %} +{{ job_template(item) }} +{% endfor %} +{% else %} +{% for node_idx_item in range(node_sequence|int) %} +{% for item in range(pod_sequence|int) %} +{{ job_template(item,node_idx_item) }} +{% endfor %} +{% endfor %} {% endif %} -{% include "metadata.yml.j2" %} + diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 839a52783..d9d2b5588 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -1,124 +1,129 @@ --- -kind: Job -apiVersion: batch/v1 -metadata: +apiVersion: v1 +kind: List +metadata: {} +items: +{% for item in resource_item %} + - kind: Job + apiVersion: batch/v1 + metadata: {% if workload_args.serviceip is sameas true %} - name: 'uperf-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}' {% else %} - name: 'uperf-client-{{item.status.podIP}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.status.podIP}}-{{ trunc_uuid }}' {% endif %} - namespace: '{{ operator_namespace }}' -spec: - template: - metadata: - labels: - app: uperf-bench-client-{{ trunc_uuid }} - clientfor: {{ item.metadata.labels.app }} - type: uperf-bench-client-{{ trunc_uuid }} + namespace: '{{ operator_namespace }}' + spec: + template: + metadata: + labels: + app: uperf-bench-client-{{ trunc_uuid }} + clientfor: {{ item.metadata.labels.app }} + type: uperf-bench-client-{{ trunc_uuid }} {% if workload_args.multus.enabled is sameas true %} - annotations: - k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} + annotations: + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} {% endif %} - spec: + spec: {% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" + runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} {% if workload_args.hostnetwork is sameas true %} - hostNetwork: true - serviceAccountName: benchmark-operator -{% endif %} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ item.metadata.labels.app }} - topologyKey: kubernetes.io/hostname - containers: - - name: benchmark - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} - env: - - name: uuid - value: "{{ uuid }}" - - name: test_user - value: "{{ test_user | default("ripsaw") }}" - - name: clustername - value: "{{ clustername }}" + hostNetwork: true + serviceAccountName: benchmark-operator +{% endif %} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ item.metadata.labels.app }} + topologyKey: kubernetes.io/hostname + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} + env: + - name: uuid + value: "{{ uuid }}" + - name: test_user + value: "{{ test_user | default("ripsaw") }}" + - name: clustername + value: "{{ clustername }}" {% if elasticsearch is defined %} - - name: es - value: "{{ elasticsearch.url }}" - - name: es_index - value: "{{ elasticsearch.index_name | default("ripsaw-uperf") }}" - - name: parallel - value: "{{ elasticsearch.parallel | default(false) }}" - - name: es_verify_cert - value: "{{ elasticsearch.verify_cert | default(true) }}" + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("ripsaw-uperf") }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(true) }}" {% endif %} {% if prometheus is defined %} - - name: prom_es - value: "{{ prometheus.es_url }}" - - name: prom_parallel - value: "{{ prometheus.es_parallel | default(false) }}" - - name: prom_token - value: "{{ prometheus.prom_token | default() }}" - - name: prom_url - value: "{{ prometheus.prom_url | default() }}" -{% endif %} - - name: client_node - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: server_node - value: "{{ uperf.pin_server|default("unknown") }}" + - name: prom_es + value: "{{ prometheus.es_url }}" + - name: prom_parallel + value: "{{ prometheus.es_parallel | default(false) }}" + - name: prom_token + value: "{{ prometheus.prom_token | default() }}" + - name: prom_url + value: "{{ prometheus.prom_url | default() }}" +{% endif %} + - name: client_node + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: server_node + value: "{{ uperf.pin_server|default("unknown") }}" {% if workload_args.client_resources is defined %} - resources: {{ workload_args.client_resources | to_json }} + resources: {{ workload_args.client_resources | to_json }} {% endif %} - imagePullPolicy: Always - command: ["/bin/sh", "-c"] - args: + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + args: {% if workload_args.serviceip is sameas true %} - - "export serviceip=true; - export h={{item.spec.clusterIP}}; + - "export serviceip=true; + export h={{item.spec.clusterIP}}; {% else %} {% if workload_args.multus.client is defined %} - - "export multus_client={{workload_args.multus.client}}; - export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; + - "export multus_client={{workload_args.multus.client}}; + export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; {% else %} - - "export h={{item.status.podIP}}; + - "export h={{item.status.podIP}}; {% endif %} {% endif %} {% if workload_args.networkpolicy is defined %} - export networkpolicy={{workload_args.networkpolicy}}; -{% endif %} - export hostnet={{workload_args.hostnetwork}}; - export my_node_idx={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; - export my_pod_idx={{ (item['metadata']['annotations']['pod_idx'] | from_json) }}; - export ips=$(hostname -I); - export num_pairs={{workload_args.pair}}; - export node_count=0 - export pod_count=0 - node_limit=0; - pod_limit=0; - STR=''; - while true; do - STR=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); - state=$(echo $STR | cut -f1 -d-); - if [[ $state =~ 'true' ]]; then - node_limit=$(echo $STR | cut -f2 -d-); - pod_limit=$(echo $STR | cut -f3 -d-); - echo 'state=' $state 'node=' $node_limit 'pod=' $pod_limit; - if [[ $my_node_idx -gt $node_limit || $my_pod_idx -gt $pod_limit ]]; then - continue; - fi; - - /bin/echo 'UPERF-run-context: num_node=' $((node_limit+1)) 'density=' $((pod_limit+1)) 'my_node_idx=' $my_node_idx 'my_pod_idx=' $my_pod_idx; - node_count=$((node_limit+1)); - pod_count=$((pod_limit+1)); + export networkpolicy={{workload_args.networkpolicy}}; +{% endif %} + export hostnet={{workload_args.hostnetwork}}; + export my_node_idx={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; + export my_pod_idx={{ (item['metadata']['annotations']['pod_idx'] | from_json) }}; + export ips=$(hostname -I); + export num_pairs={{workload_args.pair}}; + export node_count=0 + export pod_count=0 + node_limit=0; + pod_limit=0; + STR=''; + while true; do + STR=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); + state=$(echo $STR | cut -f1 -d-); + if [[ $state =~ 'true' ]]; then + node_limit=$(echo $STR | cut -f2 -d-); + pod_limit=$(echo $STR | cut -f3 -d-); + echo 'state=' $state 'node=' $node_limit 'pod=' $pod_limit; + if [[ $my_node_idx -gt $node_limit || $my_pod_idx -gt $pod_limit ]]; then + continue; + fi; + + /bin/echo 'UPERF-run-context: num_node=' $((node_limit+1)) 'density=' $((pod_limit+1)) 'my_node_idx=' $my_node_idx 'my_pod_idx=' $my_pod_idx; + node_count=$((node_limit+1)); + pod_count=$((pod_limit+1)); {% for test in workload_args.test_types %} {% for proto in workload_args.protos %} @@ -131,72 +136,73 @@ spec: {% set rsize = size %} {% endif %} {% for nthr in workload_args.nthrs %} - cat /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; - run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ + cat /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; + run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ {% if workload_args.run_id is defined %} - --run-id {{workload_args.run_id}} \ + --run-id {{workload_args.run_id}} \ {% endif %} {% if workload_args.debug is defined and workload_args.debug %} - -v \ + -v \ {% endif %} ; {% endfor %} {% endfor %} {% endfor %} {% endfor %} - redis-cli -h {{bo.resources[0].status.podIP}} incr num_completion; - while true; do - state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); + redis-cli -h {{bo.resources[0].status.podIP}} incr num_completion; + while true; do + state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); + if [[ $state =~ 'restart' ]]; then + break; + elif [[ $state =~ 'done' ]]; then + break; + else + continue; + fi; + done; if [[ $state =~ 'restart' ]]; then - break; - elif [[ $state =~ 'done' ]]; then - break; - else - continue; + continue; fi; - done; - if [[ $state =~ 'restart' ]]; then - continue; + + elif [[ $state =~ 'done' ]]; then + break; + else + continue; fi; - - elif [[ $state =~ 'done' ]]; then break; - else - continue; - fi; - break; - done; + done; {% if workload_args.node_range is not defined %} - redis-cli -h {{bo.resources[0].status.podIP}} set start false" + redis-cli -h {{bo.resources[0].status.podIP}} set start false" {% else %} - " -{% endif %} - volumeMounts: - - name: config-volume - mountPath: "/tmp/uperf-test" - volumes: - - name: config-volume - configMap: - name: uperf-test-{{ trunc_uuid }} - restartPolicy: OnFailure + " +{% endif %} + volumeMounts: + - name: config-volume + mountPath: "/tmp/uperf-test" + volumes: + - name: config-volume + configMap: + name: uperf-test-{{ trunc_uuid }} + restartPolicy: OnFailure {% if workload_args.node_range is defined %} {% if workload_args.colocate is sameas true %} - nodeSelector: - # client node same as server node - kubernetes.io/hostname: "{{ worker_node_list[item['metadata']['annotations']['node_idx'] | from_json] }}" + nodeSelector: + # client node same as server node + kubernetes.io/hostname: "{{ worker_node_list[item['metadata']['annotations']['node_idx'] | from_json] }}" {% else %} - nodeSelector: - # skew client node one position left in the woker_node_list - kubernetes.io/hostname: "{{ worker_node_list[ (1+(item['metadata']['annotations']['node_idx'] | from_json)) % (worker_node_list|length)] }}" + nodeSelector: + # skew client node one position left in the woker_node_list + kubernetes.io/hostname: "{{ worker_node_list[ (1+(item['metadata']['annotations']['node_idx'] | from_json)) % (worker_node_list|length)] }}" {% endif %} {% else %} {% if workload_args.pin is sameas true %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client }}' {% endif %} {% endif %} - -{% include "metadata.yml.j2" %} +{% macro metadata() %}{% include "metadata.yml.j2" %}{% endmacro %} + {{ metadata()|indent }} +{% endfor %} From 6f2006d2f151fda8a1abedd323d4e924253dbef3 Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Fri, 22 Jan 2021 10:20:34 -0600 Subject: [PATCH 019/249] Exclude nodes based on labels (#2) This fits in with the idea that we are viewing worker nodes as just a pool of resources and not by individual hostnames. This gives the flexibility for users to isolate the tests to only one hardware model of workers as well (since each model can be labelled with its model name). Also there was a bug in previous code where only the first node in the list was technically being excluded in `workload_args.excluded_node[0]`. Signed-off-by: Sai Sindhur Malleni --- resources/crds/ripsaw_v1alpha1_uperf_cr.yaml | 4 +++- roles/uperf/tasks/setup.yml | 19 ++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml b/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml index b4ab9cc3f..0d2d88282 100644 --- a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml +++ b/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml @@ -47,5 +47,7 @@ spec: # add2: 1,3,5,7 ... # add10: 1,11,21,31 ... # log2: 1,2,4,8,16,32 ,,, - # excluded_node: [ worker001 ] + # exclude_labels: (OR conditional, every node that matches any of these labels is excluded) + # - "bad=true" + # - "fc640=true" diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 8ab10fb1d..9ea46107a 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -48,10 +48,23 @@ set_fact: worker_node_list: "{{ node_list | json_query('resources[].metadata.name') | list }}" -- name: Exclude unhealthy nodes i.e. low diskspace +- name: List Nodes Labeled with {{ workload_args.exclude_label }} + k8s_info: + api_version: v1 + kind: Node + label_selectors: + - '{{ item }}' + with_items: "{{ workload_args.exclude_labels }}" + register: exclude_node_list + +- name: Isolate Worker Role Hostnames for label {{ workload_args.exclude_label }} + set_fact: + worker_node_exclude_list: "{{ exclude_node_list | json_query('results[].resources[].metadata.name') }}" + +- name: Exclude labelled nodes set_fact: - worker_node_list: "{{ worker_node_list | difference(workload_args.excluded_node[0]) }}" - when: workload_args.excluded_node is defined and workload_args.excluded_node|length > 0 + worker_node_list: "{{ worker_node_list | difference(worker_node_exclude_list) }}" + when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 - name: A5 Capture ServiceIP k8s_facts: From dc674e486c84f1b23c1d8a9a3bcf62bd22ad78c8 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Fri, 22 Jan 2021 12:41:04 -0500 Subject: [PATCH 020/249] Temp alleviating redis-server overload: Increase redis-server to 2 CPU. Add sleep's to workload/uperf clients Point operator resource spec to compatible image. --- resources/operator.yaml | 6 +++--- roles/uperf/tasks/setup.yml | 1 + roles/uperf/templates/workload.yml.j2 | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/resources/operator.yaml b/resources/operator.yaml index f49ac96f0..5df666dd5 100644 --- a/resources/operator.yaml +++ b/resources/operator.yaml @@ -34,14 +34,14 @@ spec: - /usr/local/bin/ao-logs - /tmp/ansible-operator/runner - stdout - image: quay.io/benchmark-operator/benchmark-operator:master + image: quay.io/hnhan/benchmark-operator:uperf-scale imagePullPolicy: "Always" volumeMounts: - mountPath: /tmp/ansible-operator/runner name: runner readOnly: true - name: benchmark-operator - image: quay.io/benchmark-operator/benchmark-operator:master + image: quay.io/hnhan/benchmark-operator:uperf-scale imagePullPolicy: Always env: - name: WATCH_NAMESPACE @@ -70,7 +70,7 @@ spec: - containerPort: 6379 resources: limits: - cpu: "0.1" + cpu: "2.0" volumeMounts: - mountPath: /redis-master-data name: data diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 9ea46107a..74863615c 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -56,6 +56,7 @@ - '{{ item }}' with_items: "{{ workload_args.exclude_labels }}" register: exclude_node_list + when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 - name: Isolate Worker Role Hostnames for label {{ workload_args.exclude_label }} set_fact: diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index d9d2b5588..cf11c1096 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -118,7 +118,7 @@ items: pod_limit=$(echo $STR | cut -f3 -d-); echo 'state=' $state 'node=' $node_limit 'pod=' $pod_limit; if [[ $my_node_idx -gt $node_limit || $my_pod_idx -gt $pod_limit ]]; then - continue; + sleep 0.5; continue; fi; /bin/echo 'UPERF-run-context: num_node=' $((node_limit+1)) 'density=' $((pod_limit+1)) 'my_node_idx=' $my_node_idx 'my_pod_idx=' $my_pod_idx; @@ -157,17 +157,17 @@ items: elif [[ $state =~ 'done' ]]; then break; else - continue; + sleep 0.5; continue; fi; done; if [[ $state =~ 'restart' ]]; then - continue; + sleep 0.5; continue; fi; elif [[ $state =~ 'done' ]]; then break; else - continue; + sleep 0.5; continue; fi; break; done; From 0d2884586d00f618d6f355ada87a57ceb0d6c80c Mon Sep 17 00:00:00 2001 From: Logan Blyth Date: Fri, 22 Jan 2021 14:56:22 -0500 Subject: [PATCH 021/249] add stepsize and colocate to environment variables --- roles/uperf/templates/workload.yml.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index cf11c1096..323a70ef0 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -97,6 +97,12 @@ items: - "export h={{item.status.podIP}}; {% endif %} {% endif %} +{% if (workload_args.colocate is defined) %} + - export colocate={{ workload_args.colocate}}; +{% endif %} +{% if workload_args.step_size is defined %} + - export stepsize={{ workload_args.step_size }}; +{% endif %} {% if workload_args.networkpolicy is defined %} export networkpolicy={{workload_args.networkpolicy}}; {% endif %} From 7a0366e8babe7c5658a206a7ad56a609fa222ade Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Mon, 25 Jan 2021 16:51:59 -0500 Subject: [PATCH 022/249] Massage after rebase --- roles/uperf/tasks/main.yml | 3 +-- roles/uperf/tasks/wait_client_ready.yml | 13 ++++--------- roles/uperf/tasks/wait_server_ready.yml | 4 ++-- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index 0c03f6921..a12f46cc1 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -53,8 +53,7 @@ when: resource_state.resources[0].status.state == "Waiting for Clients" - include_tasks: send_client_run_signal.yml - #when: resource_state.resources[0].status.state == "Clients Running" - when: resource_state.resources[0].status.state == "Clients" + when: resource_state.resources[0].status.state == "Clients Running" - include_tasks: wait_client_done.yml when: resource_state.resources[0].status.state == "Running" diff --git a/roles/uperf/tasks/wait_client_ready.yml b/roles/uperf/tasks/wait_client_ready.yml index 8668a7e95..29aae341e 100644 --- a/roles/uperf/tasks/wait_client_ready.yml +++ b/roles/uperf/tasks/wait_client_ready.yml @@ -33,14 +33,9 @@ - name: V22 set complete to false command: "redis-cli set complete false" - - name: V23 Get client vm status - k8s_facts: - kind: VirtualMachineInstance - api_version: kubevirt.io/v1alpha3 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = uperf-bench-client-{{ trunc_uuid }} - register: client_vms + - name: Get count of clients ready + command: "redis-cli get clients-{{ trunc_uuid }}" + register: clients_ready_count - name: V24 Update resource state operator_sdk.util.k8s_status: @@ -50,7 +45,7 @@ namespace: "{{ operator_namespace }}" status: state: Clients Running - when: "workload_args.pair|default('1')|int == client_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + when: "workload_args.pair|default('1')|int == clients_ready_count.stdout|int" when: resource_kind == "vm" diff --git a/roles/uperf/tasks/wait_server_ready.yml b/roles/uperf/tasks/wait_server_ready.yml index f86601f8c..80aaf55d1 100644 --- a/roles/uperf/tasks/wait_server_ready.yml +++ b/roles/uperf/tasks/wait_server_ready.yml @@ -53,10 +53,10 @@ when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - name: V16 blocking client from running uperf - command: "redis-cli set {{ trunc_uuid }} false" + command: "redis-cli set start false" with_items: "{{ server_vms.resources }}" when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" and workload_args.pair|default('1')|int|int == 1 + when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" From 9d4d506023d8b9fc0ab56b837be4193a2b90e24a Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Wed, 27 Jan 2021 11:53:05 -0600 Subject: [PATCH 023/249] Fix list of args (#3) Signed-off-by: Sai Sindhur Malleni --- roles/uperf/templates/workload.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 323a70ef0..ba9ca2a77 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -98,10 +98,10 @@ items: {% endif %} {% endif %} {% if (workload_args.colocate is defined) %} - - export colocate={{ workload_args.colocate}}; + export colocate={{ workload_args.colocate}}; {% endif %} {% if workload_args.step_size is defined %} - - export stepsize={{ workload_args.step_size }}; + export stepsize={{ workload_args.step_size }}; {% endif %} {% if workload_args.networkpolicy is defined %} export networkpolicy={{workload_args.networkpolicy}}; From 2c7c2ed4d4b1ef01881b2a168ffdb95693df9044 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Thu, 28 Jan 2021 13:31:52 +0000 Subject: [PATCH 024/249] Integrate "pin" mode into the Scale infra. ***Important*** pin=true/false "pin=true" will run Pin mode "pin=false" will run Scale mode In Pin mode, "pair=n" is now obsolete. Use denstiy_range[] instead Scale mode The default values are: node_range=[1,1], density_range=1,1], step_size=add1 and colcate=false --- resources/crds/ripsaw_v1alpha1_uperf_cr.yaml | 27 ++- roles/uperf/tasks/cleanup.yml | 6 +- roles/uperf/tasks/init.yml | 1 - roles/uperf/tasks/run_a_set.yml | 7 +- roles/uperf/tasks/setup.yml | 189 ++++++++++--------- roles/uperf/tasks/start_client.yml | 2 - roles/uperf/tasks/start_server.yml | 27 +-- roles/uperf/tasks/wait_client_done.yml | 6 +- roles/uperf/tasks/wait_client_ready.yml | 14 +- roles/uperf/tasks/wait_server_ready.yml | 13 +- roles/uperf/tasks/wait_set_done.yml | 4 - roles/uperf/templates/server.yml.j2 | 18 -- roles/uperf/templates/workload.yml.j2 | 6 +- roles/uperf/vars/main.yml | 12 +- 14 files changed, 146 insertions(+), 186 deletions(-) diff --git a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml b/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml index 0d2d88282..8e3182195 100644 --- a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml +++ b/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml @@ -15,14 +15,22 @@ spec: serviceip: false hostnetwork: false networkpolicy: false - pin: false multus: enabled: false + pin: false + # + # pin: true/false - default=false + # - true will run 'Pin' mode using 1 pin_server and 1 pin_client nodes. + # - false will run 'Scale' mode. See colocate, density_range, node_range and step_size. pin_server: "node-0" pin_client: "node-1" samples: 1 kind: pod pair: 1 + # + # 'pair' is obsolete. Instead, use 'density_range' which allows fixed or enumerated + # number of pairs. Enumeration is an enhanced capability of the 'Pin' mode. + # test_types: - stream protos: @@ -33,13 +41,14 @@ spec: - 1 runtime: 30 - # The following variables are for scale uperf. - # The scale mode will be activated with 'node_range' defined. - - # colocate: True - # density_range: [1, 32] - # node_range: [1, 10] - # step_size: log2 + # The following variables are for 'Scale' mode. + # The 'Scale' mode is activated when 'pin=false' or undefined. + # The Scale mode op params are: colocate, denstisy_range, node_range and step_size. + # + # colocate: true/false - default=false + # density_range: [n, m] - default=[1,1] + # node_range: [x, y] - default=[1,1] + # step_size: log2 - default=add1 # Valid step_size values are: addN or log2 # N can be any decimal number # Enumeration examples: @@ -47,6 +56,8 @@ spec: # add2: 1,3,5,7 ... # add10: 1,11,21,31 ... # log2: 1,2,4,8,16,32 ,,, + # + # The 'exlude_labels' is the list of ineligible worker nodes. # exclude_labels: (OR conditional, every node that matches any of these labels is excluded) # - "bad=true" # - "fc640=true" diff --git a/roles/uperf/tasks/cleanup.yml b/roles/uperf/tasks/cleanup.yml index ae67c2a6a..5a5dde1bc 100644 --- a/roles/uperf/tasks/cleanup.yml +++ b/roles/uperf/tasks/cleanup.yml @@ -3,7 +3,7 @@ - block: - block: - - name: P29 Get Server Pods + - name: Get Server Pods k8s_facts: kind: Pod api_version: v1 @@ -12,7 +12,7 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_pods - - name: P30 Pod names - to clean + - name: Pod names - to clean set_fact: clean_pods: | [ @@ -21,7 +21,7 @@ {% endfor %} ] - - name: P31 Cleanup run + - name: Cleanup run k8s: kind: pod api_version: v1 diff --git a/roles/uperf/tasks/init.yml b/roles/uperf/tasks/init.yml index 164c67a43..82051961d 100644 --- a/roles/uperf/tasks/init.yml +++ b/roles/uperf/tasks/init.yml @@ -21,4 +21,3 @@ node_idx: "{{node_low_idx}}" pod_idx: "{{pod_low_idx}}" - diff --git a/roles/uperf/tasks/run_a_set.yml b/roles/uperf/tasks/run_a_set.yml index dcf8f3a59..6620505f1 100644 --- a/roles/uperf/tasks/run_a_set.yml +++ b/roles/uperf/tasks/run_a_set.yml @@ -3,8 +3,8 @@ - block: # # Entry Condition: - # 1. A previous task has set 'node_idx' and 'pod_idx' in redis - # 2. All cliest are polling for 'start' to run its workoad + # 1. A previous task has set 'node_idx' and 'pod_idx' in benchmark ctx + # 2. All cliest are polling redis for 'start-node_idx-pod_idx' to start # Output: Clients with node_idx <= redis node_idx && pod_idx <= redis pod_ix # @@ -20,8 +20,5 @@ status: state: Set Running - - name: debug 2 - command: "redis-cli set state Set_Running" - when: resource_state.resources[0].status.state == "Clients Running" diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 74863615c..5f1d3afa7 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -1,6 +1,6 @@ --- -- name: A1 Get current state +- name: Get current state k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -18,7 +18,7 @@ complete: false when: resource_state.resources[0].status.state is not defined -- name: A3 Get current state - If it has changed +- name: Get current state - If it has changed k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -26,7 +26,7 @@ namespace: '{{ operator_namespace }}' register: resource_state -- name: A4 Capture operator information +- name: Capture operator information k8s_facts: kind: Pod api_version: v1 @@ -34,40 +34,102 @@ label_selectors: - name = benchmark-operator register: bo - -- name: List Nodes Labeled as Workers - k8s_info: - api_version: v1 - kind: Node - label_selectors: - - "node-role.kubernetes.io/worker=" - register: node_list - no_log: True - -- name: Isolate Worker Role Hostnames - set_fact: - worker_node_list: "{{ node_list | json_query('resources[].metadata.name') | list }}" - -- name: List Nodes Labeled with {{ workload_args.exclude_label }} - k8s_info: - api_version: v1 - kind: Node - label_selectors: - - '{{ item }}' - with_items: "{{ workload_args.exclude_labels }}" - register: exclude_node_list - when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 - -- name: Isolate Worker Role Hostnames for label {{ workload_args.exclude_label }} - set_fact: - worker_node_exclude_list: "{{ exclude_node_list | json_query('results[].resources[].metadata.name') }}" - -- name: Exclude labelled nodes - set_fact: - worker_node_list: "{{ worker_node_list | difference(worker_node_exclude_list) }}" - when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 - -- name: A5 Capture ServiceIP + +- block: + # + # This block is for scale mode where client and server pods are spreaded + # across all eligible nodes + # + - name: List Nodes Labeled as Workers + k8s_info: + api_version: v1 + kind: Node + label_selectors: + - "node-role.kubernetes.io/worker=" + register: node_list + no_log: True + + - name: Isolate Worker Role Hostnames + set_fact: + worker_node_list: "{{ node_list | json_query('resources[].metadata.name') | list }}" + + - name: List Nodes Labeled with {{ workload_args.exclude_label }} + k8s_info: + api_version: v1 + kind: Node + label_selectors: + - '{{ item }}' + with_items: "{{ workload_args.exclude_labels }}" + register: exclude_node_list + when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 + + - name: Isolate Worker Role Hostnames for label {{ workload_args.exclude_label }} + set_fact: + worker_node_exclude_list: "{{ exclude_node_list | json_query('results[].resources[].metadata.name') }}" + + - name: Exclude labelled nodes + set_fact: + worker_node_list: "{{ worker_node_list | difference(worker_node_exclude_list) }}" + when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 + # + # Compute node and pod limits using CR params while taking into account + # of the actual number of nodes available in the system + # + - name: init pod and node low/hi idx + set_fact: + pod_low_idx: "{{ workload_args.density_range[0] | default('1')|int - 1 }}" + pod_hi_idx: "{{ workload_args.density_range[1] | default('1')|int - 1 }}" + node_low_idx: "{{ workload_args.node_range[0] | default('1')|int - 1 }}" + node_hi_idx: "{{ workload_args.node_range[1] | default('1')|int - 1 }}" + + - name: Adjust node_hi_idx if cluster has less nodes + set_fact: + node_hi_idx: "{{ worker_node_list|length| default('0')|int -1 }}" + when: "node_hi_idx|int >= worker_node_list|length| default('0')|int " + + - name: Record num server pods using new worker_node_list + # in Scale mode, num server pods = num_node * number_pod + set_fact: + num_server_pods: "{{ (node_hi_idx|int+1) * (pod_hi_idx|int+1) }}" + + # + # End scle mode + # + when: workload_args.pin | default(False) == False + +- block: + # + # This block is for the "pin" mode where the server and the client node + # are specified by pin_server and pin_client variables. + + - name: Add "Pin" server and client node to worker list. + # The add order is significant as we will enumerate the server pods on + # the first node in the list, and client pods on the second node. + set_fact: + worker_node_list: "{{worker_node_list + [item]}}" + with_items: + - '{{workload_args.pin_server}}' + - '{{workload_args.pin_client}}' + + - name: Init "Pin" mode node and pod indices + set_fact: + pod_low_idx: "{{ workload_args.density_range[0] | default('1')|int - 1 }}" + pod_hi_idx: "{{ workload_args.density_range[1] | default('1')|int - 1 }}" + # node indices are used as client pod 'start' parameter. + node_low_idx: "0" + node_hi_idx: "0" + + - name: A8 V2 Record num Pin server pods using new worker_node_list + set_fact: + # in Pin mode, num server pods = number of pods + num_server_pods: "{{ pod_hi_idx|int +1 }}" + + # + # End pin mode where pin_client and pin_server are specified + # + when: workload_args.pin | default(False) == True + +- name: Capture ServiceIP k8s_facts: kind: Service api_version: v1 @@ -77,56 +139,3 @@ register: serviceip when: workload_args.serviceip is defined and workload_args.serviceip -# -# Compute node and pod limits using CR params while taking into account -# of the actual number of nodes available in the system -# -- name: init pod low idx - set_fact: - pod_low_idx: "{{ workload_args.density_range[0]|int | default('1')|int - 1 }}" - when: workload_args.density_range is defined - -- name: init pod hi idx - set_fact: - pod_hi_idx: "{{ workload_args.density_range[1]|int | default('1')|int - 1 }}" - when: workload_args.density_range is defined - -- name: init node low idx - set_fact: - node_low_idx: "{{ workload_args.node_range[0]|int | default('1')|int - 1 }}" - when: workload_args.node_range is defined - -- name: init node hi idx - set_fact: - node_hi_idx: "{{ workload_args.node_range[1]|int | default('1')|int - 1 }}" - when: workload_args.node_range is defined - -- name: Adjust node_hi_idx if cluster has less nodes - set_fact: - node_hi_idx: "{{ worker_node_list|length| default('0')|int -1 }}" - when: "node_hi_idx|int >= worker_node_list|length| default('0')|int " - -# -# "pin" mode exists prior to "scale" mode. If "pin: true", we will -# do the old way using pin_server and pin_client -# -- name: A6 Record num server (V1) pods using workload_args.pair - TBD - set_fact: - num_server_pods: "{{ workload_args.pair | default('1')|int }}" - when: workload_args.node_range is not defined - - #- name: A7 V2 scale run - Setup eligible node (static) list - Tobe replaced by real node list builder - # set_fact: - # worker_node_list: "{{workload_args.node_list[0]}}" - # when: workload_args.node_range is not defined - # - #- name: A7 V1 non-scale Setup eligible node (static) list - Tobe replaced by real node list builder - # set_fact: - # worker_node_list: "{{workload_args.node_list}}" - # when: workload_args.node_range is defined - -- name: A8 V2 Record num server pods using new worker_node_list - set_fact: - num_server_pods: "{{ (node_hi_idx|int+1) * (pod_hi_idx|int+1) }}" - when: workload_args.node_range is defined - diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml index 1018e1f84..41bd9068a 100644 --- a/roles/uperf/tasks/start_client.yml +++ b/roles/uperf/tasks/start_client.yml @@ -40,8 +40,6 @@ ### kind - block: - - name: debug - command: "redis-cli set state Starting_Clients" - name: V19 Wait for vms to be running.... k8s_facts: diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml index 8cf6d6f55..f693d7a3e 100644 --- a/roles/uperf/tasks/start_server.yml +++ b/roles/uperf/tasks/start_server.yml @@ -1,44 +1,27 @@ --- - #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - # Start servers - #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - block: - include_tasks: init.yml - - - name: P8 Create service for server pods + - name: Create service for server pods k8s: definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}" with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} when: workload_args.serviceip is defined and workload_args.serviceip - - name: P9 Start Server(s) - k8s: - definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" - register: servers - vars: - pod_sequence: "{{ workload_args.pair | default('1')|int }}" - when: workload_args.node_range is not defined - -############ - - - name: P10 V2 Start Server(s) - total = eligible nodes * density + - name: Start Server(s) - total = eligible nodes * density k8s: definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" vars: pod_sequence: "{{ pod_hi_idx|int +1 }}" node_sequence: "{{ node_hi_idx|int +1 }}" - when: workload_args.node_range is defined + # # Each server annotates a "node_idx" which will allow its peer client # to derive its affinity according the 'colocate' variable # - -############ - - - name: P11 Wait for pods to be running.... + - name: Wait for pods to be running.... k8s_facts: kind: Pod api_version: v1 @@ -47,7 +30,7 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_pods - - name: P12 Update resource state + - name: Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark diff --git a/roles/uperf/tasks/wait_client_done.yml b/roles/uperf/tasks/wait_client_done.yml index 08c66c0f7..4f77c73e8 100644 --- a/roles/uperf/tasks/wait_client_done.yml +++ b/roles/uperf/tasks/wait_client_done.yml @@ -2,9 +2,9 @@ - block: -#### kind + #### kind - block: - - name: P27 Waiting for pods to complete.... + - name: Waiting for pods to complete.... k8s_facts: kind: pod api_version: v1 @@ -24,7 +24,7 @@ when: "num_server_pods|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" when: resource_kind == "pod" -#### kind + #### kind - block: - name: V28 get complete diff --git a/roles/uperf/tasks/wait_client_ready.yml b/roles/uperf/tasks/wait_client_ready.yml index 29aae341e..319abef7f 100644 --- a/roles/uperf/tasks/wait_client_ready.yml +++ b/roles/uperf/tasks/wait_client_ready.yml @@ -1,12 +1,8 @@ --- - block: - -##### kind - - block: # Pod block - - name: debug - command: "redis-cli set state Waiting_for_Clients" - - - name: P22 Get client pod status + ##### kind + - block: + - name: Get client pod status k8s_facts: kind: Pod api_version: v1 @@ -15,7 +11,7 @@ - app = uperf-bench-client-{{ trunc_uuid }} register: client_pods - - name: P23 Update resource state + - name: Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -27,8 +23,8 @@ when: resource_kind == "pod" -##### kind - block: + ##### kind - name: V22 set complete to false command: "redis-cli set complete false" diff --git a/roles/uperf/tasks/wait_server_ready.yml b/roles/uperf/tasks/wait_server_ready.yml index 80aaf55d1..f5b07d095 100644 --- a/roles/uperf/tasks/wait_server_ready.yml +++ b/roles/uperf/tasks/wait_server_ready.yml @@ -1,9 +1,6 @@ --- -########### kind - block: - - - name: debug - command: "redis-cli set state Starting_Servers" + ########### kind - name: P13 Get server pods k8s_facts: @@ -14,11 +11,7 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_pods - - name: debug - debug: - msg: "{{ num_server_pods }}" - - - name: P14 Update resource state + - name: Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -30,8 +23,8 @@ when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" -######## kind - block: + ######## kind - name: V14 Wait for vms to be running.... k8s_facts: diff --git a/roles/uperf/tasks/wait_set_done.yml b/roles/uperf/tasks/wait_set_done.yml index 5461cbe44..72097690c 100644 --- a/roles/uperf/tasks/wait_set_done.yml +++ b/roles/uperf/tasks/wait_set_done.yml @@ -2,10 +2,6 @@ - block: - block: - - - name: debug - command: "redis-cli set state Set_Running" - - name: read pod completion count command: "redis-cli get num_completion" register: num_completion diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 9b803a184..754e47dc4 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -7,11 +7,7 @@ items: - kind: Job apiVersion: batch/v1 metadata: -{% if workload_args.node_range is not defined %} - name: 'uperf-server-{{ item }}-{{ trunc_uuid }}' -{% else %} name: 'uperf-server-{{worker_node_list[ node_idx_item ] }}-{{ item }}-{{ trunc_uuid }}' -{% endif %} namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -19,21 +15,15 @@ items: template: metadata: labels: -{% if workload_args.node_range is not defined %} - app: uperf-bench-server-{{item}}-{{ trunc_uuid }} -{% else %} #app: uperf-bench-server-{{ node_idx_item }}-{{ item }}-{{ trunc_uuid }} app: uperf-bench-server-{{ worker_node_list[node_idx_item] }}-{{ item }}-{{ trunc_uuid }} -{% endif %} type: uperf-bench-server-{{ trunc_uuid }} annotations: {% if workload_args.multus.enabled is sameas true %} k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} {% endif %} -{% if workload_args.node_range is defined %} node_idx: '{{ node_idx_item }}' pod_idx: '{{ item }}' -{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -60,11 +50,9 @@ items: # # V2 pin server pod to node # -{% if workload_args.node_range is defined %} nodeSelector: #kubernetes.io/hostname: '{{ node_idx_item }}' kubernetes.io/hostname: '{{ worker_node_list[node_idx_item] }}' -{% endif %} {% if workload_args.serviceip is sameas true %} securityContext: @@ -75,15 +63,9 @@ items: {% macro metadata() %}{% include "metadata.yml.j2" %}{% endmacro %} {{ metadata()|indent }} {% endmacro %} -{% if workload_args.node_range is not defined %} -{% for item in range(pod_sequence|int) %} -{{ job_template(item) }} -{% endfor %} -{% else %} {% for node_idx_item in range(node_sequence|int) %} {% for item in range(pod_sequence|int) %} {{ job_template(item,node_idx_item) }} {% endfor %} {% endfor %} -{% endif %} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index ba9ca2a77..7d9f18823 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -177,11 +177,7 @@ items: fi; break; done; -{% if workload_args.node_range is not defined %} - redis-cli -h {{bo.resources[0].status.podIP}} set start false" -{% else %} " -{% endif %} volumeMounts: - name: config-volume mountPath: "/tmp/uperf-test" @@ -190,7 +186,7 @@ items: configMap: name: uperf-test-{{ trunc_uuid }} restartPolicy: OnFailure -{% if workload_args.node_range is defined %} +{% if workload_args.pin is sameas false %} {% if workload_args.colocate is sameas true %} nodeSelector: # client node same as server node diff --git a/roles/uperf/vars/main.yml b/roles/uperf/vars/main.yml index 09cd865f9..fe27a3bca 100644 --- a/roles/uperf/vars/main.yml +++ b/roles/uperf/vars/main.yml @@ -5,12 +5,12 @@ worker_node_list: [] group_node_count: 0 max_node_count: 0 -pod_low_idx: 0 -pod_hi_idx: 0 -node_low_idx: 0 -node_hi_idx: 0 +pod_low_idx: "0" +pod_hi_idx: "0" +node_low_idx: "0" +node_hi_idx: "0" -node_idx: 0 -pod_idx: 0 +node_idx: "0" +pod_idx: "0" all_run_done: false From 1f8507d9efa624c34068905b994a2d87bafa9474 Mon Sep 17 00:00:00 2001 From: Logan Blyth Date: Fri, 29 Jan 2021 10:01:26 -0500 Subject: [PATCH 025/249] merged code --- roles/uperf/templates/workload.yml.j2 | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 7d9f18823..4d7059367 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -46,7 +46,7 @@ items: topologyKey: kubernetes.io/hostname containers: - name: benchmark - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} + image: {{ workload_args.image | default('quay.io/lblyth/lblyth-uperf:latest') }} env: - name: uuid value: "{{ uuid }}" @@ -88,11 +88,11 @@ items: args: {% if workload_args.serviceip is sameas true %} - "export serviceip=true; - export h={{item.spec.clusterIP}}; + export h={{item.spec.clusterIP}}; {% else %} {% if workload_args.multus.client is defined %} - "export multus_client={{workload_args.multus.client}}; - export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; + export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; {% else %} - "export h={{item.status.podIP}}; {% endif %} @@ -103,8 +103,17 @@ items: {% if workload_args.step_size is defined %} export stepsize={{ workload_args.step_size }}; {% endif %} +{% if workload_args.step_size is defined %} + export stepsize={{ workload_args.step_size }}; +{% endif %} +{% if workload_args.node_range is defined %} + export node_range={{ workload_args.node_range[0] }}_{{ workload_args.node_range[1] }}; +{% endif %} +{% if workload_args.density_range is defined %} + export density_range={{ workload_args.density_range[0] }}_{{ workload_args.density_range[1] }}; +{% endif %} {% if workload_args.networkpolicy is defined %} - export networkpolicy={{workload_args.networkpolicy}}; + export networkpolicy={{workload_args.networkpolicy}}; {% endif %} export hostnet={{workload_args.hostnetwork}}; export my_node_idx={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; @@ -126,8 +135,8 @@ items: if [[ $my_node_idx -gt $node_limit || $my_pod_idx -gt $pod_limit ]]; then sleep 0.5; continue; fi; - - /bin/echo 'UPERF-run-context: num_node=' $((node_limit+1)) 'density=' $((pod_limit+1)) 'my_node_idx=' $my_node_idx 'my_pod_idx=' $my_pod_idx; + + echo "UPERF-run-context num_node=" $((node_limit+1)) "density=" $((pod_limit+1)) "my_node_idx=" $my_node_idx "my_pod_idx=" $my_pod_idx; node_count=$((node_limit+1)); pod_count=$((pod_limit+1)); @@ -169,7 +178,7 @@ items: if [[ $state =~ 'restart' ]]; then sleep 0.5; continue; fi; - + elif [[ $state =~ 'done' ]]; then break; else From 416a5f460e7e9be864946d22ea40682b31e0efb9 Mon Sep 17 00:00:00 2001 From: Logan Blyth Date: Fri, 29 Jan 2021 15:07:09 -0500 Subject: [PATCH 026/249] more variable changes --- roles/uperf/templates/workload.yml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 4d7059367..513e460a5 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -107,10 +107,10 @@ items: export stepsize={{ workload_args.step_size }}; {% endif %} {% if workload_args.node_range is defined %} - export node_range={{ workload_args.node_range[0] }}_{{ workload_args.node_range[1] }}; + export node_range='{{ workload_args.node_range[0] }}_{{ workload_args.node_range[1] }}'; {% endif %} {% if workload_args.density_range is defined %} - export density_range={{ workload_args.density_range[0] }}_{{ workload_args.density_range[1] }}; + export density_range='{{ workload_args.density_range[0] }}_{{ workload_args.density_range[1] }}'; {% endif %} {% if workload_args.networkpolicy is defined %} export networkpolicy={{workload_args.networkpolicy}}; @@ -136,7 +136,7 @@ items: sleep 0.5; continue; fi; - echo "UPERF-run-context num_node=" $((node_limit+1)) "density=" $((pod_limit+1)) "my_node_idx=" $my_node_idx "my_pod_idx=" $my_pod_idx; + echo 'UPERF-run-context num_node=' $((node_limit+1)) 'density=' $((pod_limit+1)) 'my_node_idx=' $my_node_idx 'my_pod_idx=' $my_pod_idx; node_count=$((node_limit+1)); pod_count=$((pod_limit+1)); From 606974e4358141eb88c5004503189fada5c68b0b Mon Sep 17 00:00:00 2001 From: Logan Blyth Date: Fri, 29 Jan 2021 16:49:47 -0500 Subject: [PATCH 027/249] remove duplicate export --- roles/uperf/templates/workload.yml.j2 | 3 --- 1 file changed, 3 deletions(-) diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 513e460a5..aa47d470f 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -100,9 +100,6 @@ items: {% if (workload_args.colocate is defined) %} export colocate={{ workload_args.colocate}}; {% endif %} -{% if workload_args.step_size is defined %} - export stepsize={{ workload_args.step_size }}; -{% endif %} {% if workload_args.step_size is defined %} export stepsize={{ workload_args.step_size }}; {% endif %} From 106620ff6715ea919307bc54b7ecd20cb8ffb5c2 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Fri, 5 Feb 2021 13:46:08 +0000 Subject: [PATCH 028/249] Verified that VM-type works fine. Integrated "serviceip" into scale framework. Was not working until this commit. Cosmetic cleanup i.e debug TASK tags, and doublle "when" guards. --- roles/uperf/tasks/cleanup.yml | 75 ++++++------ roles/uperf/tasks/init.yml | 1 - roles/uperf/tasks/main.yml | 16 ++- roles/uperf/tasks/next_set.yml | 9 +- roles/uperf/tasks/run_a_set.yml | 8 +- roles/uperf/tasks/send_client_run_signal.yml | 4 +- roles/uperf/tasks/setup.yml | 25 +++- roles/uperf/tasks/start_client.yml | 115 +++++++++---------- roles/uperf/tasks/start_server.yml | 25 ++-- roles/uperf/tasks/wait_client_done.yml | 78 ++++++------- roles/uperf/tasks/wait_client_ready.yml | 91 ++++++++------- roles/uperf/tasks/wait_server_ready.yml | 18 +-- roles/uperf/tasks/wait_set_done.yml | 8 +- roles/uperf/templates/server.yml.j2 | 6 - roles/uperf/templates/service.yml.j2 | 65 +++++++---- roles/uperf/templates/workload.yml.j2 | 5 +- roles/uperf/vars/main.yml | 5 - 17 files changed, 289 insertions(+), 265 deletions(-) diff --git a/roles/uperf/tasks/cleanup.yml b/roles/uperf/tasks/cleanup.yml index 5a5dde1bc..0da5b68ec 100644 --- a/roles/uperf/tasks/cleanup.yml +++ b/roles/uperf/tasks/cleanup.yml @@ -1,45 +1,46 @@ --- - block: + ### kind + - name: Get Server Pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: server_pods - - block: - - name: Get Server Pods - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_pods + - name: Pod names - to clean + set_fact: + clean_pods: | + [ + {% for item in server_pods.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] - - name: Pod names - to clean - set_fact: - clean_pods: | - [ - {% for item in server_pods.resources %} - "{{ item['metadata']['name'] }}", - {% endfor %} - ] + - name: Cleanup run + k8s: + kind: pod + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_pods }}" + when: cleanup + when: resource_kind == "pod" - - name: Cleanup run - k8s: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - state: absent - name: "{{ item }}" - with_items: "{{ clean_pods }}" - when: cleanup - when: resource_kind == "pod" +# +# no kind block - We leave VM running +# - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - - when: resource_state.resources[0].status.state == "Cleanup" +- operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true diff --git a/roles/uperf/tasks/init.yml b/roles/uperf/tasks/init.yml index 82051961d..fbae05ce8 100644 --- a/roles/uperf/tasks/init.yml +++ b/roles/uperf/tasks/init.yml @@ -1,6 +1,5 @@ --- - - name: Clear start flag command: "redis-cli set start 0" diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index a12f46cc1..435a9b6b9 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -16,8 +16,9 @@ - include_tasks: wait_client_ready.yml when: resource_state.resources[0].status.state == "Waiting for Clients" - # LOOP BEGIN - # + # LOOP BEGIN + # This loop iterates density_range[] and node_range[] for "scale" mode + - include_tasks: run_a_set.yml when: resource_state.resources[0].status.state == "Clients Running" @@ -27,9 +28,10 @@ - include_tasks: next_set.yml when: resource_state.resources[0].status.state == "Run Next Set" - # will loop back to "Client Running" state, or FALLTHRU to "Running" state below and finish - # - # LOOP END + # will loop back to "Client Running" state, or FALLTHRU to "Running" + # state below and finish + + # LOOP END - include_tasks: wait_client_done.yml when: resource_state.resources[0].status.state == "Running" @@ -39,7 +41,9 @@ when: resource_kind == "pod" - +# +# kind does not support "scale" mode yet +# - block: diff --git a/roles/uperf/tasks/next_set.yml b/roles/uperf/tasks/next_set.yml index 3ce613df2..df5acbc9c 100644 --- a/roles/uperf/tasks/next_set.yml +++ b/roles/uperf/tasks/next_set.yml @@ -1,9 +1,11 @@ --- # -# This module logically implements RE-ENTRANT nested for loops +# This module logically implements an RE-ENTRANT nested "for" loops; +# # with_items: # range (node_low_idx, node_hi_idx) # range (pod_low_idx, pod_hi_idx) +# # Each iteration executes one item, and each re-entrance # continues where it left off. # @@ -95,5 +97,8 @@ when: all_run_done == False - when: resource_state.resources[0].status.state == "Run Next Set" + when: resource_kind == "pod" +# +# No block - Scale mode support is N/A +# diff --git a/roles/uperf/tasks/run_a_set.yml b/roles/uperf/tasks/run_a_set.yml index 6620505f1..1ec4d9260 100644 --- a/roles/uperf/tasks/run_a_set.yml +++ b/roles/uperf/tasks/run_a_set.yml @@ -5,7 +5,7 @@ # Entry Condition: # 1. A previous task has set 'node_idx' and 'pod_idx' in benchmark ctx # 2. All cliest are polling redis for 'start-node_idx-pod_idx' to start - # Output: Clients with node_idx <= redis node_idx && pod_idx <= redis pod_ix + # Output: Clients with node_idx <= redis node_idx && pod_idx <= redis pod_ix will run # - name: Signal group to run @@ -19,6 +19,8 @@ namespace: "{{ operator_namespace }}" status: state: Set Running + when: resource_kind == "pod" - when: resource_state.resources[0].status.state == "Clients Running" - +# +# No kind - It has not been adapted to Scale mode +# diff --git a/roles/uperf/tasks/send_client_run_signal.yml b/roles/uperf/tasks/send_client_run_signal.yml index 043d4c5cd..52fd92a97 100644 --- a/roles/uperf/tasks/send_client_run_signal.yml +++ b/roles/uperf/tasks/send_client_run_signal.yml @@ -1,8 +1,8 @@ --- - +# This module is invoked by VM-kind only - block: - - name: A25 Signal workload + - name: Signal workload command: "redis-cli set start true" - name: A26 Update resource state diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 5f1d3afa7..be74b0af1 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -67,7 +67,7 @@ set_fact: worker_node_exclude_list: "{{ exclude_node_list | json_query('results[].resources[].metadata.name') }}" - - name: Exclude labelled nodes + - name: Exclude labeled nodes set_fact: worker_node_list: "{{ worker_node_list | difference(worker_node_exclude_list) }}" when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 @@ -81,20 +81,33 @@ pod_hi_idx: "{{ workload_args.density_range[1] | default('1')|int - 1 }}" node_low_idx: "{{ workload_args.node_range[0] | default('1')|int - 1 }}" node_hi_idx: "{{ workload_args.node_range[1] | default('1')|int - 1 }}" - + # + # Next sanity check and massage the indices if necessary. + # We shall complete gracefully and not iterate wildly. + # - name: Adjust node_hi_idx if cluster has less nodes set_fact: node_hi_idx: "{{ worker_node_list|length| default('0')|int -1 }}" when: "node_hi_idx|int >= worker_node_list|length| default('0')|int " + - name: Adjust node_low_idx if necessary + set_fact: + node_low_idx: "{{node_hi_idx|int}}" + when: "node_low_idx|int > node_hi_idx|int" + + - name: Adjust pod_low_idx if necessary + set_fact: + pod_low_idx: "{{pod_hi_idx|int}}" + when: "pod_low_idx|int > pod_hi_idx|int" + - name: Record num server pods using new worker_node_list # in Scale mode, num server pods = num_node * number_pod set_fact: num_server_pods: "{{ (node_hi_idx|int+1) * (pod_hi_idx|int+1) }}" - # - # End scle mode - # + # + # End scale mode + # when: workload_args.pin | default(False) == False - block: @@ -119,7 +132,7 @@ node_low_idx: "0" node_hi_idx: "0" - - name: A8 V2 Record num Pin server pods using new worker_node_list + - name: Record num Pin server pods using new worker_node_list set_fact: # in Pin mode, num server pods = number of pods num_server_pods: "{{ pod_hi_idx|int +1 }}" diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml index 41bd9068a..222621277 100644 --- a/roles/uperf/tasks/start_client.yml +++ b/roles/uperf/tasks/start_client.yml @@ -1,76 +1,73 @@ --- -- block: +- name: Get pod info + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: server_pods -### kind - - name: A17 Get pod info - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_pods +- name: Generate uperf xml files + k8s: + definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" - - name: A18 Generate uperf xml files +- block: + ### kind + - name: Start Client(s) w/o serviceIP k8s: - definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + vars: + resource_item: "{{ server_pods.resources }}" + when: workload_args.serviceip is defined and not workload_args.serviceip|default('false') and server_pods.resources|length > 0 - - block: # Starting Clients" - - name: P19 Start Client(s) w/o serviceIP - k8s: - definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - vars: - resource_item: "{{ server_pods.resources }}" - when: workload_args.serviceip is defined and not workload_args.serviceip|default('false') and server_pods.resources|length > 0 + # + # Each server annotates a "node_idx". Each peer client will + # derive its affinity according the 'colocate' variable. + # - # - # Each server annotates a "node_idx". Each peer client will - # derive its affinity according the 'colocate' variable. - # - - - name: P20 Start Client(s) with serviceIP - k8s: - definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - vars: - resource_item: "{{ serviceip.resources }}" - when: workload_args.serviceip is defined and workload_args.serviceip and serviceip.resources|length > 0 + - name: Start Client(s) with serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + vars: + resource_item: "{{ serviceip.resources }}" + when: workload_args.serviceip is defined and workload_args.serviceip and serviceip.resources|length > 0 - when: resource_kind == "pod" + when: resource_kind == "pod" -### kind - - block: +- block: + ### kind - - name: V19 Wait for vms to be running.... - k8s_facts: - kind: VirtualMachineInstance - api_version: kubevirt.io/v1alpha3 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_vms + - name: Wait for vms to be running.... + k8s_facts: + kind: VirtualMachineInstance + api_version: kubevirt.io/v1alpha3 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: server_vms - - name: V20 Generate uperf test files - k8s: - definition: "{{ lookup('template', 'configmap_script.yml.j2') | from_yaml }}" - with_items: "{{ server_vms.resources }}" + - name: Generate uperf test files + k8s: + definition: "{{ lookup('template', 'configmap_script.yml.j2') | from_yaml }}" + with_items: "{{ server_vms.resources }}" - - name: V21 Start Client(s) - k8s: - definition: "{{ lookup('template', 'workload_vm.yml.j2') | from_yaml }}" - with_items: "{{ server_vms.resources }}" - when: server_vms.resources|length > 0 + - name: Start Client(s) + k8s: + definition: "{{ lookup('template', 'workload_vm.yml.j2') | from_yaml }}" + with_items: "{{ server_vms.resources }}" + when: server_vms.resources|length > 0 - when: resource_kind == "vm" + when: resource_kind == "vm" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Waiting for Clients +- operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Waiting for Clients - when: resource_state.resources[0].status.state == "Starting Clients" diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml index f693d7a3e..743faa176 100644 --- a/roles/uperf/tasks/start_server.yml +++ b/roles/uperf/tasks/start_server.yml @@ -1,13 +1,16 @@ --- - block: - + ### kind - include_tasks: init.yml - name: Create service for server pods k8s: definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}" - with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + vars: + pod_sequence: "{{ pod_hi_idx|int +1 }}" + node_sequence: "{{ node_hi_idx|int +1 }}" + when: workload_args.serviceip is defined and workload_args.serviceip - name: Start Server(s) - total = eligible nodes * density @@ -39,20 +42,17 @@ status: state: "Starting Servers" - when: resource_state.resources[0].status.state == "Building" and resource_kind == "pod" + when: resource_kind == "pod" -########### -# VM remains scale agnostic for now -########### - block: - - - name: V11 Start Server(s) + ### - + when: resource_kind == "vm" diff --git a/roles/uperf/tasks/wait_client_done.yml b/roles/uperf/tasks/wait_client_done.yml index 4f77c73e8..85036a28b 100644 --- a/roles/uperf/tasks/wait_client_done.yml +++ b/roles/uperf/tasks/wait_client_done.yml @@ -1,46 +1,40 @@ --- - - block: + ### kind + - name: Waiting for pods to complete.... + k8s_facts: + kind: pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-bench-client-{{ trunc_uuid }} + register: client_pods + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Cleanup + complete: false + when: "num_server_pods|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: resource_kind == "pod" - #### kind - - block: - - name: Waiting for pods to complete.... - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = uperf-bench-client-{{ trunc_uuid }} - register: client_pods - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false - when: "num_server_pods|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" - when: resource_kind == "pod" - - #### kind - - block: - - - name: V28 get complete - command: "redis-cli get complete" - register: complete_status - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false - when: complete_status.stdout == "true" - when: resource_kind == "vm" - - when: resource_state.resources[0].status.state == "Running" +- block: + ### kind + - name: get complete + command: "redis-cli get complete" + register: complete_status + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Cleanup + complete: false + when: complete_status.stdout == "true" + when: resource_kind == "vm" diff --git a/roles/uperf/tasks/wait_client_ready.yml b/roles/uperf/tasks/wait_client_ready.yml index 319abef7f..c6455fea7 100644 --- a/roles/uperf/tasks/wait_client_ready.yml +++ b/roles/uperf/tasks/wait_client_ready.yml @@ -1,49 +1,48 @@ --- + +- block: + ### kind + + - name: Get client pod status + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-bench-client-{{ trunc_uuid }} + register: client_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + when: "num_server_pods|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (client_pods | json_query('resources[].status.podIP')|length)" + + when: resource_kind == "pod" + - block: - ##### kind - - block: - - name: Get client pod status - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = uperf-bench-client-{{ trunc_uuid }} - register: client_pods - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Clients Running - when: "num_server_pods|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (client_pods | json_query('resources[].status.podIP')|length)" - - when: resource_kind == "pod" - - - block: - ##### kind - - - name: V22 set complete to false - command: "redis-cli set complete false" - - - name: Get count of clients ready - command: "redis-cli get clients-{{ trunc_uuid }}" - register: clients_ready_count - - - name: V24 Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Clients Running - when: "workload_args.pair|default('1')|int == clients_ready_count.stdout|int" - - when: resource_kind == "vm" - - when: resource_state.resources[0].status.state == "Waiting for Clients" + ### kind + + - name: set complete to false + command: "redis-cli set complete false" + + - name: Get count of clients ready + command: "redis-cli get clients-{{ trunc_uuid }}" + register: clients_ready_count + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + when: "workload_args.pair|default('1')|int == clients_ready_count.stdout|int" + + when: resource_kind == "vm" diff --git a/roles/uperf/tasks/wait_server_ready.yml b/roles/uperf/tasks/wait_server_ready.yml index f5b07d095..195125e97 100644 --- a/roles/uperf/tasks/wait_server_ready.yml +++ b/roles/uperf/tasks/wait_server_ready.yml @@ -1,8 +1,8 @@ --- - block: - ########### kind + ### kind - - name: P13 Get server pods + - name: Get server pods k8s_facts: kind: Pod api_version: v1 @@ -19,14 +19,14 @@ namespace: "{{ operator_namespace }}" status: state: "Starting Clients" - when: "num_server_pods|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + when: "num_server_pods|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" - when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" + when: resource_kind == "pod" - block: - ######## kind + ### kind - - name: V14 Wait for vms to be running.... + - name: Wait for vms to be running.... k8s_facts: kind: VirtualMachineInstance api_version: kubevirt.io/v1alpha3 @@ -35,7 +35,7 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_vms - - name: V15 Update resource state + - name: Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -45,11 +45,11 @@ state: "Starting Clients" when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - - name: V16 blocking client from running uperf + - name: blocking client from running uperf command: "redis-cli set start false" with_items: "{{ server_vms.resources }}" when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" + when: resource_kind == "vm" diff --git a/roles/uperf/tasks/wait_set_done.yml b/roles/uperf/tasks/wait_set_done.yml index 72097690c..5c620b017 100644 --- a/roles/uperf/tasks/wait_set_done.yml +++ b/roles/uperf/tasks/wait_set_done.yml @@ -2,6 +2,7 @@ - block: - block: + ### kind - name: read pod completion count command: "redis-cli get num_completion" register: num_completion @@ -18,5 +19,10 @@ when: resource_kind == "pod" - when: resource_state.resources[0].status.state == "Set Running" + ### no kind block - Run a "set" is not yet supported + when: resource_kind == "pod" + +# +# No kind block - It has not been adapted to scale mode yet. +# diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 754e47dc4..f11ff45a1 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -15,7 +15,6 @@ items: template: metadata: labels: - #app: uperf-bench-server-{{ node_idx_item }}-{{ item }}-{{ trunc_uuid }} app: uperf-bench-server-{{ worker_node_list[node_idx_item] }}-{{ item }}-{{ trunc_uuid }} type: uperf-bench-server-{{ trunc_uuid }} annotations: @@ -46,12 +45,7 @@ items: nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_server }}' {% endif %} - -# -# V2 pin server pod to node -# nodeSelector: - #kubernetes.io/hostname: '{{ node_idx_item }}' kubernetes.io/hostname: '{{ worker_node_list[node_idx_item] }}' {% if workload_args.serviceip is sameas true %} diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 89177dad0..59cde767e 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -1,27 +1,46 @@ --- -kind: Service apiVersion: v1 -metadata: - name: uperf-service-{{ item }}-{{ trunc_uuid }} - namespace: '{{ operator_namespace }}' - labels: - app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} - type: uperf-bench-server-{{ trunc_uuid }} -spec: - selector: - app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} - ports: - - name: uperf - port: 20000 - targetPort: 20000 - protocol: TCP +kind: List +metadata: {} +items: +{% macro job_template(item, node_idx_item='') %} + - kind: Service + apiVersion: v1 + metadata: + #name: uperf-service-{{ item }}-{{ trunc_uuid }} + name: uperf-service-{{worker_node_list[ node_idx_item ] }}-{{ item }}-{{ trunc_uuid }} + + namespace: '{{ operator_namespace }}' + labels: + #app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{worker_node_list[ node_idx_item ] }}-{{ item }}-{{ trunc_uuid }} + type: uperf-bench-server-{{ trunc_uuid }} + annotations: + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' + spec: + selector: + #app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{worker_node_list[ node_idx_item ] }}-{{ item }}-{{ trunc_uuid }} + ports: + - name: uperf + port: 20000 + targetPort: 20000 + protocol: TCP {% for num in range(20001,20012,1) %} - - name: uperf-control-tcp-{{num}} - port: {{num}} - targetPort: {{num}} - protocol: TCP - - name: uperf-control-udp-{{num}} - port: {{num}} - targetPort: {{num}} - protocol: UDP + - name: uperf-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: TCP + - name: uperf-control-udp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: UDP {% endfor %} +{% endmacro %} +{% for node_idx_item in range(node_sequence|int) %} +{% for item in range(pod_sequence|int) %} +{{ job_template(item,node_idx_item) }} +{% endfor %} +{% endfor %} + diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index aa47d470f..243c5d3a8 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -117,8 +117,8 @@ items: export my_pod_idx={{ (item['metadata']['annotations']['pod_idx'] | from_json) }}; export ips=$(hostname -I); export num_pairs={{workload_args.pair}}; - export node_count=0 - export pod_count=0 + export node_count=0; + export pod_count=0; node_limit=0; pod_limit=0; STR=''; @@ -128,7 +128,6 @@ items: if [[ $state =~ 'true' ]]; then node_limit=$(echo $STR | cut -f2 -d-); pod_limit=$(echo $STR | cut -f3 -d-); - echo 'state=' $state 'node=' $node_limit 'pod=' $pod_limit; if [[ $my_node_idx -gt $node_limit || $my_pod_idx -gt $pod_limit ]]; then sleep 0.5; continue; fi; diff --git a/roles/uperf/vars/main.yml b/roles/uperf/vars/main.yml index fe27a3bca..aaee090cd 100644 --- a/roles/uperf/vars/main.yml +++ b/roles/uperf/vars/main.yml @@ -2,15 +2,10 @@ # vars file for bench cleanup: true worker_node_list: [] -group_node_count: 0 -max_node_count: 0 - pod_low_idx: "0" pod_hi_idx: "0" node_low_idx: "0" node_hi_idx: "0" - node_idx: "0" pod_idx: "0" - all_run_done: false From c2da2518b1a643021f514954965e201ee0b48310 Mon Sep 17 00:00:00 2001 From: hnhan Date: Mon, 8 Feb 2021 15:16:35 -0500 Subject: [PATCH 029/249] Update uperf.md --- docs/uperf.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/docs/uperf.md b/docs/uperf.md index 3d05dbc55..4bef880fc 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -44,7 +44,6 @@ spec: multus: enabled: false samples: 1 - pair: 1 test_types: - stream protos: @@ -54,6 +53,10 @@ spec: nthrs: - 1 runtime: 30 + colocate: false + density_range: [low, high] + node_range: [low, high] + step_size: addN, log2 ``` `client_resources` and `server_resources` will create uperf client's and server's containers with the given k8s compute resources respectively [k8s resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) @@ -82,7 +85,7 @@ spec: ```yaml samples: 3 - pair: 1 + density: [1,1] test_types: - stream protos: @@ -108,7 +111,7 @@ size. For example: ```yaml samples: 3 - pair: 1 + density: [1,1] test_types: - rr protos: @@ -189,6 +192,55 @@ To enable Multus in Ripsaw, here is the relevant config. ... ``` +### Scale +Scale in this context refers to the ability to enumerate UPERF +client-server pairs during test in a control fashion using the following knobs. + +`colocate: true` will place each client and server pod pair on the same node. + +`density_range` to specify the range of client-server pairs that the test will iterate. + +`node_range` to specify the range of nodes that the test will iterate. + +`step_size` to specify the incrementing method. + +Here is one scale example: + +``` + ... + pin: false + colocate: false + density_range: [1,10] + node_range: [1,128] + step_size: log2 + ... +``` +Note, the `scale` mode is mutual exlusive to `pin` mode with the `pin` mode has higher precedence. +In other words, if `pin:true` the test will deploy pods on `pin_server` and `pin_client` nodes +and ignore `colocate`, `node_range`, and the number of pairs to deploy is specified by the + `density_range.high` value. + +In the above sample, the `scale` mode will be activated since `pin: false`. In the first phase, the +pod instantion phase, the system gathers node inventory and may reduce the `node_range.high` value +to match the number of worker node available in the cluster. + +According to `node_range: [1,128]`, and `density_range:[1,10]`, the system will instantiate 10 pairs on +each of 128 nodes. Each pair has a node_idx and a pod_idx that are used later to control +which one and when they should run the UPERF workload, After all pairs are up and ready, +next comes the test execution phase. + +The scale mode iterates the test as a double nested loop as follows: +``` + for node with node_idx less-or-equal node_range(low, high. step_size): + for pod with pod_idx less-or-equal density_range(low, high, step_size): + run uperf +``` +Hence, with the above params, the first iteration runs the pair with node_idx/pod_idx of {1,1}. After the first +run has completed, the second interation runs 2 pairs of {1,1} and {1,2} and so on. + +The valid `step_size` methods are: addN and log2. `N` can be any integer and `log2` will double the value at each iteration i.e. 1,2,4,8,16 ... +By choosing the appropriate values for `density_range` and `node_range`, the user can generate most if not all +combinations of UPERF data points to exercise datapath performance from many angles. Once done creating/editing the resource file, you can run it by: From e356c2130dd5ef7728ca1fc0b9fd13b9c379e339 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Tue, 9 Feb 2021 12:47:23 -0500 Subject: [PATCH 030/249] 1 Address review comments. a. Keep 'pair' for pin mode b. Remove custom image URLs 2. Rebase to upstream --- resources/crds/ripsaw_v1alpha1_uperf_cr.yaml | 12 ++++++------ resources/operator.yaml | 4 ++-- roles/uperf/tasks/setup.yml | 15 ++++++++++++++- roles/uperf/templates/workload.yml.j2 | 5 +++-- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml b/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml index 8e3182195..8d44d1a88 100644 --- a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml +++ b/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml @@ -20,7 +20,7 @@ spec: pin: false # # pin: true/false - default=false - # - true will run 'Pin' mode using 1 pin_server and 1 pin_client nodes. + # - true will run 'Pin' mode using 1 server (pin_server:) and 1 client (pin_clien:) nodes. # - false will run 'Scale' mode. See colocate, density_range, node_range and step_size. pin_server: "node-0" pin_client: "node-1" @@ -28,9 +28,9 @@ spec: kind: pod pair: 1 # - # 'pair' is obsolete. Instead, use 'density_range' which allows fixed or enumerated - # number of pairs. Enumeration is an enhanced capability of the 'Pin' mode. - # + # 'pair' sepcifies fixed number of client-server pairs for "Pin" mode, + # If 'pair' is NOT present, it will use 'density_range' which allows + # enumeration in addition to fixed number of pair. test_types: - stream protos: @@ -43,7 +43,7 @@ spec: # The following variables are for 'Scale' mode. # The 'Scale' mode is activated when 'pin=false' or undefined. - # The Scale mode op params are: colocate, denstisy_range, node_range and step_size. + # The Scale mode params are: colocate, denstisy_range, node_range and step_size. # # colocate: true/false - default=false # density_range: [n, m] - default=[1,1] @@ -57,7 +57,7 @@ spec: # add10: 1,11,21,31 ... # log2: 1,2,4,8,16,32 ,,, # - # The 'exlude_labels' is the list of ineligible worker nodes. + # 'exclude_labels' specifies the list of ineligible worker nodes. # exclude_labels: (OR conditional, every node that matches any of these labels is excluded) # - "bad=true" # - "fc640=true" diff --git a/resources/operator.yaml b/resources/operator.yaml index 5df666dd5..dd4e0291a 100644 --- a/resources/operator.yaml +++ b/resources/operator.yaml @@ -34,14 +34,14 @@ spec: - /usr/local/bin/ao-logs - /tmp/ansible-operator/runner - stdout - image: quay.io/hnhan/benchmark-operator:uperf-scale + image: quay.io/benchmark-operator/benchmark-operator:master imagePullPolicy: "Always" volumeMounts: - mountPath: /tmp/ansible-operator/runner name: runner readOnly: true - name: benchmark-operator - image: quay.io/hnhan/benchmark-operator:uperf-scale + image: quay.io/benchmark-operator/benchmark-operator:master imagePullPolicy: Always env: - name: WATCH_NAMESPACE diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index be74b0af1..7fad55290 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -123,14 +123,27 @@ with_items: - '{{workload_args.pin_server}}' - '{{workload_args.pin_client}}' + # + # In 'Pin' mode, 'pair' specifies number of pairs (classic behavior), If 'pair' + # is undefined use 'density_range' (new bahavior with "Scale" enhancement) + # + - name: Init "Pin" mode indices using 'pair' + set_fact: + pod_low_idx: "{{ workload_args.pair | default('1')|int - 1 }}" + pod_hi_idx: "{{ workload_args.pair | default('1')|int - 1 }}" + # node indices are used as client pod 'start' parameter. + node_low_idx: "0" + node_hi_idx: "0" + when: workload_args.pair is defined - - name: Init "Pin" mode node and pod indices + - name: Init "Pin" mode indices using 'density_range' set_fact: pod_low_idx: "{{ workload_args.density_range[0] | default('1')|int - 1 }}" pod_hi_idx: "{{ workload_args.density_range[1] | default('1')|int - 1 }}" # node indices are used as client pod 'start' parameter. node_low_idx: "0" node_hi_idx: "0" + when: workload_args.pair is not defined - name: Record num Pin server pods using new worker_node_list set_fact: diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 243c5d3a8..608a3dfbc 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -46,7 +46,7 @@ items: topologyKey: kubernetes.io/hostname containers: - name: benchmark - image: {{ workload_args.image | default('quay.io/lblyth/lblyth-uperf:latest') }} + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} env: - name: uuid value: "{{ uuid }}" @@ -116,7 +116,7 @@ items: export my_node_idx={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; export my_pod_idx={{ (item['metadata']['annotations']['pod_idx'] | from_json) }}; export ips=$(hostname -I); - export num_pairs={{workload_args.pair}}; + export num_pairs=1 export node_count=0; export pod_count=0; node_limit=0; @@ -135,6 +135,7 @@ items: echo 'UPERF-run-context num_node=' $((node_limit+1)) 'density=' $((pod_limit+1)) 'my_node_idx=' $my_node_idx 'my_pod_idx=' $my_pod_idx; node_count=$((node_limit+1)); pod_count=$((pod_limit+1)); + num_pairs=$((pod_limit+1)); {% for test in workload_args.test_types %} {% for proto in workload_args.protos %} From 2c9f6af60927c67b715b7b9d621eca31734a3d0e Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Wed, 10 Feb 2021 03:15:53 +0000 Subject: [PATCH 031/249] Truncate label length to pass CI --- roles/uperf/templates/server.yml.j2 | 2 +- roles/uperf/templates/service.yml.j2 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index f11ff45a1..77716dcde 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -15,7 +15,7 @@ items: template: metadata: labels: - app: uperf-bench-server-{{ worker_node_list[node_idx_item] }}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(16,true,'') }}-{{ item }}-{{ trunc_uuid }} type: uperf-bench-server-{{ trunc_uuid }} annotations: {% if workload_args.multus.enabled is sameas true %} diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 59cde767e..85b77e7a9 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -13,7 +13,7 @@ items: namespace: '{{ operator_namespace }}' labels: #app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} - app: uperf-bench-server-{{worker_node_list[ node_idx_item ] }}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(16,true,'') }}-{{ item }}-{{ trunc_uuid }} type: uperf-bench-server-{{ trunc_uuid }} annotations: node_idx: '{{ node_idx_item }}' @@ -21,7 +21,7 @@ items: spec: selector: #app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} - app: uperf-bench-server-{{worker_node_list[ node_idx_item ] }}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(16,true,'')}}-{{ item }}-{{ trunc_uuid }} ports: - name: uperf port: 20000 From 148213dfa01339bd590b1d04881b29d85d5ab9a6 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Wed, 10 Feb 2021 23:48:06 +0000 Subject: [PATCH 032/249] More name and lalel truncationc to keep within k8s API 63-chars max --- roles/uperf/templates/server.yml.j2 | 4 ++-- roles/uperf/templates/service.yml.j2 | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 77716dcde..669ad572f 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -7,7 +7,7 @@ items: - kind: Job apiVersion: batch/v1 metadata: - name: 'uperf-server-{{worker_node_list[ node_idx_item ] }}-{{ item }}-{{ trunc_uuid }}' + name: 'uperf-server-{{worker_node_list[ node_idx_item ] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }}' namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -15,7 +15,7 @@ items: template: metadata: labels: - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(16,true,'') }}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} type: uperf-bench-server-{{ trunc_uuid }} annotations: {% if workload_args.multus.enabled is sameas true %} diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 85b77e7a9..9baec672d 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -8,12 +8,12 @@ items: apiVersion: v1 metadata: #name: uperf-service-{{ item }}-{{ trunc_uuid }} - name: uperf-service-{{worker_node_list[ node_idx_item ] }}-{{ item }}-{{ trunc_uuid }} + name: uperf-service-{{worker_node_list[ node_idx_item ] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' labels: #app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} - app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(16,true,'') }}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} type: uperf-bench-server-{{ trunc_uuid }} annotations: node_idx: '{{ node_idx_item }}' @@ -21,7 +21,7 @@ items: spec: selector: #app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} - app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(16,true,'')}}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(32,true,'')}}-{{ item }}-{{ trunc_uuid }} ports: - name: uperf port: 20000 From 03f84c40d90c3a809d81854281f1b6d9495fc51c Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Fri, 12 Feb 2021 20:27:38 +0000 Subject: [PATCH 033/249] Address PR review #3: reduce clusterrole's scope, and several cosmetics --- deploy/25_role.yaml | 9 +++++---- roles/uperf/tasks/start_client.yml | 4 ++-- roles/uperf/templates/service.yml.j2 | 3 --- roles/uperf/templates/workload.yml.j2 | 14 +++++++------- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/deploy/25_role.yaml b/deploy/25_role.yaml index 62b2971ab..eb9e77c48 100644 --- a/deploy/25_role.yaml +++ b/deploy/25_role.yaml @@ -1,13 +1,14 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: benchmark-operator rules: - apiGroups: - - "*" + - "" resources: - - "*" + - nodes verbs: - - '*' + - get + - list + - patch diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml index 222621277..9d6ec9ab9 100644 --- a/roles/uperf/tasks/start_client.yml +++ b/roles/uperf/tasks/start_client.yml @@ -20,7 +20,7 @@ definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" vars: resource_item: "{{ server_pods.resources }}" - when: workload_args.serviceip is defined and not workload_args.serviceip|default('false') and server_pods.resources|length > 0 + when: workload_args.serviceip|default(False) == False and server_pods.resources|length > 0 # # Each server annotates a "node_idx". Each peer client will @@ -32,7 +32,7 @@ definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" vars: resource_item: "{{ serviceip.resources }}" - when: workload_args.serviceip is defined and workload_args.serviceip and serviceip.resources|length > 0 + when: workload_args.serviceip|default(False) == True and serviceip.resources|length > 0 when: resource_kind == "pod" diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 9baec672d..0a7edb7b7 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -7,12 +7,10 @@ items: - kind: Service apiVersion: v1 metadata: - #name: uperf-service-{{ item }}-{{ trunc_uuid }} name: uperf-service-{{worker_node_list[ node_idx_item ] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' labels: - #app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} type: uperf-bench-server-{{ trunc_uuid }} annotations: @@ -20,7 +18,6 @@ items: pod_idx: '{{ item }}' spec: selector: - #app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(32,true,'')}}-{{ item }}-{{ trunc_uuid }} ports: - name: uperf diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 608a3dfbc..47175acfa 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -88,11 +88,11 @@ items: args: {% if workload_args.serviceip is sameas true %} - "export serviceip=true; - export h={{item.spec.clusterIP}}; + export h={{item.spec.clusterIP}}; {% else %} {% if workload_args.multus.client is defined %} - "export multus_client={{workload_args.multus.client}}; - export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; + export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; {% else %} - "export h={{item.status.podIP}}; {% endif %} @@ -101,16 +101,16 @@ items: export colocate={{ workload_args.colocate}}; {% endif %} {% if workload_args.step_size is defined %} - export stepsize={{ workload_args.step_size }}; + export stepsize={{ workload_args.step_size }}; {% endif %} {% if workload_args.node_range is defined %} - export node_range='{{ workload_args.node_range[0] }}_{{ workload_args.node_range[1] }}'; + export node_range='{{ workload_args.node_range[0] }}_{{ workload_args.node_range[1] }}'; {% endif %} {% if workload_args.density_range is defined %} - export density_range='{{ workload_args.density_range[0] }}_{{ workload_args.density_range[1] }}'; + export density_range='{{ workload_args.density_range[0] }}_{{ workload_args.density_range[1] }}'; {% endif %} {% if workload_args.networkpolicy is defined %} - export networkpolicy={{workload_args.networkpolicy}}; + export networkpolicy={{workload_args.networkpolicy}}; {% endif %} export hostnet={{workload_args.hostnetwork}}; export my_node_idx={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; @@ -199,7 +199,7 @@ items: kubernetes.io/hostname: "{{ worker_node_list[item['metadata']['annotations']['node_idx'] | from_json] }}" {% else %} nodeSelector: - # skew client node one position left in the woker_node_list + # skew client node one position to the right in the worker_node_list kubernetes.io/hostname: "{{ worker_node_list[ (1+(item['metadata']['annotations']['node_idx'] | from_json)) % (worker_node_list|length)] }}" {% endif %} From 833ec61106844cc95887b90c3b9cee13591079ad Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Tue, 16 Feb 2021 15:55:23 +0000 Subject: [PATCH 034/249] Tried fix a CI uncovered issue by using worker node's label instead of Name. --- roles/uperf/tasks/setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 7fad55290..86ca12e9e 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -51,8 +51,8 @@ - name: Isolate Worker Role Hostnames set_fact: - worker_node_list: "{{ node_list | json_query('resources[].metadata.name') | list }}" - + worker_node_list: "{{ node_list | json_query('resources[].metadata.labels.\"kubernetes.io/hostname\"') | list }}" + - name: List Nodes Labeled with {{ workload_args.exclude_label }} k8s_info: api_version: v1 From 23f01dedb8fc84e2c2555bc60c1bc72bf40d224c Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Thu, 25 Feb 2021 15:30:45 +0000 Subject: [PATCH 035/249] Reverse a server pod label, "app: xxx" to before scale enhancement work for CI. --- roles/uperf/templates/server.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 669ad572f..2bf8d813c 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -15,7 +15,7 @@ items: template: metadata: labels: - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} type: uperf-bench-server-{{ trunc_uuid }} annotations: {% if workload_args.multus.enabled is sameas true %} From 50c74a105ca9f18f08f06c73e446f0bc8e652b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Sat, 27 Feb 2021 19:33:17 +0100 Subject: [PATCH 036/249] Update service selector to match the uperf pods server label --- roles/uperf/templates/server.yml.j2 | 2 +- roles/uperf/templates/service.yml.j2 | 12 +++++------- tests/test_uperf.sh | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 2bf8d813c..669ad572f 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -15,7 +15,7 @@ items: template: metadata: labels: - app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} type: uperf-bench-server-{{ trunc_uuid }} annotations: {% if workload_args.multus.enabled is sameas true %} diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 0a7edb7b7..5b787858a 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -7,18 +7,17 @@ items: - kind: Service apiVersion: v1 metadata: - name: uperf-service-{{worker_node_list[ node_idx_item ] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} - + name: uperf-service-{{ worker_node_list[ node_idx_item ] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' labels: - app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{ worker_node_list[ node_idx_item ] | truncate(32,true,'')}}-{{ item }}-{{ trunc_uuid }} type: uperf-bench-server-{{ trunc_uuid }} annotations: - node_idx: '{{ node_idx_item }}' - pod_idx: '{{ item }}' + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' spec: selector: - app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(32,true,'')}}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(32,true,'')}}-{{ item }}-{{ trunc_uuid }} ports: - name: uperf port: 20000 @@ -40,4 +39,3 @@ items: {{ job_template(item,node_idx_item) }} {% endfor %} {% endfor %} - diff --git a/tests/test_uperf.sh b/tests/test_uperf.sh index 65abcc0f5..4f1b81272 100755 --- a/tests/test_uperf.sh +++ b/tests/test_uperf.sh @@ -27,8 +27,8 @@ function functional_test_uperf { uuid=${long_uuid:0:8} pod_count "type=uperf-bench-server-$uuid" 1 900 - uperf_server_pod=$(get_pod "app=uperf-bench-server-0-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=uperf-bench-server-0-$uuid pods --timeout=300s" "300s" $uperf_server_pod + uperf_server_pod=$(get_pod "type=uperf-bench-server-${uuid}" 300) + wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l type=uperf-bench-server-${uuid} pods --timeout=300s" "300s" $uperf_server_pod uperf_client_pod=$(get_pod "app=uperf-bench-client-$uuid" 900) wait_for "kubectl wait -n my-ripsaw --for=condition=Initialized pods/$uperf_client_pod --timeout=500s" "500s" $uperf_client_pod wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=uperf-bench-client-$uuid jobs --timeout=500s" "500s" $uperf_client_pod From dfd688744d87d0819de7deebf1eff1a2400f2151 Mon Sep 17 00:00:00 2001 From: hnhan Date: Fri, 5 Mar 2021 09:48:24 -0500 Subject: [PATCH 037/249] Update uperf.md --- docs/uperf.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/uperf.md b/docs/uperf.md index 4bef880fc..877104944 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -41,6 +41,7 @@ spec: kind: pod pin_server: "node-0" pin_client: "node-1" + pair: 1 multus: enabled: false samples: 1 @@ -77,6 +78,11 @@ spec: `pin_client` what node to pin the client pod to. +`pair` how many instances of uperf client-server pairs. `pair` is applicable for `pin: true` only. +If `pair` is not specified, the operator will use the value in `density_range` to detemine the number of pairs. +See **Scale** section for more info. `density_range` can do more than `pair` can, but `pair` support is retained +for backward compatibility. + `multus[1]` Configure our pods to use multus. `samples` how many times to run the tests. For example @@ -85,7 +91,7 @@ spec: ```yaml samples: 3 - density: [1,1] + density_range: [1,1] test_types: - stream protos: @@ -111,7 +117,7 @@ size. For example: ```yaml samples: 3 - density: [1,1] + density_range: [1,1] test_types: - rr protos: From 5be182a65979aa1a43c0201de814bef813f17924 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Tue, 9 Mar 2021 00:36:15 +0000 Subject: [PATCH 038/249] Support multi Pod uperf sessions running in paralell. --- roles/uperf/tasks/cleanup.yml | 7 +++++++ roles/uperf/tasks/init.yml | 4 ++-- roles/uperf/tasks/next_set.yml | 6 +++--- roles/uperf/tasks/run_a_set.yml | 2 +- roles/uperf/tasks/wait_set_done.yml | 2 +- roles/uperf/templates/workload.yml.j2 | 6 +++--- 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/roles/uperf/tasks/cleanup.yml b/roles/uperf/tasks/cleanup.yml index 0da5b68ec..a0dff46ae 100644 --- a/roles/uperf/tasks/cleanup.yml +++ b/roles/uperf/tasks/cleanup.yml @@ -29,8 +29,15 @@ name: "{{ item }}" with_items: "{{ clean_pods }}" when: cleanup + + - name: Cleanup redis + command: "redis-cli del num_completion-{{trunc_uuid}}" + command: "redis-cli del start-{{trunc_uuid}}" + when: resource_kind == "pod" + + # # no kind block - We leave VM running # diff --git a/roles/uperf/tasks/init.yml b/roles/uperf/tasks/init.yml index fbae05ce8..7c9b7220b 100644 --- a/roles/uperf/tasks/init.yml +++ b/roles/uperf/tasks/init.yml @@ -1,10 +1,10 @@ --- - name: Clear start flag - command: "redis-cli set start 0" + command: "redis-cli set start-{{trunc_uuid}} 0" - name: Clear num_completion - command: "redis-cli set num_completion 0" + command: "redis-cli set num_completion-{{trunc_uuid}} 0" - name: Init node and pod indices in benchmark context operator_sdk.util.k8s_status: diff --git a/roles/uperf/tasks/next_set.yml b/roles/uperf/tasks/next_set.yml index df5acbc9c..3f2a5f4cf 100644 --- a/roles/uperf/tasks/next_set.yml +++ b/roles/uperf/tasks/next_set.yml @@ -61,7 +61,7 @@ # All done # - name: Unpause pods to complete - command: "redis-cli set start done" + command: "redis-cli set start-{{trunc_uuid}} done" - name: Change state to proceed to exit operator_sdk.util.k8s_status: @@ -79,10 +79,10 @@ # More round(s) to run. # - name: Send redis restart signal - command: "redis-cli set start restart" + command: "redis-cli set start-{{trunc_uuid}} restart" - name: Reset redis num_completion - command: "redis-cli set num_completion 0" + command: "redis-cli set num_completion-{{trunc_uuid}} 0" - name: Change state to run next round operator_sdk.util.k8s_status: diff --git a/roles/uperf/tasks/run_a_set.yml b/roles/uperf/tasks/run_a_set.yml index 1ec4d9260..46c08eb9d 100644 --- a/roles/uperf/tasks/run_a_set.yml +++ b/roles/uperf/tasks/run_a_set.yml @@ -9,7 +9,7 @@ # - name: Signal group to run - command: "redis-cli set start true-{{resource_state.resources[0].status.node_idx|int}}-{{resource_state.resources[0].status.pod_idx|int}}" + command: "redis-cli set start-{{trunc_uuid}} true-{{resource_state.resources[0].status.node_idx|int}}-{{resource_state.resources[0].status.pod_idx|int}}" - name: Update state to "Set Running" operator_sdk.util.k8s_status: diff --git a/roles/uperf/tasks/wait_set_done.yml b/roles/uperf/tasks/wait_set_done.yml index 5c620b017..3f596d681 100644 --- a/roles/uperf/tasks/wait_set_done.yml +++ b/roles/uperf/tasks/wait_set_done.yml @@ -4,7 +4,7 @@ - block: ### kind - name: read pod completion count - command: "redis-cli get num_completion" + command: "redis-cli get num_completion-{{trunc_uuid}}" register: num_completion - operator_sdk.util.k8s_status: diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 47175acfa..38cc4cbf6 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -123,7 +123,7 @@ items: pod_limit=0; STR=''; while true; do - STR=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); + STR=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{trunc_uuid}}); state=$(echo $STR | cut -f1 -d-); if [[ $state =~ 'true' ]]; then node_limit=$(echo $STR | cut -f2 -d-); @@ -161,9 +161,9 @@ items: {% endfor %} {% endfor %} {% endfor %} - redis-cli -h {{bo.resources[0].status.podIP}} incr num_completion; + redis-cli -h {{bo.resources[0].status.podIP}} incr num_completion-{{trunc_uuid}}; while true; do - state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start); + state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{trunc_uuid}}); if [[ $state =~ 'restart' ]]; then break; elif [[ $state =~ 'done' ]]; then From 6c1e7363db7a2b0bfec3de382c1614d51b6cd6b1 Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Thu, 11 Mar 2021 14:08:07 +0000 Subject: [PATCH 039/249] Fix clean up task syntax error --- roles/uperf/tasks/cleanup.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/uperf/tasks/cleanup.yml b/roles/uperf/tasks/cleanup.yml index a0dff46ae..43c35ea25 100644 --- a/roles/uperf/tasks/cleanup.yml +++ b/roles/uperf/tasks/cleanup.yml @@ -31,8 +31,10 @@ when: cleanup - name: Cleanup redis - command: "redis-cli del num_completion-{{trunc_uuid}}" - command: "redis-cli del start-{{trunc_uuid}}" + command: "{{ item }}" + with_items: + - redis-cli del num_completion-{{trunc_uuid}} + - redis-cli del start-{{trunc_uuid}} when: resource_kind == "pod" From a4b9fb285ada3cd60f76ccf64b32376ca422e20a Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Thu, 11 Mar 2021 23:34:16 +0000 Subject: [PATCH 040/249] Fix cleanup task to delete server Pods by default. --- roles/uperf/tasks/cleanup.yml | 37 ++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/roles/uperf/tasks/cleanup.yml b/roles/uperf/tasks/cleanup.yml index 43c35ea25..a1e7b8348 100644 --- a/roles/uperf/tasks/cleanup.yml +++ b/roles/uperf/tasks/cleanup.yml @@ -2,6 +2,16 @@ - block: ### kind + # Cleanup servers, but leave clients around mostly for further examining of results. + - name: Get Server Jobs + k8s_facts: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = uperf-bench-server-{{ trunc_uuid }} + register: server_jobs + - name: Get Server Pods k8s_facts: kind: Pod @@ -11,8 +21,14 @@ - type = uperf-bench-server-{{ trunc_uuid }} register: server_pods - - name: Pod names - to clean + - name: Server Job and Pod names - to clean set_fact: + clean_jobs: | + [ + {% for item in server_jobs.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] clean_pods: | [ {% for item in server_pods.resources %} @@ -20,23 +36,34 @@ {% endfor %} ] - - name: Cleanup run + - name: Cleanup server Job k8s: - kind: pod + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_jobs }}" + + - name: Cleanup server Pod + k8s: + kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' state: absent name: "{{ item }}" with_items: "{{ clean_pods }}" - when: cleanup + when: resource_kind == "pod" and cleanup == True + +- block: - name: Cleanup redis command: "{{ item }}" with_items: - redis-cli del num_completion-{{trunc_uuid}} - redis-cli del start-{{trunc_uuid}} + when: resource_kind == "pod" - when: resource_kind == "pod" From 0b5da1bf04517626c98ca7efc81493a2a125faf1 Mon Sep 17 00:00:00 2001 From: hnhan Date: Wed, 17 Mar 2021 16:13:06 -0400 Subject: [PATCH 041/249] Fixed a uperf.md typos --- docs/uperf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uperf.md b/docs/uperf.md index 877104944..66955208b 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -221,7 +221,7 @@ Here is one scale example: step_size: log2 ... ``` -Note, the `scale` mode is mutual exlusive to `pin` mode with the `pin` mode has higher precedence. +Note, the `scale` mode is mutually exlusive to `pin` mode with the `pin` mode having higher precedence. In other words, if `pin:true` the test will deploy pods on `pin_server` and `pin_client` nodes and ignore `colocate`, `node_range`, and the number of pairs to deploy is specified by the `density_range.high` value. From 441b2374ca1eb9e6a250a88a25cbf6f871095f57 Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy <70236227+mukrishn@users.noreply.github.com> Date: Fri, 19 Mar 2021 09:37:13 -0400 Subject: [PATCH 042/249] corrected uperf pod labels (#5) --- roles/uperf/tasks/cleanup.yml | 4 ++-- roles/uperf/tasks/setup.yml | 2 +- roles/uperf/tasks/start_client.yml | 4 ++-- roles/uperf/tasks/start_server.yml | 4 ++-- roles/uperf/tasks/wait_server_ready.yml | 4 ++-- roles/uperf/templates/server.yml.j2 | 2 +- roles/uperf/templates/server_vm.yml.j2 | 2 +- roles/uperf/templates/service.yml.j2 | 2 +- roles/uperf/templates/workload.yml.j2 | 2 +- roles/uperf/templates/workload_vm.yml.j2 | 2 +- tests/test_crs/valid_uperf.yaml | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/roles/uperf/tasks/cleanup.yml b/roles/uperf/tasks/cleanup.yml index a1e7b8348..b17c8b9af 100644 --- a/roles/uperf/tasks/cleanup.yml +++ b/roles/uperf/tasks/cleanup.yml @@ -9,7 +9,7 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} register: server_jobs - name: Get Server Pods @@ -18,7 +18,7 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods - name: Server Job and Pod names - to clean diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 86ca12e9e..c00f2e244 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -161,7 +161,7 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} register: serviceip when: workload_args.serviceip is defined and workload_args.serviceip diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml index 9d6ec9ab9..eb83a4f25 100644 --- a/roles/uperf/tasks/start_client.yml +++ b/roles/uperf/tasks/start_client.yml @@ -6,7 +6,7 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods - name: Generate uperf xml files @@ -45,7 +45,7 @@ api_version: kubevirt.io/v1alpha3 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} register: server_vms diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml index 743faa176..9c01c9b2a 100644 --- a/roles/uperf/tasks/start_server.yml +++ b/roles/uperf/tasks/start_server.yml @@ -30,7 +30,7 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods - name: Update resource state @@ -58,7 +58,7 @@ api_version: kubevirt.io/v1alpha3 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} register: server_vms - name: Update resource state diff --git a/roles/uperf/tasks/wait_server_ready.yml b/roles/uperf/tasks/wait_server_ready.yml index 195125e97..9b5b707d5 100644 --- a/roles/uperf/tasks/wait_server_ready.yml +++ b/roles/uperf/tasks/wait_server_ready.yml @@ -8,7 +8,7 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods - name: Update resource state @@ -32,7 +32,7 @@ api_version: kubevirt.io/v1alpha3 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} register: server_vms - name: Update resource state diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 669ad572f..3f58cc318 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -16,7 +16,7 @@ items: metadata: labels: app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} - type: uperf-bench-server-{{ trunc_uuid }} + type: {{ meta.name }}-bench-server-{{ trunc_uuid }} annotations: {% if workload_args.multus.enabled is sameas true %} k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} diff --git a/roles/uperf/templates/server_vm.yml.j2 b/roles/uperf/templates/server_vm.yml.j2 index 28c7d21f1..3c2d5444d 100644 --- a/roles/uperf/templates/server_vm.yml.j2 +++ b/roles/uperf/templates/server_vm.yml.j2 @@ -6,7 +6,7 @@ metadata: namespace: '{{ operator_namespace }}' labels: app : uperf-bench-server-{{ item }}-{{ trunc_uuid }} - type : uperf-bench-server-{{ trunc_uuid }} + type : {{ meta.name }}-bench-server-{{ trunc_uuid }} spec: domain: cpu: diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 5b787858a..c5fc64da1 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -11,7 +11,7 @@ items: namespace: '{{ operator_namespace }}' labels: app: uperf-bench-server-{{ worker_node_list[ node_idx_item ] | truncate(32,true,'')}}-{{ item }}-{{ trunc_uuid }} - type: uperf-bench-server-{{ trunc_uuid }} + type: {{ meta.name }}-bench-server-{{ trunc_uuid }} annotations: node_idx: '{{ node_idx_item }}' pod_idx: '{{ item }}' diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 38cc4cbf6..5f3cf61ff 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -19,7 +19,7 @@ items: labels: app: uperf-bench-client-{{ trunc_uuid }} clientfor: {{ item.metadata.labels.app }} - type: uperf-bench-client-{{ trunc_uuid }} + type: {{ meta.name }}-bench-client-{{ trunc_uuid }} {% if workload_args.multus.enabled is sameas true %} annotations: k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} diff --git a/roles/uperf/templates/workload_vm.yml.j2 b/roles/uperf/templates/workload_vm.yml.j2 index ca0fac148..e04f216c5 100644 --- a/roles/uperf/templates/workload_vm.yml.j2 +++ b/roles/uperf/templates/workload_vm.yml.j2 @@ -6,7 +6,7 @@ metadata: namespace: '{{ operator_namespace }}' labels: app: uperf-bench-client-{{ trunc_uuid }} - type: uperf-bench-client-{{ trunc_uuid }} + type: {{ meta.name }}-bench-client-{{ trunc_uuid }} spec: domain: cpu: diff --git a/tests/test_crs/valid_uperf.yaml b/tests/test_crs/valid_uperf.yaml index 3dedadad8..585f2d626 100644 --- a/tests/test_crs/valid_uperf.yaml +++ b/tests/test_crs/valid_uperf.yaml @@ -1,7 +1,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark + name: uperf namespace: my-ripsaw spec: elasticsearch: From 5812533ce30ab7e929bc6d3980adfccdac4eabb0 Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy <70236227+mukrishn@users.noreply.github.com> Date: Mon, 22 Mar 2021 12:15:51 -0400 Subject: [PATCH 043/249] updated few more CRs (#7) --- tests/test_crs/valid_uperf_networkpolicy.yaml | 2 +- tests/test_crs/valid_uperf_resources.yaml | 2 +- tests/test_crs/valid_uperf_serviceip.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_crs/valid_uperf_networkpolicy.yaml b/tests/test_crs/valid_uperf_networkpolicy.yaml index a236b2a21..0cfa51546 100644 --- a/tests/test_crs/valid_uperf_networkpolicy.yaml +++ b/tests/test_crs/valid_uperf_networkpolicy.yaml @@ -1,7 +1,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark + name: uperf namespace: my-ripsaw spec: elasticsearch: diff --git a/tests/test_crs/valid_uperf_resources.yaml b/tests/test_crs/valid_uperf_resources.yaml index 70341ee76..42bbdb0ef 100644 --- a/tests/test_crs/valid_uperf_resources.yaml +++ b/tests/test_crs/valid_uperf_resources.yaml @@ -1,7 +1,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark + name: uperf namespace: my-ripsaw spec: elasticsearch: diff --git a/tests/test_crs/valid_uperf_serviceip.yaml b/tests/test_crs/valid_uperf_serviceip.yaml index 9e5e5e707..d06652301 100644 --- a/tests/test_crs/valid_uperf_serviceip.yaml +++ b/tests/test_crs/valid_uperf_serviceip.yaml @@ -1,7 +1,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark + name: uperf namespace: my-ripsaw spec: elasticsearch: From 514340da70b21c0dcfcec25eeb825a04d74ae097 Mon Sep 17 00:00:00 2001 From: Asis Patra Date: Wed, 24 Mar 2021 16:05:20 +0530 Subject: [PATCH 044/249] Remove iperf3 compulsory arguments (#550) * iPerf3: Remove compulsory arguments --- roles/iperf3/templates/client.yml.j2 | 11 ++++++++++- roles/iperf3/templates/client_store.yml.j2 | 12 +++++++++++- roles/iperf3/templates/server.yml.j2 | 4 +++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/roles/iperf3/templates/client.yml.j2 b/roles/iperf3/templates/client.yml.j2 index cad36864f..acae9610b 100644 --- a/roles/iperf3/templates/client.yml.j2 +++ b/roles/iperf3/templates/client.yml.j2 @@ -36,7 +36,16 @@ spec: imagePullPolicy: Always command: ["/bin/sh", "-c"] args: - - "iperf3 -c {{ item.status.podIP }} -p {{ workload_args.port }} --{{ workload_args.transmit_type }} {{ workload_args.transmit_value }} -l {{ workload_args.length_buffer }} -P {{ workload_args.streams }} -w {{ workload_args.window_size }} -M {{ workload_args.mss }} -S {{ workload_args.ip_tos }} -O {{ workload_args.omit_start }} {{ workload_args.extra_options_client }}" + - "iperf3 -c {{ item.status.podIP }} + {% if workload_args.port is defined %} -p {{ workload_args.port }} {% endif %} + {% if workload_args.transmit_type is defined and workload_args.transmit_value is defined %} --{{ workload_args.transmit_type }} {{ workload_args.transmit_value }} {% endif %} + {% if workload_args.length_buffer is defined %} -l {{ workload_args.length_buffer }} {% endif %} + {% if workload_args.streams is defined %} -P {{ workload_args.streams }} {% endif %} + {% if workload_args.window_size is defined %} -w {{ workload_args.window_size }} {% endif %} + {% if workload_args.mss is defined %} -M {{ workload_args.mss }} {% endif %} + {% if workload_args.ip_tos is defined %} -S {{ workload_args.ip_tos }} {% endif %} + {% if workload_args.omit_start is defined %} -O {{ workload_args.omit_start }} {% endif %} + {% if workload_args.extra_options_client is defined %} {{ workload_args.extra_options_client }} {% endif %}" restartPolicy: OnFailure {% if workload_args.pin_client is defined %} nodeSelector: diff --git a/roles/iperf3/templates/client_store.yml.j2 b/roles/iperf3/templates/client_store.yml.j2 index 2260547b2..bd47b8a60 100644 --- a/roles/iperf3/templates/client_store.yml.j2 +++ b/roles/iperf3/templates/client_store.yml.j2 @@ -37,7 +37,17 @@ spec: command: ["/bin/sh", "-c"] args: - "mkdir -p {{results.path}}/iperf3-{{ uuid }}; - iperf3 -c {{ item.status.podIP }} -J -p {{ workload_args.port }} --{{ workload_args.transmit_type }} {{ workload_args.transmit_value }} -l {{ workload_args.length_buffer }} -P {{ workload_args.streams }} -w {{ workload_args.window_size }} -M {{ workload_args.mss }} -S {{ workload_args.ip_tos }} -O {{ workload_args.omit_start }} {{ workload_args.extra_options_client }} >> {{results.path}}/iperf3-{{ uuid }}/iperf_result.json" + iperf3 -c {{ item.status.podIP }} -J + {% if workload_args.port is defined %} -p {{ workload_args.port }} {% endif %} + {% if workload_args.transmit_type is defined and workload_args.transmit_value is defined %} --{{ workload_args.transmit_type }} {{ workload_args.transmit_value }} {% endif %} + {% if workload_args.length_buffer is defined %} -l {{ workload_args.length_buffer }} {% endif %} + {% if workload_args.streams is defined %} -P {{ workload_args.streams }} {% endif %} + {% if workload_args.window_size is defined %} -w {{ workload_args.window_size }} {% endif %} + {% if workload_args.mss is defined %} -M {{ workload_args.mss }} {% endif %} + {% if workload_args.ip_tos is defined %} -S {{ workload_args.ip_tos }} {% endif %} + {% if workload_args.omit_start is defined %} -O {{ workload_args.omit_start }} {% endif %} + {% if workload_args.extra_options_client is defined %} {{ workload_args.extra_options_client }} {% endif %} + >> {{results.path}}/iperf3-{{ uuid }}/iperf_result.json" volumeMounts: - name: result-data mountPath: "{{results.path}}" diff --git a/roles/iperf3/templates/server.yml.j2 b/roles/iperf3/templates/server.yml.j2 index d982e2d61..77558c253 100644 --- a/roles/iperf3/templates/server.yml.j2 +++ b/roles/iperf3/templates/server.yml.j2 @@ -36,7 +36,9 @@ spec: imagePullPolicy: Always command: ["/bin/sh", "-c"] args: - - "iperf3 -s -p {{ workload_args.port }} {{ workload_args.extra_options_server }}" + - "iperf3 -s + {% if workload_args.port is defined %} -p {{ workload_args.port }} {% endif %} + {% if workload_args.extra_options_server is defined %} {{ workload_args.extra_options_server }} {% endif %}" restartPolicy: OnFailure {% if workload_args.pin_server is defined %} nodeSelector: From cb4eba4368745f7f116fa9ddfb984c297847534b Mon Sep 17 00:00:00 2001 From: Hugh Nhan Date: Wed, 24 Mar 2021 17:28:53 -0400 Subject: [PATCH 045/249] Truncate names to 27 chars. It was discovered that jinja truncate filter does not truncate for less than 5 chars. So truncate(32) would not take effect for names with length between 32 to 37 chars. This cause some name and label to exceed 63 chars violating k8s API --- roles/uperf/templates/server.yml.j2 | 4 ++-- roles/uperf/templates/service.yml.j2 | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 3f58cc318..53e76bd78 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -7,7 +7,7 @@ items: - kind: Job apiVersion: batch/v1 metadata: - name: 'uperf-server-{{worker_node_list[ node_idx_item ] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }}' + name: 'uperf-server-{{worker_node_list[ node_idx_item ] | truncate(27,true,'') }}-{{ item }}-{{ trunc_uuid }}' namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -15,7 +15,7 @@ items: template: metadata: labels: - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'') }}-{{ item }}-{{ trunc_uuid }} type: {{ meta.name }}-bench-server-{{ trunc_uuid }} annotations: {% if workload_args.multus.enabled is sameas true %} diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index c5fc64da1..3c68308dd 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -7,17 +7,17 @@ items: - kind: Service apiVersion: v1 metadata: - name: uperf-service-{{ worker_node_list[ node_idx_item ] | truncate(32,true,'') }}-{{ item }}-{{ trunc_uuid }} + name: uperf-service-{{ worker_node_list[ node_idx_item ] | truncate(27,true,'') }}-{{ item }}-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' labels: - app: uperf-bench-server-{{ worker_node_list[ node_idx_item ] | truncate(32,true,'')}}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{ worker_node_list[ node_idx_item ] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} type: {{ meta.name }}-bench-server-{{ trunc_uuid }} annotations: node_idx: '{{ node_idx_item }}' pod_idx: '{{ item }}' spec: selector: - app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(32,true,'')}}-{{ item }}-{{ trunc_uuid }} + app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} ports: - name: uperf port: 20000 From dded013c943ba8af65fc7fa2664dd0004b81598f Mon Sep 17 00:00:00 2001 From: Frank Ritchie <12985912+fritchie@users.noreply.github.com> Date: Fri, 26 Mar 2021 12:57:56 -0400 Subject: [PATCH 046/249] Always use benchmark-operator for service account The service account name is hard coded in some of the workload roles, ie: https://github.com/cloud-bulldozer/benchmark-operator/blob/master/roles/fio_distributed/templates/servers.yaml#L48 If someone were to change the fullnameOverride in values.yaml the workload would fail. --- charts/benchmark-operator/templates/service-account.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/benchmark-operator/templates/service-account.yaml b/charts/benchmark-operator/templates/service-account.yaml index 71a3ffc67..eb24d0509 100644 --- a/charts/benchmark-operator/templates/service-account.yaml +++ b/charts/benchmark-operator/templates/service-account.yaml @@ -1,5 +1,6 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "benchmark-operator.fullname" . }} + # do not change - hard coded in ansible roles + name: "benchmark-operator" namespace: {{ .Release.Namespace }} From 65f85ca9fd1d6bf82dcf29698a81c691f6aa6f1f Mon Sep 17 00:00:00 2001 From: prubenda Date: Mon, 29 Mar 2021 10:02:57 -0400 Subject: [PATCH 047/249] adding contributing doc version updates --- CONTRIBUTING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index faeee6d78..62489562c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -129,11 +129,14 @@ case of failure or when disabled. This ensures no interference with subsequent w ### The operator container image Any changes to the [roles](roles/) tree or to the [playbook](playbook.yml) file will necessitate a new build of the operator container image. -The container is built using the [Operator SDK](https://github.com/operator-framework/operator-sdk) and pushed to a public repository. +The container is built using the [Operator SDK](https://github.com/operator-framework/operator-sdk) (v.0.10.0 and before) and pushed to a public repository. The public repository could be [quay](https://quay.io) in which case you'll need to: ```bash $ operator-sdk build quay.io//benchmark-operator:testing --image-builder podman +OR +$ podman build -f build/Dockerfile -t quay.io//benchmark-operator:testing . + $ podman push quay.io//benchmark-operator:testing ``` From 784dd66e2699fbb9fa61b889fba7c215cd482ff2 Mon Sep 17 00:00:00 2001 From: Naga Ravi Chaitanya Elluri Date: Fri, 26 Mar 2021 15:00:02 -0400 Subject: [PATCH 048/249] Add more metrics to capture during the kube-burner runs This commit: - Modifies the metrics profiles used by the kube-burner to capture the prometheus replicas CPU and Memory usage as the current implementation only collects the aggregated metrics ( average ) on the infra nodes. - Also modifies the aggregated queries to include registry and logging components resource usage on the infra nodes. --- roles/kube-burner/files/metrics-aggregated.yaml | 10 ++++++++-- roles/kube-burner/files/metrics.yaml | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/roles/kube-burner/files/metrics-aggregated.yaml b/roles/kube-burner/files/metrics-aggregated.yaml index 7ca4445b0..8df5b3979 100644 --- a/roles/kube-burner/files/metrics-aggregated.yaml +++ b/roles/kube-burner/files/metrics-aggregated.yaml @@ -2,10 +2,13 @@ metrics: - query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|sdn|ovn-kubernetes|.*apiserver|authentication|.*controller-manager|.*scheduler)"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 metricName: containerCPU-Masters + - query: (sum(irate(container_cpu_usage_seconds_total{pod!="",container="prometheus",namespace="openshift-monitoring"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0 + metricName: containerCPU-Prometheus + - query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) by (namespace, container)) > 0 metricName: containerCPU-AggregatedWorkers - - query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring)"}[2m]) * 100 and on (node) kube_node_role{role="infra"}) by (namespace, container)) > 0 + - query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"}[2m]) * 100 and on (node) kube_node_role{role="infra"}) by (namespace, container)) > 0 metricName: containerCPU-AggregatedInfra - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!="WATCH"}[5m])) by (le, verb)) > 0 @@ -14,10 +17,13 @@ metrics: - query: (sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 metricName: containerMemory-Masters + - query: (sum(container_memory_rss{pod!="",namespace="openshift-monitoring",name!="",container="prometheus"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0 + metricName: containerMemory-Prometheus + - query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"} and on (node) kube_node_role{role="worker"}) by (container, namespace) metricName: containerMemory-AggregatedWorkers - - query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring)"} and on (node) kube_node_role{role="infra"}) by (container, namespace) + - query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"} and on (node) kube_node_role{role="infra"}) by (container, namespace) metricName: containerMemory-AggregatedInfra - query: avg(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) diff --git a/roles/kube-burner/files/metrics.yaml b/roles/kube-burner/files/metrics.yaml index 452df2e8f..7992d6212 100644 --- a/roles/kube-burner/files/metrics.yaml +++ b/roles/kube-burner/files/metrics.yaml @@ -1,11 +1,11 @@ metrics: - - query: sum(irate(container_cpu_usage_seconds_total{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}[2m]) * 100) by (pod, namespace, node) + - query: sum(irate(container_cpu_usage_seconds_total{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}[2m]) * 100) by (pod, namespace, node) metricName: podCPU - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!="WATCH"}[5m])) by (le, verb)) > 0 metricName: API99thLatency - - query: sum(container_memory_rss{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}) by (pod, namespace, node) + - query: sum(container_memory_rss{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}) by (pod, namespace, node) metricName: podMemory - query: sum(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} From fee594c0d79cbaf1bcad6d1cba18fc3b82d3b66d Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 25 Mar 2021 11:11:19 +0100 Subject: [PATCH 049/249] Disable snake_case conversion Signed-off-by: Raul Sevilla --- watches.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/watches.yaml b/watches.yaml index d3dffda1e..7b2aee108 100644 --- a/watches.yaml +++ b/watches.yaml @@ -5,3 +5,4 @@ playbook: /opt/ansible/playbook.yml reconcilePeriod: 3s manageStatus: false + snakeCaseParameters: false From 59bb92b7266629af643a016fd9e1383b2b4615db Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 29 Mar 2021 12:15:22 +0200 Subject: [PATCH 050/249] Fix clusterrolebinding naming in CI --- deploy/25_role.yaml | 3 +-- deploy/35_role_binding.yaml | 5 ++--- tests/common.sh | 1 + 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/deploy/25_role.yaml b/deploy/25_role.yaml index eb9e77c48..4284753e1 100644 --- a/deploy/25_role.yaml +++ b/deploy/25_role.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: benchmark-operator + name: benchmark-operator-role rules: - apiGroups: - "" @@ -11,4 +11,3 @@ rules: - get - list - patch - diff --git a/deploy/35_role_binding.yaml b/deploy/35_role_binding.yaml index cd0b952a9..1deac0165 100644 --- a/deploy/35_role_binding.yaml +++ b/deploy/35_role_binding.yaml @@ -1,13 +1,12 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: benchmark-operator + name: benchmark-operator-rolebinding subjects: - kind: ServiceAccount name: benchmark-operator namespace: my-ripsaw roleRef: kind: ClusterRole - name: benchmark-operator + name: benchmark-operator-role apiGroup: rbac.authorization.k8s.io - diff --git a/tests/common.sh b/tests/common.sh index b7d9a9d7e..76e8722db 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -61,6 +61,7 @@ function wait_clean { then kubectl delete benchmarks -n my-ripsaw --all --ignore-not-found fi + kubectl delete -f deploy/25_role.yaml -f deploy/35_role_binding.yaml --ignore-not-found kubectl delete namespace my-ripsaw --ignore-not-found } From f24c399c4c0241cb57b5648459f09ea3090d3e7e Mon Sep 17 00:00:00 2001 From: Frank Ritchie <12985912+fritchie@users.noreply.github.com> Date: Thu, 25 Mar 2021 16:42:32 -0400 Subject: [PATCH 051/249] Make redis image a variable This adds a new variable to values.yaml named redisImage. Specifying a custom location for the image can be needed in secure environments. --- charts/benchmark-operator/README.md | 3 +++ charts/benchmark-operator/templates/operator.yaml | 4 +--- charts/benchmark-operator/values.yaml | 10 ++++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/charts/benchmark-operator/README.md b/charts/benchmark-operator/README.md index 6ff20dfaf..aff72d471 100644 --- a/charts/benchmark-operator/README.md +++ b/charts/benchmark-operator/README.md @@ -14,6 +14,9 @@ A Helm chart for Kubernetes | operator.image.pullPolicy | string | `"Always"` | | | operator.image.repository | string | `"quay.io/benchmark-operator/benchmark-operator"` | | | operator.image.tag | string | `""` | | +| operator.redisImage.pullPolicy | string | `"Always"` | | +| operator.redisImage.repository | string | `"k8s.gcr.io/redis"` | | +| operator.redisImage.tag | string | `"v1"` | | | operator.nodeSelector | object | `{}` | | | operator.replicaCount | int | `1` | how many replicas for the operator deployment | | operator.resources.limits.cpu | string | `"0.1"` | | diff --git a/charts/benchmark-operator/templates/operator.yaml b/charts/benchmark-operator/templates/operator.yaml index a2f5b7cb9..62cd15934 100644 --- a/charts/benchmark-operator/templates/operator.yaml +++ b/charts/benchmark-operator/templates/operator.yaml @@ -48,7 +48,7 @@ spec: - mountPath: /tmp/ansible-operator/runner name: runner - name: redis-master - image: k8s.gcr.io/redis:v1 + image: "{{ .Values.operator.redisImage.repository }}:{{ .Values.operator.redisImage.tag }}" env: - name: MASTER value: "true" @@ -72,5 +72,3 @@ spec: emptyDir: {} - name: runner emptyDir: {} - - diff --git a/charts/benchmark-operator/values.yaml b/charts/benchmark-operator/values.yaml index 600cdfd18..379f03664 100644 --- a/charts/benchmark-operator/values.yaml +++ b/charts/benchmark-operator/values.yaml @@ -12,8 +12,7 @@ rbac: kubevirt: false selfProvisioner: false - -operator: +operator: # -- how many replicas for the operator deployment replicaCount: 1 @@ -23,14 +22,17 @@ operator: pullPolicy: Always tag: master + redisImage: + repository: k8s.gcr.io/redis + pullPolicy: Always + tag: v1 + resources: limits: cpu: "0.1" - nodeSelector: {} - tolerations: # -- schedule on a workload node even if it's tainted - key: role From b6f1eb4d9812b04f4e74f65026a94156edc95e64 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 6 Apr 2021 16:30:34 +0200 Subject: [PATCH 052/249] Make es_url parameter optional, so kube-burner can run w/o indexing Signed-off-by: Raul Sevilla --- roles/kube-burner/templates/kube-burner.yml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/kube-burner/templates/kube-burner.yml.j2 b/roles/kube-burner/templates/kube-burner.yml.j2 index 9e4c5485a..69fb1c48e 100644 --- a/roles/kube-burner/templates/kube-burner.yml.j2 +++ b/roles/kube-burner/templates/kube-burner.yml.j2 @@ -30,9 +30,11 @@ spec: - name: kube-burner image: {{ workload_args.image | default('quay.io/cloud-bulldozer/kube-burner:latest') }} imagePullPolicy: Always +{% if prometheus.es_url is defined %} env: - name: prom_es value: "{{ prometheus.es_url }}" +{% endif %} workingDir: /tmp/kube-burner command: ["/bin/sh", "-c"] args: From 90de2d3d82627640b70c27e72c1245724f5f5827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Tue, 6 Apr 2021 18:39:10 +0200 Subject: [PATCH 053/249] System-metrics collection (#548) System-metrics collection Signed-off-by: Raul Sevilla --- docs/system-metrics.md | 57 +++++++++++++++++++ playbook.yml | 19 +++++-- .../crds/ripsaw_v1alpha1_ripsaw_crd.yaml | 33 ++++++++++- .../templates/db_mssql_workload.yml.j2 | 2 +- roles/system-metrics/tasks/main.yml | 56 ++++++++++++++++++ roles/system-metrics/templates/index.yml | 9 +++ .../templates/kube-burner.yml.j2 | 41 +++++++++++++ .../system-metrics/templates/node-metrics.yml | 32 +++++++++++ tests/test_byowl.sh | 3 +- tests/test_crs/valid_byowl.yaml | 7 +++ tests/test_crs/valid_fiod.yaml | 6 ++ tests/test_crs/valid_fiod_bsrange.yaml | 6 ++ tests/test_crs/valid_fiod_hostpath.yaml | 6 ++ tests/test_crs/valid_flent.yaml | 6 ++ tests/test_crs/valid_flent_resources.yaml | 6 ++ tests/test_crs/valid_fs_drift.yaml | 6 ++ tests/test_crs/valid_fs_drift_hostpath.yaml | 6 ++ tests/test_crs/valid_hammerdb.yaml | 6 ++ tests/test_crs/valid_iperf3.yaml | 6 ++ tests/test_crs/valid_pgbench.yaml | 6 ++ tests/test_crs/valid_scale_down.yaml | 6 ++ tests/test_crs/valid_scale_up.yaml | 6 ++ tests/test_crs/valid_smallfile.yaml | 6 ++ tests/test_crs/valid_smallfile_hostpath.yaml | 6 ++ tests/test_crs/valid_stressng.yaml | 6 ++ tests/test_crs/valid_sysbench.yaml | 6 ++ tests/test_crs/valid_uperf.yaml | 6 ++ tests/test_crs/valid_uperf_networkpolicy.yaml | 6 ++ tests/test_crs/valid_uperf_resources.yaml | 6 ++ tests/test_crs/valid_uperf_serviceip.yaml | 6 ++ tests/test_crs/valid_vegeta.yaml | 6 ++ tests/test_crs/valid_vegeta_hostnetwork.yaml | 6 ++ tests/test_crs/valid_ycsb-mongo.yaml | 6 ++ tests/test_fiod.sh | 2 + tests/test_flent.sh | 3 +- tests/test_fs_drift.sh | 3 +- tests/test_hammerdb.sh | 3 +- tests/test_iperf3.sh | 3 +- tests/test_smallfile.sh | 3 +- tests/test_stressng.sh | 3 +- tests/test_uperf.sh | 3 +- tests/test_vegeta.sh | 3 +- tests/test_ycsb.sh | 3 +- 43 files changed, 410 insertions(+), 16 deletions(-) create mode 100644 docs/system-metrics.md create mode 100644 roles/system-metrics/tasks/main.yml create mode 100644 roles/system-metrics/templates/index.yml create mode 100644 roles/system-metrics/templates/kube-burner.yml.j2 create mode 100644 roles/system-metrics/templates/node-metrics.yml diff --git a/docs/system-metrics.md b/docs/system-metrics.md new file mode 100644 index 000000000..7eb818f20 --- /dev/null +++ b/docs/system-metrics.md @@ -0,0 +1,57 @@ +# System-metrics collection + +Benchmark-operator is able to collect prometheus metrics from the cluster at the end of a benchmark. To do so, it creates a k8s job that uses [kube-burner](https://github.com/cloud-bulldozer/kube-burner) to collect the Prometheus metrics given by a configuration file. This system metrics collection mechanism in available for all workloads except `kube-burner` and `backpack`. + +This feature is disabled by default, it can be enabled by adding a `system_metrics` section to a benchmark CR. + +```yaml + system_metrics: + collection: true (Defaults to false) + prom_url: + index_name: + es_url: + prom_token: + metrics_profile: + step: + image: +``` + +As stated in the example above, `metrics_profile` points to node-metrics.yml, (this file is available within the system-metrics role of this repo), however it can be configured pointing to an external URL like in the example below: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: example-benchmark + namespace: my-ripsaw +spec: + system_metrics: + enabled: true + prom_token: eyJhbGciOiJSUzI1NiIsImtpZCI6IlljTUxlUHBTY2hvUVJQYUlZWmV5MTE4d3VnRFpjUUh5MWZtdE9hdnlvNFUifQ.eyJpc3MiOiJrdWJlcnopeVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJvcGVuc2hpZnQtbW9uaXRvcmluZyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cy10b2tlbi12NGo3YyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjFkYTdkMTRkLWE2MTktNDZjYS1iZGRlLTMzOTYxOWYxMmM4MiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpvcGVuc2hpZnQtbW9uaXRvcmluZzpwcm9tZXRoZXVzLWs4cyJ9.PJp5pD_CjMG05vVLFdxUDRWGA8C71TNyRsUcHmpMlnZLQWBwxZSDZ-Uh3y6g1O-Yz3nopeCoZLB6lugxxalcT1DhiEC9yNK53Lr6HLqaz8nWUbRPbex0913KcuSsnpeRj7tzlwQ2K3WbtIeyyHpG5vAeff07LDvHUcPsc3B_dyetGnInClBHFVEJRES6f5DbIUidtXZEfYKANJNcssly0qZMZicwvM4a_pRp6ctGB-zzR6Ac4lh3b1JLfl_5TLGuuoYEOAeJPVUF4TjemsNNJ5BlycEkVI377LKNdHf83wua5pn3ItJtKE5gdrG833203p-y0pj-UDJj2bAv0cjUQ + metrics_profile: https://raw.githubusercontent.com/cloud-bulldozer/benchmark-operator/master/roles/kube-burner/files/metrics-aggregated.yaml + elasticsearch: + url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443 + index_name: ripsaw-uperf + metadata: + collection: false + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: false + pin: false + multus: + enabled: false + samples: 1 + pair: 1 + test_types: + - stream + protos: + - tcp + sizes: + - 1024 + nthrs: + - 1 + runtime: 120 +``` diff --git a/playbook.yml b/playbook.yml index 8cf814a6d..5ce043713 100644 --- a/playbook.yml +++ b/playbook.yml @@ -56,12 +56,13 @@ when: workload is defined and (cr_state.resources[0].status is not defined or cr_state.resources[0].status.uuid is not defined) + + - set_fact: + uuid: "{{ cr_state.resources[0].status.uuid }}" + trunc_uuid: "{{ cr_state.resources[0].status.suuid }}" + - name: Run Workload block: - - - set_fact: - uuid: "{{ cr_state.resources[0].status.uuid }}" - trunc_uuid: "{{ cr_state.resources[0].status.suuid }}" - block: @@ -93,3 +94,13 @@ when: metadata is not defined or not metadata.collection | default('false') | bool or (cr_state.resources[0].status.metadata is defined and cr_state.resources[0].status.metadata == "Complete") or metadata.targeted | default('true') | bool when: cr_state is defined and cr_state.resources[0].status is defined and not cr_state.resources[0].status.complete|bool and (cr_state.resources[0].status.state is not defined or cr_state.resources[0].status.state != "Error") + + - include_role: + name: system-metrics + vars: + workload_args: "{{ workload.args }}" + when: + - cr_state.resources[0].status.state is defined + - cr_state.resources[0].status.state == "Complete" + - system_metrics.collection | bool + - workload.name not in ["kube-burner", "backpack"] diff --git a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml b/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml index 2aa13a7a3..d2c6f1426 100644 --- a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml +++ b/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml @@ -45,6 +45,31 @@ spec: type: string prom_token: type: string + system_metrics: + type: object + properties: + collection: + type: boolean + default: false + es_url: + type: string + index_name: + type: string + default: system-metrics + step: + type: string + default: 30s + metrics_profile: + type: string + default: node-metrics.yml + prom_url: + type: string + default: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + image: + type: string + default: quay.io/cloud-bulldozer/kube-burner:latest + prom_token: + type: string workload: x-kubernetes-preserve-unknown-fields: true type: object @@ -135,7 +160,9 @@ spec: type: string node_idx: type: string - + system_metrics: + type: string + default: Not collected additionalPrinterColumns: - name: Type type: string @@ -153,6 +180,10 @@ spec: type: string description: If Cerberus is connected or not jsonPath: .status.cerberus + - name: System metrics + type: string + description: System metrics collect status + jsonPath: .status.system_metrics - name: UUID type: string jsonPath: .status.uuid diff --git a/roles/hammerdb/templates/db_mssql_workload.yml.j2 b/roles/hammerdb/templates/db_mssql_workload.yml.j2 index e5513f0c0..ef681a2e1 100644 --- a/roles/hammerdb/templates/db_mssql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mssql_workload.yml.j2 @@ -40,7 +40,7 @@ spec: value: "{{ prometheus.es_parallel | default(false) }}" - name: prom_token value: "{{ prometheus.prom_token | default() }}" - - name: prom_url + - name: prom_url value: "{{ prometheus.prom_url | default() }}" {% endif %} command: ["/bin/sh", "-c"] diff --git a/roles/system-metrics/tasks/main.yml b/roles/system-metrics/tasks/main.yml new file mode 100644 index 000000000..9d07d3685 --- /dev/null +++ b/roles/system-metrics/tasks/main.yml @@ -0,0 +1,56 @@ +--- + +- block: + + - name: Create kube-burner configmap + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: system-metrics-collector-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + index.yml: "{{ lookup('template', 'index.yml')}}" + metrics.yml: "{{ lookup('template', 'node-metrics.yml')}}" + + - name: Launching kube-burner job to index system-metrics + k8s: + definition: "{{ lookup('template', 'kube-burner.yml.j2') | from_yaml }}" + + - name: Get job status + k8s_info: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + name: system-metrics-collector-{{ trunc_uuid }} + register: job_state + + - name: Set collecting state to system-metrics benchmark + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + system_metrics: Collecting + + when: cr_state.resources[0].status.system_metrics == "Not collected" + +- name: Get job status + k8s_info: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + name: system-metrics-collector-{{ trunc_uuid }} + register: job_state + +- name: Set collected state to system-metrics benchmark + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + system_metrics: Collected + when: job_state.resources[0].status.succeeded is defined and (job_state.resources[0].status.succeeded | int) > 0 diff --git a/roles/system-metrics/templates/index.yml b/roles/system-metrics/templates/index.yml new file mode 100644 index 000000000..8dd80c9d1 --- /dev/null +++ b/roles/system-metrics/templates/index.yml @@ -0,0 +1,9 @@ +--- +global: + writeToFile: false + indexerConfig: + enabled: true + esServers: ["{{ system_metrics.es_url }}"] + insecureSkipVerify: true + defaultIndex: "{{ system_metrics.index_name }}" + type: elastic diff --git a/roles/system-metrics/templates/kube-burner.yml.j2 b/roles/system-metrics/templates/kube-burner.yml.j2 new file mode 100644 index 000000000..9117725a7 --- /dev/null +++ b/roles/system-metrics/templates/kube-burner.yml.j2 @@ -0,0 +1,41 @@ +--- +kind: Job +apiVersion: batch/v1 +metadata: + name: system-metrics-collector-{{ trunc_uuid }} + namespace: {{ operator_namespace }} +spec: + backoffLimit: 0 + activeDeadlineSeconds: 3600 + template: + metadata: + labels: + app: system-metrics-collector-{{ trunc_uuid }} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} + restartPolicy: Never + containers: + - name: kube-burner + image: {{ system_metrics.image }} + imagePullPolicy: Always + workingDir: /tmp/kube-burner + command: ["/bin/sh", "-c"] + args: + - > + kube-burner index + --uuid={{ uuid }} + -c index.yml + -u {{ system_metrics.prom_url }} + -t {{ system_metrics.prom_token }} + --start={{ (cr_state.resources[0].metadata.creationTimestamp|to_datetime('%Y-%m-%dT%H:%M:%SZ')).strftime('%s') }} + -m {{ system_metrics.metrics_profile|default("metrics.yml") }} + --step={{ system_metrics.step }} + volumeMounts: + - name: system-metrics-collector + mountPath: /tmp/kube-burner + volumes: + - name: system-metrics-collector + configMap: + name: system-metrics-collector-{{ trunc_uuid }} diff --git a/roles/system-metrics/templates/node-metrics.yml b/roles/system-metrics/templates/node-metrics.yml new file mode 100644 index 000000000..3e9c7f7ef --- /dev/null +++ b/roles/system-metrics/templates/node-metrics.yml @@ -0,0 +1,32 @@ +metrics: + - query: sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) > 0 + metricName: nodeCPU + + - query: avg(node_memory_MemAvailable_bytes) by (instance) + metricName: nodeMemoryAvailable + + - query: avg(node_memory_Active_bytes) by (instance) + metricName: nodeMemoryActive + + - query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) + metricName: nodeMemoryCached+nodeMemoryBuffers + + - query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: rxNetworkBytes + + - query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: txNetworkBytes + + - query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskWrittenBytes + + - query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskReadBytes + + - query: kube_node_role + metricName: nodeRoles + instant: true + + - query: cluster_version{type="completed"} + metricName: clusterVersion + instant: true diff --git a/tests/test_byowl.sh b/tests/test_byowl.sh index 974b2bc65..8edd2d15c 100644 --- a/tests/test_byowl.sh +++ b/tests/test_byowl.sh @@ -20,7 +20,8 @@ trap finish EXIT function functional_test_byowl { wait_clean apply_operator - kubectl apply -f tests/test_crs/valid_byowl.yaml + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" tests/test_crs/valid_byowl.yaml | kubectl apply -f - long_uuid=$(get_uuid 20) uuid=${long_uuid:0:8} diff --git a/tests/test_crs/valid_byowl.yaml b/tests/test_crs/valid_byowl.yaml index 9f1260357..bcbc45f7a 100644 --- a/tests/test_crs/valid_byowl.yaml +++ b/tests/test_crs/valid_byowl.yaml @@ -4,6 +4,13 @@ metadata: name: byowl-benchmark namespace: my-ripsaw spec: + prometheus: + system_metrics_collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + system_metrics_index: system-metrics + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER metadata: diff --git a/tests/test_crs/valid_fiod.yaml b/tests/test_crs/valid_fiod.yaml index ca8c1b39a..a11ef4386 100644 --- a/tests/test_crs/valid_fiod.yaml +++ b/tests/test_crs/valid_fiod.yaml @@ -4,6 +4,12 @@ metadata: name: example-benchmark namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-fio diff --git a/tests/test_crs/valid_fiod_bsrange.yaml b/tests/test_crs/valid_fiod_bsrange.yaml index c86b09773..d2f798f7f 100644 --- a/tests/test_crs/valid_fiod_bsrange.yaml +++ b/tests/test_crs/valid_fiod_bsrange.yaml @@ -4,6 +4,12 @@ metadata: name: example-benchmark namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-fio diff --git a/tests/test_crs/valid_fiod_hostpath.yaml b/tests/test_crs/valid_fiod_hostpath.yaml index 9b826bbf0..053198618 100644 --- a/tests/test_crs/valid_fiod_hostpath.yaml +++ b/tests/test_crs/valid_fiod_hostpath.yaml @@ -4,6 +4,12 @@ metadata: name: example-benchmark namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-fio diff --git a/tests/test_crs/valid_flent.yaml b/tests/test_crs/valid_flent.yaml index 73f9d882a..966c549ef 100644 --- a/tests/test_crs/valid_flent.yaml +++ b/tests/test_crs/valid_flent.yaml @@ -4,6 +4,12 @@ metadata: name: flent-benchmark namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-flent diff --git a/tests/test_crs/valid_flent_resources.yaml b/tests/test_crs/valid_flent_resources.yaml index 75b75f4b2..81aabf6f2 100644 --- a/tests/test_crs/valid_flent_resources.yaml +++ b/tests/test_crs/valid_flent_resources.yaml @@ -5,6 +5,12 @@ metadata: namespace: my-ripsaw spec: clustername: myk8scluster + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-flent diff --git a/tests/test_crs/valid_fs_drift.yaml b/tests/test_crs/valid_fs_drift.yaml index f949c665a..ceb7cd6fc 100644 --- a/tests/test_crs/valid_fs_drift.yaml +++ b/tests/test_crs/valid_fs_drift.yaml @@ -7,6 +7,12 @@ spec: test_user: homer_simpson # to separate this test run from everyone else's clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml # where elastic search is running elasticsearch: url: ES_SERVER diff --git a/tests/test_crs/valid_fs_drift_hostpath.yaml b/tests/test_crs/valid_fs_drift_hostpath.yaml index bc1e338f0..3983b219d 100644 --- a/tests/test_crs/valid_fs_drift_hostpath.yaml +++ b/tests/test_crs/valid_fs_drift_hostpath.yaml @@ -7,6 +7,12 @@ spec: test_user: homer_simpson # to separate this test run from everyone else's clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml # where elastic search is running elasticsearch: url: ES_SERVER diff --git a/tests/test_crs/valid_hammerdb.yaml b/tests/test_crs/valid_hammerdb.yaml index 74c1bf74e..dff12f4a5 100644 --- a/tests/test_crs/valid_hammerdb.yaml +++ b/tests/test_crs/valid_hammerdb.yaml @@ -4,6 +4,12 @@ metadata: name: hammerdb namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-hammerdb diff --git a/tests/test_crs/valid_iperf3.yaml b/tests/test_crs/valid_iperf3.yaml index 5ac1faac3..7660fd256 100644 --- a/tests/test_crs/valid_iperf3.yaml +++ b/tests/test_crs/valid_iperf3.yaml @@ -4,6 +4,12 @@ metadata: name: example-benchmark namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER metadata: diff --git a/tests/test_crs/valid_pgbench.yaml b/tests/test_crs/valid_pgbench.yaml index b0d9b72f6..485a5f12a 100644 --- a/tests/test_crs/valid_pgbench.yaml +++ b/tests/test_crs/valid_pgbench.yaml @@ -4,6 +4,12 @@ metadata: name: pgbench-benchmark namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-pgbench diff --git a/tests/test_crs/valid_scale_down.yaml b/tests/test_crs/valid_scale_down.yaml index ade94fe09..e6d451f8d 100644 --- a/tests/test_crs/valid_scale_down.yaml +++ b/tests/test_crs/valid_scale_down.yaml @@ -4,6 +4,12 @@ metadata: name: scale namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: openshift-cluster-timings diff --git a/tests/test_crs/valid_scale_up.yaml b/tests/test_crs/valid_scale_up.yaml index a54cc658a..0fd5ed785 100644 --- a/tests/test_crs/valid_scale_up.yaml +++ b/tests/test_crs/valid_scale_up.yaml @@ -4,6 +4,12 @@ metadata: name: scale namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: openshift-cluster-timings diff --git a/tests/test_crs/valid_smallfile.yaml b/tests/test_crs/valid_smallfile.yaml index ab9cec08f..71efff93c 100644 --- a/tests/test_crs/valid_smallfile.yaml +++ b/tests/test_crs/valid_smallfile.yaml @@ -7,6 +7,12 @@ spec: test_user: homer_simpson # to separate this test run from everyone else's clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml # where elastic search is running elasticsearch: url: ES_SERVER diff --git a/tests/test_crs/valid_smallfile_hostpath.yaml b/tests/test_crs/valid_smallfile_hostpath.yaml index f57942152..9d0e32477 100644 --- a/tests/test_crs/valid_smallfile_hostpath.yaml +++ b/tests/test_crs/valid_smallfile_hostpath.yaml @@ -8,6 +8,12 @@ spec: test_user: homer_simpson # to separate this test run from everyone else's clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-smallfile diff --git a/tests/test_crs/valid_stressng.yaml b/tests/test_crs/valid_stressng.yaml index d30f5afc1..19b9ecf8b 100644 --- a/tests/test_crs/valid_stressng.yaml +++ b/tests/test_crs/valid_stressng.yaml @@ -4,6 +4,12 @@ metadata: name: stressng namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-stressng diff --git a/tests/test_crs/valid_sysbench.yaml b/tests/test_crs/valid_sysbench.yaml index 28c4ff235..d291501ca 100644 --- a/tests/test_crs/valid_sysbench.yaml +++ b/tests/test_crs/valid_sysbench.yaml @@ -4,6 +4,12 @@ metadata: name: example-benchmark namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER metadata: diff --git a/tests/test_crs/valid_uperf.yaml b/tests/test_crs/valid_uperf.yaml index 585f2d626..e023a3989 100644 --- a/tests/test_crs/valid_uperf.yaml +++ b/tests/test_crs/valid_uperf.yaml @@ -4,6 +4,12 @@ metadata: name: uperf namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-uperf diff --git a/tests/test_crs/valid_uperf_networkpolicy.yaml b/tests/test_crs/valid_uperf_networkpolicy.yaml index 0cfa51546..286a8e73a 100644 --- a/tests/test_crs/valid_uperf_networkpolicy.yaml +++ b/tests/test_crs/valid_uperf_networkpolicy.yaml @@ -4,6 +4,12 @@ metadata: name: uperf namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-uperf diff --git a/tests/test_crs/valid_uperf_resources.yaml b/tests/test_crs/valid_uperf_resources.yaml index 42bbdb0ef..148c1f889 100644 --- a/tests/test_crs/valid_uperf_resources.yaml +++ b/tests/test_crs/valid_uperf_resources.yaml @@ -4,6 +4,12 @@ metadata: name: uperf namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-uperf diff --git a/tests/test_crs/valid_uperf_serviceip.yaml b/tests/test_crs/valid_uperf_serviceip.yaml index d06652301..83fb356b6 100644 --- a/tests/test_crs/valid_uperf_serviceip.yaml +++ b/tests/test_crs/valid_uperf_serviceip.yaml @@ -4,6 +4,12 @@ metadata: name: uperf namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-uperf diff --git a/tests/test_crs/valid_vegeta.yaml b/tests/test_crs/valid_vegeta.yaml index a40213e34..fb6a93368 100644 --- a/tests/test_crs/valid_vegeta.yaml +++ b/tests/test_crs/valid_vegeta.yaml @@ -5,6 +5,12 @@ metadata: name: vegeta-benchmark namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-vegeta diff --git a/tests/test_crs/valid_vegeta_hostnetwork.yaml b/tests/test_crs/valid_vegeta_hostnetwork.yaml index 287aafd92..4ea03f4e7 100644 --- a/tests/test_crs/valid_vegeta_hostnetwork.yaml +++ b/tests/test_crs/valid_vegeta_hostnetwork.yaml @@ -5,6 +5,12 @@ metadata: name: vegeta-benchmark-hostnetwork namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-vegeta diff --git a/tests/test_crs/valid_ycsb-mongo.yaml b/tests/test_crs/valid_ycsb-mongo.yaml index 3953bc790..91ea4c469 100644 --- a/tests/test_crs/valid_ycsb-mongo.yaml +++ b/tests/test_crs/valid_ycsb-mongo.yaml @@ -4,6 +4,12 @@ metadata: name: ycsb-mongo-benchmark namespace: my-ripsaw spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-ycsb diff --git a/tests/test_fiod.sh b/tests/test_fiod.sh index f063e4450..cdb5d33e5 100755 --- a/tests/test_fiod.sh +++ b/tests/test_fiod.sh @@ -24,6 +24,8 @@ function functional_test_fio { test_name=$1 cr=$2 echo "Performing: ${test_name}" + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - kubectl apply -f ${cr} long_uuid=$(get_uuid 20) uuid=${long_uuid:0:8} diff --git a/tests/test_flent.sh b/tests/test_flent.sh index 0dda96d87..4602dd519 100755 --- a/tests/test_flent.sh +++ b/tests/test_flent.sh @@ -22,7 +22,8 @@ function functional_test_flent { test_name=$1 cr=$2 echo "Performing: ${test_name}" - kubectl apply -f ${cr} + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - long_uuid=$(get_uuid 20) uuid=${long_uuid:0:8} diff --git a/tests/test_fs_drift.sh b/tests/test_fs_drift.sh index 6ef75b81b..67072a454 100755 --- a/tests/test_fs_drift.sh +++ b/tests/test_fs_drift.sh @@ -23,7 +23,8 @@ function functional_test_fs_drift { test_name=$1 cr=$2 echo "Performing: ${test_name}" - kubectl apply -f ${cr} + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - long_uuid=$(get_uuid 20) uuid=${long_uuid:0:8} diff --git a/tests/test_hammerdb.sh b/tests/test_hammerdb.sh index 7da818483..6b6cbec38 100755 --- a/tests/test_hammerdb.sh +++ b/tests/test_hammerdb.sh @@ -23,7 +23,8 @@ function functional_test_hammerdb { wait_clean apply_operator initdb_pod - kubectl apply -f tests/test_crs/valid_hammerdb.yaml + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" tests/test_crs/valid_hammerdb.yaml | kubectl apply -f - long_uuid=$(get_uuid 20) uuid=${long_uuid:0:8} diff --git a/tests/test_iperf3.sh b/tests/test_iperf3.sh index accdb960d..25ce92a0e 100644 --- a/tests/test_iperf3.sh +++ b/tests/test_iperf3.sh @@ -21,7 +21,8 @@ function functional_test_iperf { wait_clean apply_operator echo "Performing iperf3: ${1}" - sed -e "s/hostnetwork:.*/${1}/g" tests/test_crs/valid_iperf3.yaml | kubectl apply -f - + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" -e "s/hostnetwork:.*/${1}/g" tests/test_crs/valid_iperf3.yaml | kubectl apply -f - long_uuid=$(get_uuid 20) uuid=${long_uuid:0:8} diff --git a/tests/test_smallfile.sh b/tests/test_smallfile.sh index 30852b199..58e5c102b 100755 --- a/tests/test_smallfile.sh +++ b/tests/test_smallfile.sh @@ -25,7 +25,8 @@ function functional_test_smallfile { test_name=$1 cr=$2 echo "Performing: ${test_name}" - kubectl apply -f ${cr} + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - long_uuid=$(get_uuid 20) uuid=${long_uuid:0:8} diff --git a/tests/test_stressng.sh b/tests/test_stressng.sh index 9fa07c10b..cb15567bb 100755 --- a/tests/test_stressng.sh +++ b/tests/test_stressng.sh @@ -19,7 +19,8 @@ trap finish EXIT function functional_test_stressng { wait_clean apply_operator - kubectl apply -f tests/test_crs/valid_stressng.yaml + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" tests/test_crs/valid_stressng.yaml | kubectl apply -f - long_uuid=$(get_uuid 20) uuid=${long_uuid:0:8} diff --git a/tests/test_uperf.sh b/tests/test_uperf.sh index 4f1b81272..543ce966d 100755 --- a/tests/test_uperf.sh +++ b/tests/test_uperf.sh @@ -19,10 +19,11 @@ trap finish EXIT function functional_test_uperf { wait_clean apply_operator + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) test_name=$1 cr=$2 echo "Performing: ${test_name}" - kubectl apply -f ${cr} + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - long_uuid=$(get_uuid 20) uuid=${long_uuid:0:8} diff --git a/tests/test_vegeta.sh b/tests/test_vegeta.sh index 57bd144b1..687d48fa0 100644 --- a/tests/test_vegeta.sh +++ b/tests/test_vegeta.sh @@ -25,7 +25,8 @@ function functional_test_vegeta { test_name=$1 cr=$2 echo "Performing: ${test_name}" - kubectl apply -f ${cr} + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - long_uuid=$(get_uuid 20) uuid=${long_uuid:0:8} diff --git a/tests/test_ycsb.sh b/tests/test_ycsb.sh index e2aa69ecd..143ca0e61 100644 --- a/tests/test_ycsb.sh +++ b/tests/test_ycsb.sh @@ -64,7 +64,8 @@ spec: ports: - containerPort: 27017 EOF - kubectl apply -f tests/test_crs/valid_ycsb-mongo.yaml + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" tests/test_crs/valid_ycsb-mongo.yaml | kubectl apply -f - long_uuid=$(get_uuid 20) uuid=${long_uuid:0:8} From 25b2ed59d7f1b15019c5119853dac04ea8104afe Mon Sep 17 00:00:00 2001 From: Russell Zaleski Date: Wed, 7 Apr 2021 06:25:13 -0400 Subject: [PATCH 054/249] Add log generator workload --- README.md | 1 + docs/log_generator.md | 106 ++++++++++++++++ .../ripsaw_v1alpha1_log_generator_cr.yaml | 38 ++++++ roles/log_generator/tasks/main.yml | 89 ++++++++++++++ .../log_generator/templates/log_generator.yml | 115 ++++++++++++++++++ tests/common.sh | 2 + tests/test_crs/valid_log_generator.yaml | 18 +++ tests/test_list | 1 + tests/test_log_generator.sh | 47 +++++++ 9 files changed, 417 insertions(+) create mode 100644 docs/log_generator.md create mode 100644 resources/crds/ripsaw_v1alpha1_log_generator_cr.yaml create mode 100644 roles/log_generator/tasks/main.yml create mode 100644 roles/log_generator/templates/log_generator.yml create mode 100644 tests/test_crs/valid_log_generator.yaml create mode 100755 tests/test_log_generator.sh diff --git a/README.md b/README.md index a8246c0fc..2790e28de 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ helm delete benchmark-operator -n my-ripsaw --purge | [oslat](docs/oslat.md) | Real-Time Latency | Yes | Used, default : 3second | Not Supported | Preview | No | | [testpmd](docs/testpmd.md) | TestPMD DPDK App | No | Used | Not Supported | Preview | No | | [Flent](docs/flent.md) | Network Performance | Yes | Used, default : 3second | Not Supported | Not Supported | Yes | +| [Log-Generator](docs/log_generator.md) | Log Throughput to Backend | Yes | Used, default : 3second | Not Supported | Yes | Yes | ### Reconciliation diff --git a/docs/log_generator.md b/docs/log_generator.md new file mode 100644 index 000000000..7019a4dd0 --- /dev/null +++ b/docs/log_generator.md @@ -0,0 +1,106 @@ +# Log Generator + +## What does it do? + +The Log Generator writes messages of a set size and at a set rate to stdout. If provided +it will also verify that all messages were recieved by the backend log aggregator. +This data will also be indexed if Elasticsearch information is provided. + +*NOTE* This workload will not deploy the backend log aggregator + +## Variables + +### Required variables: + +`size` the size, in bytes, of the message to send + +`duration` how long, in minutes, messages should be sent for + +`messages_per_second` the number of messages per second (mutually exclusive with messages_per_minute) + +`messages_per_minute` the number of messages per minute (mutually exclusive with messages_per_second) + +### Optional variables: + +`pod-count` total number of log generator pods to launch (default: 1) + +`timeout` how long, in seconds, after have been sent to allow the backend service to receive all the messages (default: 600) + +### Verification variables: + +To verify your messages have been received by the backend aggregator you must provide information for ONLY ONE of the supported +backends: Elasticsearch or AWS CloudWatch + +Elasticsearch Backend: + +`es_url` the elasticsearch url to query for results. + +`es_index` the index to search for the sent messages (default: app*) + +`es_token` the bearer token to use to access elasticsearch if required + +AWS CloudWatch: + +`cloudwatch-log-group` the aws cloudwatch log group to query + +`aws_region` the region that cloudwatch is deployed to + +`aws_access_key` the access key to use to query cloudwatch + +`aws_secret_key` the secret key to use to query cloudwatch + +Your resource file may look like this when using an Elasticsearch Backend: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: my-ripsaw +spec: + elasticsearch: + url: "http://es-instance.com:9200" + index_name: log-generator + workload: + name: log_generator + args: + pod_count: 2 + size: 512 + messages_per_second: 10 + duration: 1 + es_url: "https://my-es-backend.com" + es_token: "sha256~myToken" + timeout: 600 + label: + key: foo + value: " +``` + +Your resource file may look like this when using an AWS CloudWatch Backend: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: my-ripsaw +spec: + elasticsearch: + url: "http://es-instance.com:9200" + index_name: log-generator + workload: + name: log_generator + args: + pod_count: 10 + size: 1024 + messages_per_second: 100 + duration: 10 + cloudwatch_log_group: "my_log_group" + aws_region: "us-west-2" + aws_access_key: "myKey" + aws_secret_token: "myToken" + timeout: 800 + label: + key: bar + value: " +``` diff --git a/resources/crds/ripsaw_v1alpha1_log_generator_cr.yaml b/resources/crds/ripsaw_v1alpha1_log_generator_cr.yaml new file mode 100644 index 000000000..5940f8ac3 --- /dev/null +++ b/resources/crds/ripsaw_v1alpha1_log_generator_cr.yaml @@ -0,0 +1,38 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: my-ripsaw +spec: + elasticsearch: + # Elastic search instance with full URL format. https://elastic.apps.org:9200 + url: http://my.elasticsearch.server:80 + index_name: log-generator + prometheus: + # Elastic search instance with full URL format. https://elastic.apps.org:9200 + es_url: http://my.elasticsearch.server:80 + # Prometheus bearer token + prom_token: PROMETHEUS_BEARER_TOKEN + # Prometheus URL with full URL format. https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: log_generator + args: + # Number of log generator pods to launch + pod_count: 1 + # size of message in bytes + size: 512 + # number of messages per second to send + messages_per_second: 10 + # duration of time to send messages in minutes + duration: 1 + # url of the backend elasticsearch to be used for verification + es_url: "https://my_es_server" + # bearer token to use to access the backend elasticsearch + es_token: "sha256~myToken" + # amount of time after sending all the messages to wait for the backend to receive them all + timeout: 600 + # Label of host to target + label: + key: foo + value: "" diff --git a/roles/log_generator/tasks/main.yml b/roles/log_generator/tasks/main.yml new file mode 100644 index 000000000..f9cdc66f5 --- /dev/null +++ b/roles/log_generator/tasks/main.yml @@ -0,0 +1,89 @@ +--- +- name: Get benchmark state + k8s_facts: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + register: benchmark_state + +- operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Log Generator Pods" + complete: false + when: benchmark_state.resources[0].status.state is not defined + +- name: Get benchmark state + k8s_facts: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + register: benchmark_state + +- block: + + - name: Create log generator pods + k8s: + state: present + definition: "{{ lookup('template', 'log_generator.yml') }}" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Building Pods" + + when: benchmark_state.resources[0].status.state == "Starting Log Generator Pods" + +- block: + + - name: Get server pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = log-generator-{{ trunc_uuid }} + register: log_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Log Generator Running" + when: "workload_args.pod_count|default('1')|int == log_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + + when: benchmark_state.resources[0].status.state == "Building Pods" + +- block: + + - name: Get server pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = log-generator-{{ trunc_uuid }} + register: log_pods + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + when: "workload_args.pod_count|default('1')|int == (log_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + + when: benchmark_state.resources[0].status.state == "Log Generator Running" diff --git a/roles/log_generator/templates/log_generator.yml b/roles/log_generator/templates/log_generator.yml new file mode 100644 index 000000000..94137b441 --- /dev/null +++ b/roles/log_generator/templates/log_generator.yml @@ -0,0 +1,115 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: 'log-generator-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' +spec: + parallelism: {{ workload_args.pod_count | default(1) | int }} + template: + metadata: + labels: + app: log-generator-{{ trunc_uuid }} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.tolerations is defined %} + tolerations: + - key: {{ workload_args.tolerations.key }} + value: {{ workload_args.tolerations.value }} + effect: {{ workload_args.tolerations.effect }} +{% endif %} +{% if workload_args.label is defined %} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ workload_args.label.key }} + operator: In + values: + - {{ workload_args.label.value }} +{% endif %} + containers: + - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/log_generator:latest') }} + name: log-generator + env: + - name: my_node_name + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: my_pod_name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: uuid + value: "{{ uuid }}" + - name: test_user + value: "{{ test_user | default("ripsaw") }}" + - name: clustername + value: "{{ clustername }}" +{% if elasticsearch is defined %} + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("log-generator") }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(true) }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" +{% endif %} +{% if prometheus is defined %} + - name: prom_es + value: "{{ prometheus.es_url }}" + - name: prom_parallel + value: "{{ prometheus.es_parallel | default(false) }}" + - name: prom_token + value: "{{ prometheus.prom_token | default() }}" + - name: prom_url + value: "{{ prometheus.prom_url | default() }}" +{% endif %} + command: ["/bin/sh", "-c"] + args: + - > +{% if workload_args.pod_count | default(1) | int > 1 %} + echo "Waiting for all pods to be Ready"; + redis-cli -h {{ bo.resources[0].status.podIP }} INCR "log-generator-{{ trunc_uuid }}" > /dev/null 2>&1; + pods=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "log-generator-{{ trunc_uuid }}"`; + while [[ $pods != {{ workload_args.pod_count | int }} ]]; do + pods=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "log-generator-{{ trunc_uuid }}"`; + sleep 1; + done; +{% endif %} + run_snafu + --tool log_generator +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + -u "{{ uuid }}" + --size "{{ workload_args.size | default(512) }}" +{% if workload_args.messages_per_second is defined %} + --messages-per-second "{{ workload_args.messages_per_second }}" +{% endif %} +{% if workload_args.messages_per_minute is defined %} + --messages-per-minue "{{ workload_args.messages_per_minute }}" +{% endif %} + --duration "{{ workload_args.duration | default(10) }}" +{% if workload_args.es_url is defined %} + --es-url "{{ workload_args.es_url }}" +{% endif %} +{% if workload_args.es_token is defined %} + --es-token "{{ workload_args.es_token }}" +{% endif %} +{% if workload_args.cloudwatch_log_group is defined %} + --cloudwatch-log-group "{{ workload_args.cloudwatch_log_group }}" + --aws-region "{{ workload_args.aws_region }}" + --aws-access-key "{{ workload_args.aws_access_key }}" + --aws-secret-key "{{ workload_args.aws_secret_key }}" +{% endif %} + --pod-name ${my_pod_name} + --timeout "{{ workload_args.timeout | default(600) }}" + --pod-count {{ workload_args.pod_count | default(1) | int }} + imagePullPolicy: Always + restartPolicy: Never +{% include "metadata.yml.j2" %} diff --git a/tests/common.sh b/tests/common.sh index 76e8722db..ef93c038a 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -28,6 +28,7 @@ function populate_test_list { if [[ $(echo ${item} | grep 'roles/scale_openshift') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/kube-burner') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/flent') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/log_generator') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi # Check for changes in cr files @@ -47,6 +48,7 @@ function populate_test_list { if [[ $(echo ${item} | grep 'valid_scale*') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_kube-burner*') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_flent*') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_log_generator*') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi # Check for changes in test scripts diff --git a/tests/test_crs/valid_log_generator.yaml b/tests/test_crs/valid_log_generator.yaml new file mode 100644 index 000000000..cf3e56e44 --- /dev/null +++ b/tests/test_crs/valid_log_generator.yaml @@ -0,0 +1,18 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: my-ripsaw +spec: + elasticsearch: + url: ES_SERVER + index_name: log-generator + workload: + name: log_generator + args: + pod_count: 1 + size: 512 + messages_per_second: 1 + duration: 1 + es_url: ES_SERVER + timeout: 600 diff --git a/tests/test_list b/tests/test_list index 3825b39de..0e0208c32 100644 --- a/tests/test_list +++ b/tests/test_list @@ -14,3 +14,4 @@ test_scale_openshift.sh test_stressng.sh test_kubeburner.sh test_flent.sh +test_log_generator.sh diff --git a/tests/test_log_generator.sh b/tests/test_log_generator.sh new file mode 100755 index 000000000..bd49169b6 --- /dev/null +++ b/tests/test_log_generator.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up Log Generator Test" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_log_generator { + wait_clean + apply_operator + test_name=$1 + cr=$2 + + echo "Performing: ${test_name}" + kubectl apply -f ${cr} + long_uuid=$(get_uuid 20) + uuid=${long_uuid:0:8} + + log_gen_pod=$(get_pod "app=log-generator-$uuid" 300) + wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=log-generator-$uuid pods --timeout=300s" "300s" $log_gen_pod + wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=log-generator-$uuid jobs --timeout=300s" "300s" $log_gen_pod + + index="log-generator-results" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + kubectl logs "$log_gen_pod" -n my-ripsaw + exit 1 + fi + kubectl delete -f ${cr} +} + +figlet $(basename $0) +functional_test_log_generator "Log Generator" tests/test_crs/valid_log_generator.yaml From 8217a8c8b96df9a1a69554bf59ed0a64023f8c19 Mon Sep 17 00:00:00 2001 From: Russell Zaleski Date: Thu, 8 Apr 2021 07:59:50 -0400 Subject: [PATCH 055/249] Fixing documented variables to have _'s instead of -'s (#561) --- docs/log_generator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/log_generator.md b/docs/log_generator.md index 7019a4dd0..329aff9df 100644 --- a/docs/log_generator.md +++ b/docs/log_generator.md @@ -22,7 +22,7 @@ This data will also be indexed if Elasticsearch information is provided. ### Optional variables: -`pod-count` total number of log generator pods to launch (default: 1) +`pod_count` total number of log generator pods to launch (default: 1) `timeout` how long, in seconds, after have been sent to allow the backend service to receive all the messages (default: 600) @@ -41,7 +41,7 @@ Elasticsearch Backend: AWS CloudWatch: -`cloudwatch-log-group` the aws cloudwatch log group to query +`cloudwatch_log_group` the aws cloudwatch log group to query `aws_region` the region that cloudwatch is deployed to From 550ac7ec4107181f951cb31aa6a0a0138a884f78 Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Wed, 14 Apr 2021 04:31:27 -0500 Subject: [PATCH 056/249] Enhance node-density heavy to use livenessprobes (#564) * Enhance node-density heavy to use livenessprobes Using livenessprobes in addition to readinessprobes increases the load on the kubelet which is what we want with this test. * Remove duplicated imagePullPolicy Signed-off-by: Raul Sevilla Co-authored-by: Raul Sevilla --- roles/kube-burner/files/app-deployment.yml | 10 +++++++++- roles/kube-burner/templates/max-namespaces.yml.j2 | 1 + roles/kube-burner/templates/node-density-heavy.yml.j2 | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/roles/kube-burner/files/app-deployment.yml b/roles/kube-burner/files/app-deployment.yml index 026e4de77..6043f37fa 100644 --- a/roles/kube-burner/files/app-deployment.yml +++ b/roles/kube-burner/files/app-deployment.yml @@ -13,6 +13,7 @@ spec: containers: - name: perfapp image: quay.io/rsevilla/perfapp:latest + imagePullPolicy: Always readinessProbe: httpGet: path: /ready @@ -21,6 +22,14 @@ spec: failureThreshold: 1 timeoutSeconds: 60 initialDelaySeconds: 30 + livenessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: {{ .livenessPeriod }} + failureThreshold: 1 + timeoutSeconds: 15 + initialDelaySeconds: 30 ports: - containerPort: 8080 protocol: TCP @@ -37,7 +46,6 @@ spec: value: '5432' - name: POSTGRESQL_RETRY_INTERVAL value: '5' - imagePullPolicy: IfNotPresent securityContext: privileged: false restartPolicy: Always diff --git a/roles/kube-burner/templates/max-namespaces.yml.j2 b/roles/kube-burner/templates/max-namespaces.yml.j2 index e612b88b5..2fa8b7b91 100644 --- a/roles/kube-burner/templates/max-namespaces.yml.j2 +++ b/roles/kube-burner/templates/max-namespaces.yml.j2 @@ -37,6 +37,7 @@ jobs: replicas: 5 inputVars: readinessPeriod: 10 + livenessPeriod: 10 nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" diff --git a/roles/kube-burner/templates/node-density-heavy.yml.j2 b/roles/kube-burner/templates/node-density-heavy.yml.j2 index 8d1aeaf17..10584b18f 100644 --- a/roles/kube-burner/templates/node-density-heavy.yml.j2 +++ b/roles/kube-burner/templates/node-density-heavy.yml.j2 @@ -37,6 +37,7 @@ jobs: replicas: 1 inputVars: readinessPeriod: 10 + livenessPeriod: 10 nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" From f4adc3fcb6f0bca3c27a30dd8f3398f5e02c78b1 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 12 Apr 2021 12:00:19 +0200 Subject: [PATCH 057/249] API metrics Signed-off-by: Raul Sevilla --- .../kube-burner/files/metrics-aggregated.yaml | 79 +++++++++---------- roles/kube-burner/files/metrics.yaml | 44 ++++++----- 2 files changed, 62 insertions(+), 61 deletions(-) diff --git a/roles/kube-burner/files/metrics-aggregated.yaml b/roles/kube-burner/files/metrics-aggregated.yaml index 8df5b3979..e5a6d4e0a 100644 --- a/roles/kube-burner/files/metrics-aggregated.yaml +++ b/roles/kube-burner/files/metrics-aggregated.yaml @@ -1,4 +1,18 @@ metrics: +# API server + - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0 + metricName: API99thLatency + + - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0 + metricName: APIRequestRate + + - query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0 + metricName: APIInflightRequests + +# Container & pod metrics + - query: (sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 + metricName: containerMemory-Masters + - query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|sdn|ovn-kubernetes|.*apiserver|authentication|.*controller-manager|.*scheduler)"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 metricName: containerCPU-Masters @@ -11,12 +25,6 @@ metrics: - query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"}[2m]) * 100 and on (node) kube_node_role{role="infra"}) by (namespace, container)) > 0 metricName: containerCPU-AggregatedInfra - - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!="WATCH"}[5m])) by (le, verb)) > 0 - metricName: API99thLatency - - - query: (sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 - metricName: containerMemory-Masters - - query: (sum(container_memory_rss{pod!="",namespace="openshift-monitoring",name!="",container="prometheus"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0 metricName: containerMemory-Prometheus @@ -26,18 +34,7 @@ metrics: - query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"} and on (node) kube_node_role{role="infra"}) by (container, namespace) metricName: containerMemory-AggregatedInfra - - query: avg(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) - metricName: KubeletCPU-AggregatedWorkers - - - query: avg(process_resident_memory_bytes{service="kubelet",job="kubelet"} and on (node) kube_node_role{role="worker"}) - metricName: KubeletMemory-AggregatedWorkers - - - query: avg(irate(process_cpu_seconds_total{service="kubelet",job="crio"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) - metricName: CrioCPU-AggregatedWorkers - - - query: avg(process_resident_memory_bytes{service="kubelet",job="crio"} and on (node) kube_node_role{role="worker"}) - metricName: CrioMemory-AggregatedWorkers - +# Node metrics - query: (sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)")) > 0 metricName: nodeCPU-Masters @@ -92,24 +89,6 @@ metrics: - query: avg(irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) metricName: txNetworkBytes-AggregatedInfra - - query: (irate(node_network_receive_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)")) > 0 - metricName: rxDroppedPackets-Masters - - - query: (avg(irate(node_network_receive_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device)) > 0 - metricName: rxDroppedPackets-AggregatedWorkers - - - query: (avg(irate(node_network_receive_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device)) > 0 - metricName: rxDroppedPackets-AggregatedInfra - - - query: irate(node_network_transmit_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: txDroppedPackets-Masters - - - query: (avg(irate(node_network_transmit_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device)) > 0 - metricName: txDroppedPackets-AggregatedWorkers - - - query: (avg(irate(node_network_transmit_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device)) > 0 - metricName: txDroppedPackets-AggregatedInfra - - query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") metricName: nodeDiskWrittenBytes-Masters @@ -128,6 +107,7 @@ metrics: - query: avg(rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) metricName: nodeDiskReadBytes-AggregatedInfra +# Etcd metrics - query: sum(rate(etcd_server_leader_changes_seen_total[2m])) metricName: etcdLeaderChangesRate @@ -149,6 +129,17 @@ metrics: - query: etcd_mvcc_db_total_size_in_use_in_bytes metricName: etcdDBLogicalSizeBytes + - query: sum by (cluster_version)(etcd_cluster_version) + metricName: etcdVersion + instant: true + + - query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 + metricName: etcdObjectCount + + - query: histogram_quantile(0.99,sum(rate(etcd_request_duration_seconds_bucket[2m])) by (le,operation,apiserver)) > 0 + metricName: P99APIEtcdRequestLatency + +# Cluster metrics - query: count(kube_namespace_created) metricName: namespaceCount @@ -177,13 +168,17 @@ metrics: - query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 metricName: containerDiskUsage - - query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 - metricName: etcdObjectCount - - query: cluster_version{type="completed"} metricName: clusterVersion instant: true - - query: sum by (cluster_version)(etcd_cluster_version) - metricName: etcdVersion - instant: true +# Golang metrics + + - query: go_memstats_heap_alloc_bytes{job=~"apiserver|api|etcd"} + metricName: goHeapAllocBytes + + - query: go_memstats_heap_inuse_bytes{job=~"apiserver|api|etcd"} + metricName: goHeapInuseBytes + + - query: go_gc_duration_seconds{job=~"apiserver|api|etcd",quantile="1"} + metricName: goGCDurationSeconds diff --git a/roles/kube-burner/files/metrics.yaml b/roles/kube-burner/files/metrics.yaml index 7992d6212..bdad2be8f 100644 --- a/roles/kube-burner/files/metrics.yaml +++ b/roles/kube-burner/files/metrics.yaml @@ -1,13 +1,25 @@ metrics: +# API server + - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0 + metricName: API99thLatency + + - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0 + metricName: APIRequestRate + + - query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0 + metricName: APIInflightRequests + +# Containers & pod metrics - query: sum(irate(container_cpu_usage_seconds_total{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}[2m]) * 100) by (pod, namespace, node) metricName: podCPU - - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!="WATCH"}[5m])) by (le, verb)) > 0 - metricName: API99thLatency - - query: sum(container_memory_rss{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}) by (pod, namespace, node) metricName: podMemory + - query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 + metricName: containerDiskUsage + +# Kubelet & CRI-O metrics - query: sum(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} metricName: kubeletCPU @@ -20,6 +32,7 @@ metrics: - query: sum(process_resident_memory_bytes{service="kubelet",job="crio"}) by (node) and on (node) kube_node_role{role="worker"} metricName: crioMemory +# Node metrics - query: sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) > 0 metricName: nodeCPU @@ -38,12 +51,6 @@ metrics: - query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) metricName: txNetworkBytes - - query: irate(node_network_receive_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) - metricName: rxDroppedPackets - - - query: irate(node_network_transmit_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) - metricName: txDroppedPackets - - query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) metricName: nodeDiskWrittenBytes @@ -53,6 +60,7 @@ metrics: - query: sum(rate(etcd_server_leader_changes_seen_total[2m])) metricName: etcdLeaderChangesRate +# Etcd metrics - query: etcd_server_is_leader > 0 metricName: etcdServerIsLeader @@ -71,6 +79,14 @@ metrics: - query: etcd_mvcc_db_total_size_in_use_in_bytes metricName: etcdDBLogicalSizeBytes + - query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 + metricName: etcdObjectCount + + - query: sum by (cluster_version)(etcd_cluster_version) + metricName: etcdVersion + instant: true + +# Cluster metrics - query: count(kube_namespace_created) metricName: namespaceCount @@ -96,16 +112,6 @@ metrics: - query: sum(kube_node_status_condition{status="true"}) by (condition) metricName: nodeStatus - - query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 - metricName: containerDiskUsage - - - query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 - metricName: etcdObjectCount - - query: cluster_version{type="completed"} metricName: clusterVersion instant: true - - - query: sum by (cluster_version)(etcd_cluster_version) - metricName: etcdVersion - instant: true From f6981d6f9aefd74358ea7e4e0a9e1bf1dcc2e246 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 31 Mar 2021 11:07:16 +0200 Subject: [PATCH 058/249] Update operator-sdk to version v1.5.0 Signed-off-by: Raul Sevilla --- CONTRIBUTING.md | 2 +- build/Dockerfile | 2 +- .../templates/operator.yaml | 11 ----- group_vars/all.yml | 2 +- playbook.yml | 6 +-- resources/operator.yaml | 11 ----- roles/backpack/tasks/main.yml | 14 +++--- roles/byowl/tasks/main.yml | 10 ++-- roles/cerberus/tasks/main.yml | 4 +- roles/common/tasks/main.yml | 2 +- roles/common/templates/networkpolicy.yml.j2 | 6 +-- roles/cyclictest/tasks/main.yml | 18 +++---- roles/cyclictest/templates/cyclictestjob.yaml | 4 +- roles/fio_distributed/tasks/main.yml | 16 +++---- roles/flent/tasks/main.yml | 20 ++++---- roles/fs-drift/tasks/main.yml | 12 ++--- roles/fs-drift/templates/workload_job.yml.j2 | 2 +- roles/hammerdb/tasks/main.yaml | 38 +++++++-------- roles/hammerdb/templates/db_creation.yml | 4 +- .../templates/db_mssql_workload.yml.j2 | 4 +- .../templates/db_mysql_workload.yml.j2 | 4 +- .../templates/db_postgres_workload.yml.j2 | 4 +- roles/iperf3/tasks/main.yml | 16 +++---- roles/kube-burner/tasks/main.yml | 12 ++--- roles/oslat/tasks/main.yml | 18 +++---- roles/oslat/templates/oslatjob.yaml | 4 +- roles/pgbench/tasks/check_clients.yml | 2 +- roles/pgbench/tasks/init.yml | 2 +- roles/pgbench/tasks/main.yml | 6 +-- roles/pgbench/tasks/prep_workload.yml | 2 +- roles/pgbench/tasks/run_workload.yml | 4 +- roles/scale_openshift/tasks/main.yml | 10 ++-- roles/servicemesh/tasks/main.yml | 48 +++++++++---------- roles/servicemesh/templates/job.yaml.j2 | 6 +-- roles/servicemesh/templates/smcp.yaml.j2 | 2 +- roles/smallfile/tasks/main.yml | 10 ++-- roles/stressng/tasks/main.yaml | 18 +++---- .../templates/stressng_workload.yml.j2 | 4 +- .../templates/stressng_workload_vm.yml.j2 | 4 +- roles/sysbench/tasks/main.yml | 10 ++-- roles/testpmd/tasks/main.yml | 16 +++---- roles/uperf/tasks/cleanup.yml | 6 +-- roles/uperf/tasks/init.yml | 2 +- roles/uperf/tasks/next_set.yml | 4 +- roles/uperf/tasks/run_a_set.yml | 2 +- roles/uperf/tasks/send_client_run_signal.yml | 2 +- roles/uperf/tasks/setup.yml | 8 ++-- roles/uperf/tasks/start_client.yml | 6 +-- roles/uperf/tasks/start_server.yml | 8 ++-- roles/uperf/tasks/wait_client_done.yml | 4 +- roles/uperf/tasks/wait_client_ready.yml | 4 +- roles/uperf/tasks/wait_server_ready.yml | 8 ++-- roles/uperf/tasks/wait_set_done.yml | 2 +- roles/uperf/templates/server.yml.j2 | 2 +- roles/uperf/templates/server_vm.yml.j2 | 2 +- roles/uperf/templates/service.yml.j2 | 2 +- roles/uperf/templates/workload.yml.j2 | 2 +- roles/uperf/templates/workload_vm.yml.j2 | 2 +- roles/vegeta/tasks/main.yml | 10 ++-- roles/ycsb/tasks/main.yml | 38 +++++++-------- roles/ycsb/templates/ycsb_load.yaml | 2 +- roles/ycsb/templates/ycsb_run.yaml | 2 +- 62 files changed, 243 insertions(+), 265 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 62489562c..4490a9814 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,7 +112,7 @@ to [uperf](docs/uperf.md) * Add the link for your workload guide to [installation guide](docs/installation.md#running-workloads) * Ensure all resources created are within the `my-ripsaw` namespace, this can be done by setting namespace to use `operator_namespace` var. This is to ensure that the resources aren't defaulted to current active -namespace which is what `meta.namespace` would default to. +namespace which is what `ansible_operator_meta.namespace` would default to. * All resources created as part of your role should use `trunc_uuid` ansible var in their names and labels, so for example [fio-client job template](roles/fio-distributed/templates/client.yaml) has the name `fio-client` and label `app: fiod-client`, instead we'll append the var `trunc_uuid` to both the name and label so it'll be `fio-client-{{ trunc_uuid }}` and label would be `app:fiod-client-{{ trunc_uuid }}`. The reason for this diff --git a/build/Dockerfile b/build/Dockerfile index a32b21982..998fe4f19 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/operator-framework/ansible-operator:v0.17.1 +FROM quay.io/operator-framework/ansible-operator:v1.5.0 USER root COPY requirements.yml ${HOME}/requirements.yml diff --git a/charts/benchmark-operator/templates/operator.yaml b/charts/benchmark-operator/templates/operator.yaml index 62cd15934..92f1a6a7e 100644 --- a/charts/benchmark-operator/templates/operator.yaml +++ b/charts/benchmark-operator/templates/operator.yaml @@ -15,17 +15,6 @@ spec: spec: serviceAccountName: {{ include "benchmark-operator.fullname" . }} containers: - - name: ansible - command: - - /usr/local/bin/ao-logs - - /tmp/ansible-operator/runner - - stdout - image: "{{ .Values.operator.image.repository }}:{{ .Values.operator.image.tag }}" - imagePullPolicy: {{ .Values.operator.image.pullPolicy}} - volumeMounts: - - mountPath: /tmp/ansible-operator/runner - name: runner - readOnly: true - name: benchmark-operator image: "{{ .Values.operator.image.repository }}:{{ .Values.operator.image.tag }}" imagePullPolicy: {{ .Values.operator.image.pullPolicy}} diff --git a/group_vars/all.yml b/group_vars/all.yml index 940081af5..60cc68ab0 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1,4 +1,4 @@ --- -operator_namespace: '{{ meta.namespace }}' +operator_namespace: '{{ ansible_operator_meta.namespace }}' clustername: 'myk8scluster' kernel_cache_drop_svc_port: 9222 diff --git a/playbook.yml b/playbook.yml index 5ce043713..915451755 100644 --- a/playbook.yml +++ b/playbook.yml @@ -20,7 +20,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: cr_state @@ -45,7 +45,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: uuid: "{{ uuid }}" @@ -73,7 +73,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: metadata: "Collecting" diff --git a/resources/operator.yaml b/resources/operator.yaml index dd4e0291a..d057ead85 100644 --- a/resources/operator.yaml +++ b/resources/operator.yaml @@ -29,17 +29,6 @@ spec: - "" serviceAccountName: benchmark-operator containers: - - name: ansible - command: - - /usr/local/bin/ao-logs - - /tmp/ansible-operator/runner - - stdout - image: quay.io/benchmark-operator/benchmark-operator:master - imagePullPolicy: "Always" - volumeMounts: - - mountPath: /tmp/ansible-operator/runner - name: runner - readOnly: true - name: benchmark-operator image: quay.io/benchmark-operator/benchmark-operator:master imagePullPolicy: Always diff --git a/roles/backpack/tasks/main.yml b/roles/backpack/tasks/main.yml index 549651f70..68fcfa6c4 100644 --- a/roles/backpack/tasks/main.yml +++ b/roles/backpack/tasks/main.yml @@ -3,14 +3,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: benchmark_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Metadata Collecting" @@ -22,7 +22,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: benchmark_state @@ -61,7 +61,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Metadata Collecting" @@ -73,7 +73,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: benchmark_state @@ -92,7 +92,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: null @@ -103,7 +103,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: benchmark_state diff --git a/roles/byowl/tasks/main.yml b/roles/byowl/tasks/main.yml index c9a3b7ce8..05042d6d4 100644 --- a/roles/byowl/tasks/main.yml +++ b/roles/byowl/tasks/main.yml @@ -4,14 +4,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Starting @@ -22,7 +22,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -37,7 +37,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Running @@ -59,7 +59,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/cerberus/tasks/main.yml b/roles/cerberus/tasks/main.yml index d7e50d607..9a53a2c41 100644 --- a/roles/cerberus/tasks/main.yml +++ b/roles/cerberus/tasks/main.yml @@ -9,7 +9,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: cerberus: "Connected" @@ -20,7 +20,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: cerberus: "Connected" diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 1f1d60fd1..1270b776c 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -5,7 +5,7 @@ kind: NetworkPolicy api_version: networking.k8s.io/v1 namespace: '{{ operator_namespace }}' - name: "{{ meta.name }}-networkpolicy-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-networkpolicy-{{ trunc_uuid }}" register: network_policy - name: Create Network policy if enabled diff --git a/roles/common/templates/networkpolicy.yml.j2 b/roles/common/templates/networkpolicy.yml.j2 index 6267d2196..f1b2703a0 100644 --- a/roles/common/templates/networkpolicy.yml.j2 +++ b/roles/common/templates/networkpolicy.yml.j2 @@ -1,14 +1,14 @@ kind: NetworkPolicy apiVersion: networking.k8s.io/v1 metadata: - name: "{{ meta.name }}-networkpolicy-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-networkpolicy-{{ trunc_uuid }}" namespace: '{{ operator_namespace }}' spec: podSelector: matchLabels: - type: "{{ meta.name }}-bench-server-{{ trunc_uuid }}" + type: "{{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}" ingress: - from: - podSelector: matchLabels: - type: "{{ meta.name }}-bench-client-{{ trunc_uuid }}" + type: "{{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }}" diff --git a/roles/cyclictest/tasks/main.yml b/roles/cyclictest/tasks/main.yml index 86c426944..a9fd46996 100644 --- a/roles/cyclictest/tasks/main.yml +++ b/roles/cyclictest/tasks/main.yml @@ -4,14 +4,14 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Building @@ -22,7 +22,7 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -34,7 +34,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: cyclictest.sh: "{{ lookup('file', 'cyclictest.sh') }}" @@ -43,7 +43,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: ConfigMaps Created @@ -60,7 +60,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Benchmark Running @@ -74,7 +74,7 @@ k8s_facts: kind: Job api_version: batch/v1 - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: "{{ operator_namespace }}" register: cyclictest_status @@ -82,7 +82,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Benchmark Complete @@ -93,7 +93,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Failed diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index c6bf0b398..7eb02f23f 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: template: @@ -85,7 +85,7 @@ spec: path: /dev/cpu_dma_latency - name: cyclictest-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0555 restartPolicy: Never serviceAccountName: benchmark-operator diff --git a/roles/fio_distributed/tasks/main.yml b/roles/fio_distributed/tasks/main.yml index 484ce865a..f74ae4b94 100644 --- a/roles/fio_distributed/tasks/main.yml +++ b/roles/fio_distributed/tasks/main.yml @@ -3,14 +3,14 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Building @@ -21,7 +21,7 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -75,7 +75,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: StartingServers @@ -97,7 +97,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: StartingClient @@ -125,7 +125,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: StartingClient @@ -169,7 +169,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Running @@ -190,7 +190,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/flent/tasks/main.yml b/roles/flent/tasks/main.yml index 998a3f6c5..e34c35852 100644 --- a/roles/flent/tasks/main.yml +++ b/roles/flent/tasks/main.yml @@ -4,14 +4,14 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Building @@ -22,7 +22,7 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -56,7 +56,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Servers" @@ -78,7 +78,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Clients" @@ -107,7 +107,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Waiting for Clients @@ -130,7 +130,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Clients Running @@ -147,7 +147,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Running" @@ -168,7 +168,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Cleanup @@ -211,7 +211,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/fs-drift/tasks/main.yml b/roles/fs-drift/tasks/main.yml index 2faf68df1..8c23bbe6f 100644 --- a/roles/fs-drift/tasks/main.yml +++ b/roles/fs-drift/tasks/main.yml @@ -4,14 +4,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Starting @@ -22,7 +22,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -83,7 +83,7 @@ kind: Job apiVersion: batch/v1 metadata: - name: "{{ meta.name }}-fs-drift-publisher-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-fs-drift-publisher-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -119,7 +119,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Running @@ -141,7 +141,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/fs-drift/templates/workload_job.yml.j2 b/roles/fs-drift/templates/workload_job.yml.j2 index 41f706088..4f743d4ea 100644 --- a/roles/fs-drift/templates/workload_job.yml.j2 +++ b/roles/fs-drift/templates/workload_job.yml.j2 @@ -2,7 +2,7 @@ kind: Job apiVersion: batch/v1 metadata: - name: "{{ meta.name }}-fs-drift-client-benchmark-{{ trunc_uuid }}-pod-{{ item }}" + name: "{{ ansible_operator_meta.name }}-fs-drift-client-benchmark-{{ trunc_uuid }}-pod-{{ item }}" namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 diff --git a/roles/hammerdb/tasks/main.yaml b/roles/hammerdb/tasks/main.yaml index 115c86696..617dc2bf8 100644 --- a/roles/hammerdb/tasks/main.yaml +++ b/roles/hammerdb/tasks/main.yaml @@ -3,14 +3,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Building @@ -21,7 +21,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -33,7 +33,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-creator-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: createdb.tcl: "{{ lookup('template', 'createdb_mysql.tcl.j2') }}" @@ -45,7 +45,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-creator-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: createdb.tcl: "{{ lookup('template', 'createdb_mssql.tcl.j2') }}" @@ -57,7 +57,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-creator-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: createdb.tcl: "{{ lookup('template', 'createdb_pg.tcl.j2') }}" @@ -69,7 +69,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: tpcc-workload-mysql.tcl: "{{ lookup('template', 'tpcc-workload-mysql.tcl.j2') }}" @@ -81,7 +81,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: tpcc-workload-mssql.tcl: "{{ lookup('template', 'tpcc-workload-mssql.tcl.j2') }}" @@ -93,7 +93,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: tpcc-workload-pg.tcl: "{{ lookup('template', 'tpcc-workload-pg.tcl.j2') }}" @@ -102,7 +102,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Starting DB @@ -112,7 +112,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Starting DB Workload @@ -122,7 +122,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: ConfigMaps Created @@ -140,7 +140,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: DB Creating @@ -154,14 +154,14 @@ k8s_facts: kind: Job api_version: batch/v1 - name: '{{ meta.name }}-creator-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' namespace: "{{ operator_namespace }}" register: hammerdb_creator_pod - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Starting DB Workload @@ -171,7 +171,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: DB Created @@ -202,7 +202,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: DB workload running @@ -215,14 +215,14 @@ k8s_facts: kind: Job api_version: batch/v1 - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: "{{ operator_namespace }}" register: hammerdb_workload_pod - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: DB Workload Complete diff --git a/roles/hammerdb/templates/db_creation.yml b/roles/hammerdb/templates/db_creation.yml index bf3f844ed..83edfc0ab 100644 --- a/roles/hammerdb/templates/db_creation.yml +++ b/roles/hammerdb/templates/db_creation.yml @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: "job" metadata: - name: "{{ meta.name }}-creator-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -26,7 +26,7 @@ spec: volumes: - name: hammerdb-creator-volume configMap: - name: "{{ meta.name }}-creator-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" defaultMode: 0640 restartPolicy: OnFailure {% include "metadata.yml.j2" %} diff --git a/roles/hammerdb/templates/db_mssql_workload.yml.j2 b/roles/hammerdb/templates/db_mssql_workload.yml.j2 index ef681a2e1..1a366a976 100644 --- a/roles/hammerdb/templates/db_mssql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mssql_workload.yml.j2 @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: "job" metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -89,7 +89,7 @@ spec: volumes: - name: hammerdb-workload-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0640 restartPolicy: OnFailure {% include "metadata.yml.j2" %} diff --git a/roles/hammerdb/templates/db_mysql_workload.yml.j2 b/roles/hammerdb/templates/db_mysql_workload.yml.j2 index 90e93146e..c1a6f1467 100644 --- a/roles/hammerdb/templates/db_mysql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mysql_workload.yml.j2 @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: "job" metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -83,7 +83,7 @@ spec: volumes: - name: hammerdb-workload-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0640 restartPolicy: OnFailure {% include "metadata.yml.j2" %} diff --git a/roles/hammerdb/templates/db_postgres_workload.yml.j2 b/roles/hammerdb/templates/db_postgres_workload.yml.j2 index 7569a7be9..4622808d8 100644 --- a/roles/hammerdb/templates/db_postgres_workload.yml.j2 +++ b/roles/hammerdb/templates/db_postgres_workload.yml.j2 @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: "job" metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -87,7 +87,7 @@ spec: volumes: - name: hammerdb-workload-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0640 restartPolicy: OnFailure {% include "metadata.yml.j2" %} diff --git a/roles/iperf3/tasks/main.yml b/roles/iperf3/tasks/main.yml index 3889060d9..db2d34d8e 100644 --- a/roles/iperf3/tasks/main.yml +++ b/roles/iperf3/tasks/main.yml @@ -3,14 +3,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Starting @@ -21,7 +21,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -36,7 +36,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Server" @@ -58,7 +58,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Clients" @@ -86,7 +86,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Waiting for Clients" @@ -107,7 +107,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Running @@ -129,7 +129,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/kube-burner/tasks/main.yml b/roles/kube-burner/tasks/main.yml index eb4792007..fac7173c3 100644 --- a/roles/kube-burner/tasks/main.yml +++ b/roles/kube-burner/tasks/main.yml @@ -4,14 +4,14 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Building @@ -22,7 +22,7 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -144,7 +144,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Running @@ -165,7 +165,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete @@ -176,7 +176,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Failed diff --git a/roles/oslat/tasks/main.yml b/roles/oslat/tasks/main.yml index e2c3b7d4e..03afb5c88 100644 --- a/roles/oslat/tasks/main.yml +++ b/roles/oslat/tasks/main.yml @@ -4,14 +4,14 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Building @@ -22,7 +22,7 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -34,7 +34,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: oslat.sh: "{{ lookup('file', 'oslat.sh') }}" @@ -43,7 +43,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: ConfigMaps Created @@ -60,7 +60,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Benchmark Running @@ -74,7 +74,7 @@ k8s_facts: kind: Job api_version: batch/v1 - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: "{{ operator_namespace }}" register: oslat_status @@ -82,7 +82,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Benchmark Complete @@ -93,7 +93,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Failed diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index 9ed2636e5..348d35883 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: template: @@ -85,7 +85,7 @@ spec: path: /dev/cpu_dma_latency - name: oslat-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0555 restartPolicy: Never serviceAccountName: benchmark-operator diff --git a/roles/pgbench/tasks/check_clients.yml b/roles/pgbench/tasks/check_clients.yml index cbec31cfd..102b03690 100644 --- a/roles/pgbench/tasks/check_clients.yml +++ b/roles/pgbench/tasks/check_clients.yml @@ -6,7 +6,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Run Workload" diff --git a/roles/pgbench/tasks/init.yml b/roles/pgbench/tasks/init.yml index f9a352a7b..f3e144f7c 100644 --- a/roles/pgbench/tasks/init.yml +++ b/roles/pgbench/tasks/init.yml @@ -5,7 +5,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Prep Workload" diff --git a/roles/pgbench/tasks/main.yml b/roles/pgbench/tasks/main.yml index 0ba50dd7b..2bea5e9d3 100644 --- a/roles/pgbench/tasks/main.yml +++ b/roles/pgbench/tasks/main.yml @@ -3,14 +3,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Init" @@ -21,7 +21,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state diff --git a/roles/pgbench/tasks/prep_workload.yml b/roles/pgbench/tasks/prep_workload.yml index c361c1287..b886672a4 100644 --- a/roles/pgbench/tasks/prep_workload.yml +++ b/roles/pgbench/tasks/prep_workload.yml @@ -14,7 +14,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Check Clients" diff --git a/roles/pgbench/tasks/run_workload.yml b/roles/pgbench/tasks/run_workload.yml index 5ef258178..3d1d365c7 100644 --- a/roles/pgbench/tasks/run_workload.yml +++ b/roles/pgbench/tasks/run_workload.yml @@ -18,7 +18,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Prep Workload" @@ -28,7 +28,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Complete" diff --git a/roles/scale_openshift/tasks/main.yml b/roles/scale_openshift/tasks/main.yml index 3ebfa25aa..e2ade80e9 100644 --- a/roles/scale_openshift/tasks/main.yml +++ b/roles/scale_openshift/tasks/main.yml @@ -3,14 +3,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: benchmark_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Scale Pod" @@ -21,7 +21,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: benchmark_state @@ -35,7 +35,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Scaling Cluster" @@ -56,7 +56,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/servicemesh/tasks/main.yml b/roles/servicemesh/tasks/main.yml index 65e45eb1a..71371e653 100644 --- a/roles/servicemesh/tasks/main.yml +++ b/roles/servicemesh/tasks/main.yml @@ -3,19 +3,19 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - name: Define variables set_fact: servicemesh: "{{ servicemesh_defaults | combine(workload_args, recursive=True) }}" - controlplane_namespace: "{{ meta.name }}-controlplane-{{ trunc_uuid }}" - workload_namespace: "{{ meta.name }}-workload-{{ trunc_uuid }}" + controlplane_namespace: "{{ ansible_operator_meta.name }}-controlplane-{{ trunc_uuid }}" + workload_namespace: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" owner: - apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" uid: "{{ resource_state.resources[0].metadata.uid }}" @@ -59,7 +59,7 @@ apiVersion: hyperfoil.io/v1alpha2 kind: Hyperfoil metadata: - name: '{{ meta.name }}-hyperfoil-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-hyperfoil-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' ownerReferences: "{{ owner }}" spec: @@ -69,7 +69,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Init" @@ -80,7 +80,7 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -90,7 +90,7 @@ k8s_info: api_version: hyperfoil.io/v1alpha2 kind: Hyperfoil - name: "{{ meta.name }}-hyperfoil-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-hyperfoil-{{ trunc_uuid }}" namespace: '{{ operator_namespace }}' register: hf_state @@ -98,7 +98,7 @@ k8s_info: api_version: maistra.io/v1 kind: ServiceMeshControlPlane - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: "{{ controlplane_namespace }}" register: smcp_state @@ -110,7 +110,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Deploy" @@ -120,7 +120,7 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -136,7 +136,7 @@ when: ingress_route.resources | length > 0 set_fact: #TODO: maybe use trunc_uuid in the hostname? - app_route: "{{ meta.name }}-app-{{ trunc_uuid}}{{ ingress_route.resources[0].status.ingress[0].host | regex_replace('^[^.]*', '')}}" + app_route: "{{ ansible_operator_meta.name }}-app-{{ trunc_uuid}}{{ ingress_route.resources[0].status.ingress[0].host | regex_replace('^[^.]*', '')}}" - when: resource_state.resources[0].status.state == "Deploy" block: @@ -200,7 +200,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Ready" @@ -210,7 +210,7 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -223,7 +223,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: "{{ meta.name }}-test-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-test-{{ trunc_uuid }}" namespace: '{{ operator_namespace }}' ownerReferences: "{{ owner }}" data: @@ -236,7 +236,7 @@ kind: Pod namespace: '{{ operator_namespace }}' label_selectors: - - app = {{ meta.name }}-hyperfoil-{{ trunc_uuid }} + - app = {{ ansible_operator_meta.name }}-hyperfoil-{{ trunc_uuid }} - role = controller register: hf_pod @@ -248,7 +248,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Running" @@ -258,7 +258,7 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -269,7 +269,7 @@ api_version: v1 kind: Job namespace: '{{ operator_namespace }}' - name: "{{ meta.name }}-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-{{ trunc_uuid }}" register: job_state - name: Complete benchmark @@ -277,7 +277,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Complete" @@ -288,7 +288,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Failed" @@ -298,7 +298,7 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -311,7 +311,7 @@ apiVersion: hyperfoil.io/v1alpha2 kind: Hyperfoil metadata: - name: '{{ meta.name }}-hyperfoil-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-hyperfoil-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' - name: Delete config map @@ -321,7 +321,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-test-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-test-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' - name: Delete workload namespace diff --git a/roles/servicemesh/templates/job.yaml.j2 b/roles/servicemesh/templates/job.yaml.j2 index e0fff579f..2c08feeb0 100644 --- a/roles/servicemesh/templates/job.yaml.j2 +++ b/roles/servicemesh/templates/job.yaml.j2 @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ meta.name }}-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-{{ trunc_uuid }}" namespace: '{{ operator_namespace }}' ownerReferences: "{{ owner }}" spec: @@ -9,7 +9,7 @@ spec: template: metadata: labels: - app: "{{ meta.name }}-{{ trunc_uuid }}" + app: "{{ ansible_operator_meta.name }}-{{ trunc_uuid }}" spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -31,4 +31,4 @@ spec: - name: files configMap: defaultMode: 493 - name: "{{ meta.name }}-test-{{ trunc_uuid }}" \ No newline at end of file + name: "{{ ansible_operator_meta.name }}-test-{{ trunc_uuid }}" \ No newline at end of file diff --git a/roles/servicemesh/templates/smcp.yaml.j2 b/roles/servicemesh/templates/smcp.yaml.j2 index 7babdc7b1..77892ed09 100644 --- a/roles/servicemesh/templates/smcp.yaml.j2 +++ b/roles/servicemesh/templates/smcp.yaml.j2 @@ -1,7 +1,7 @@ apiVersion: maistra.io/v1 kind: ServiceMeshControlPlane metadata: - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ controlplane_namespace }}" spec: istio: diff --git a/roles/smallfile/tasks/main.yml b/roles/smallfile/tasks/main.yml index 46b49efd3..d18148aa9 100644 --- a/roles/smallfile/tasks/main.yml +++ b/roles/smallfile/tasks/main.yml @@ -3,7 +3,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -11,7 +11,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Starting @@ -22,7 +22,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -120,7 +120,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Running @@ -142,7 +142,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/stressng/tasks/main.yaml b/roles/stressng/tasks/main.yaml index 060906c06..c11e878fa 100644 --- a/roles/stressng/tasks/main.yaml +++ b/roles/stressng/tasks/main.yaml @@ -3,14 +3,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Building @@ -21,7 +21,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -33,7 +33,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: jobfile: "{{ lookup ('template', 'jobfile.j2') }}" @@ -41,7 +41,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: ConfigMaps Created @@ -68,7 +68,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Benchmark running @@ -82,14 +82,14 @@ k8s_facts: kind: Job api_version: batch/v1 - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: "{{ operator_namespace }}" register: stressng_workload_pod - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Benchmark Complete @@ -104,7 +104,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Benchmark Complete diff --git a/roles/stressng/templates/stressng_workload.yml.j2 b/roles/stressng/templates/stressng_workload.yml.j2 index 040f2deb0..4686d3dbf 100644 --- a/roles/stressng/templates/stressng_workload.yml.j2 +++ b/roles/stressng/templates/stressng_workload.yml.j2 @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: "job" metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: parallelism: {{ workload_args.instances }} @@ -87,7 +87,7 @@ spec: volumes: - name: stressng-workload-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0660 restartPolicy: OnFailure {% include "metadata.yml.j2" %} diff --git a/roles/stressng/templates/stressng_workload_vm.yml.j2 b/roles/stressng/templates/stressng_workload_vm.yml.j2 index e00394749..1fb6bb4ac 100644 --- a/roles/stressng/templates/stressng_workload_vm.yml.j2 +++ b/roles/stressng/templates/stressng_workload_vm.yml.j2 @@ -2,7 +2,7 @@ apiVersion: kubevirt.io/v1alpha3 kind: VirtualMachineInstance metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: domain: @@ -73,6 +73,6 @@ spec: ; name: cloudinitdisk - configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" name: stressng-workload-volume status: {} diff --git a/roles/sysbench/tasks/main.yml b/roles/sysbench/tasks/main.yml index 9edd25fbb..0d270ee89 100644 --- a/roles/sysbench/tasks/main.yml +++ b/roles/sysbench/tasks/main.yml @@ -4,14 +4,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Starting @@ -22,7 +22,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -53,7 +53,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Running @@ -75,7 +75,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/testpmd/tasks/main.yml b/roles/testpmd/tasks/main.yml index b7b58642d..f1d093c66 100644 --- a/roles/testpmd/tasks/main.yml +++ b/roles/testpmd/tasks/main.yml @@ -3,14 +3,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Starting @@ -21,7 +21,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -55,7 +55,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting TestPMD" @@ -77,7 +77,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting TRex" @@ -114,7 +114,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Waiting for TRex" @@ -135,7 +135,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Running @@ -157,7 +157,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/uperf/tasks/cleanup.yml b/roles/uperf/tasks/cleanup.yml index b17c8b9af..4e8714f3b 100644 --- a/roles/uperf/tasks/cleanup.yml +++ b/roles/uperf/tasks/cleanup.yml @@ -9,7 +9,7 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_jobs - name: Get Server Pods @@ -18,7 +18,7 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods - name: Server Job and Pod names - to clean @@ -74,7 +74,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/uperf/tasks/init.yml b/roles/uperf/tasks/init.yml index 7c9b7220b..39c65791a 100644 --- a/roles/uperf/tasks/init.yml +++ b/roles/uperf/tasks/init.yml @@ -10,7 +10,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: pod_hi_idx: "{{pod_hi_idx}}" diff --git a/roles/uperf/tasks/next_set.yml b/roles/uperf/tasks/next_set.yml index 3f2a5f4cf..3f3503d62 100644 --- a/roles/uperf/tasks/next_set.yml +++ b/roles/uperf/tasks/next_set.yml @@ -67,7 +67,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Running @@ -88,7 +88,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Clients Running diff --git a/roles/uperf/tasks/run_a_set.yml b/roles/uperf/tasks/run_a_set.yml index 46c08eb9d..997a0271a 100644 --- a/roles/uperf/tasks/run_a_set.yml +++ b/roles/uperf/tasks/run_a_set.yml @@ -15,7 +15,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Set Running diff --git a/roles/uperf/tasks/send_client_run_signal.yml b/roles/uperf/tasks/send_client_run_signal.yml index 52fd92a97..739b5ceef 100644 --- a/roles/uperf/tasks/send_client_run_signal.yml +++ b/roles/uperf/tasks/send_client_run_signal.yml @@ -9,7 +9,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Running" diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index c00f2e244..1f99e4764 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -4,14 +4,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Building @@ -22,7 +22,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -161,7 +161,7 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: serviceip when: workload_args.serviceip is defined and workload_args.serviceip diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml index eb83a4f25..2d1b30809 100644 --- a/roles/uperf/tasks/start_client.yml +++ b/roles/uperf/tasks/start_client.yml @@ -6,7 +6,7 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods - name: Generate uperf xml files @@ -45,7 +45,7 @@ api_version: kubevirt.io/v1alpha3 namespace: '{{ operator_namespace }}' label_selectors: - - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_vms @@ -65,7 +65,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Waiting for Clients diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml index 9c01c9b2a..8036c3769 100644 --- a/roles/uperf/tasks/start_server.yml +++ b/roles/uperf/tasks/start_server.yml @@ -30,14 +30,14 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods - name: Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Servers" @@ -58,14 +58,14 @@ api_version: kubevirt.io/v1alpha3 namespace: '{{ operator_namespace }}' label_selectors: - - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_vms - name: Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Servers" diff --git a/roles/uperf/tasks/wait_client_done.yml b/roles/uperf/tasks/wait_client_done.yml index 85036a28b..9c2f1e200 100644 --- a/roles/uperf/tasks/wait_client_done.yml +++ b/roles/uperf/tasks/wait_client_done.yml @@ -13,7 +13,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Cleanup @@ -30,7 +30,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Cleanup diff --git a/roles/uperf/tasks/wait_client_ready.yml b/roles/uperf/tasks/wait_client_ready.yml index c6455fea7..ad06014f0 100644 --- a/roles/uperf/tasks/wait_client_ready.yml +++ b/roles/uperf/tasks/wait_client_ready.yml @@ -16,7 +16,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Clients Running @@ -38,7 +38,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Clients Running diff --git a/roles/uperf/tasks/wait_server_ready.yml b/roles/uperf/tasks/wait_server_ready.yml index 9b5b707d5..dca5b4baf 100644 --- a/roles/uperf/tasks/wait_server_ready.yml +++ b/roles/uperf/tasks/wait_server_ready.yml @@ -8,14 +8,14 @@ api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods - name: Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Clients" @@ -32,14 +32,14 @@ api_version: kubevirt.io/v1alpha3 namespace: '{{ operator_namespace }}' label_selectors: - - type = {{ meta.name }}-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_vms - name: Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Clients" diff --git a/roles/uperf/tasks/wait_set_done.yml b/roles/uperf/tasks/wait_set_done.yml index 3f596d681..0db9c67ed 100644 --- a/roles/uperf/tasks/wait_set_done.yml +++ b/roles/uperf/tasks/wait_set_done.yml @@ -10,7 +10,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Run Next Set diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 53e76bd78..0dc36a0fe 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -16,7 +16,7 @@ items: metadata: labels: app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'') }}-{{ item }}-{{ trunc_uuid }} - type: {{ meta.name }}-bench-server-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} annotations: {% if workload_args.multus.enabled is sameas true %} k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} diff --git a/roles/uperf/templates/server_vm.yml.j2 b/roles/uperf/templates/server_vm.yml.j2 index 3c2d5444d..71c130fbe 100644 --- a/roles/uperf/templates/server_vm.yml.j2 +++ b/roles/uperf/templates/server_vm.yml.j2 @@ -6,7 +6,7 @@ metadata: namespace: '{{ operator_namespace }}' labels: app : uperf-bench-server-{{ item }}-{{ trunc_uuid }} - type : {{ meta.name }}-bench-server-{{ trunc_uuid }} + type : {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} spec: domain: cpu: diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 3c68308dd..89ae4b71a 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -11,7 +11,7 @@ items: namespace: '{{ operator_namespace }}' labels: app: uperf-bench-server-{{ worker_node_list[ node_idx_item ] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} - type: {{ meta.name }}-bench-server-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} annotations: node_idx: '{{ node_idx_item }}' pod_idx: '{{ item }}' diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 5f3cf61ff..b6e9ae475 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -19,7 +19,7 @@ items: labels: app: uperf-bench-client-{{ trunc_uuid }} clientfor: {{ item.metadata.labels.app }} - type: {{ meta.name }}-bench-client-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} {% if workload_args.multus.enabled is sameas true %} annotations: k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} diff --git a/roles/uperf/templates/workload_vm.yml.j2 b/roles/uperf/templates/workload_vm.yml.j2 index e04f216c5..bc149e02e 100644 --- a/roles/uperf/templates/workload_vm.yml.j2 +++ b/roles/uperf/templates/workload_vm.yml.j2 @@ -6,7 +6,7 @@ metadata: namespace: '{{ operator_namespace }}' labels: app: uperf-bench-client-{{ trunc_uuid }} - type: {{ meta.name }}-bench-client-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} spec: domain: cpu: diff --git a/roles/vegeta/tasks/main.yml b/roles/vegeta/tasks/main.yml index 2505e6f89..f9b01d614 100644 --- a/roles/vegeta/tasks/main.yml +++ b/roles/vegeta/tasks/main.yml @@ -4,14 +4,14 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Building @@ -22,7 +22,7 @@ k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state @@ -49,7 +49,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Running @@ -72,7 +72,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/ycsb/tasks/main.yml b/roles/ycsb/tasks/main.yml index a5be7b1a3..6bc2882cd 100644 --- a/roles/ycsb/tasks/main.yml +++ b/roles/ycsb/tasks/main.yml @@ -3,14 +3,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: resource_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Building @@ -21,7 +21,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: resource_state @@ -44,7 +44,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Running Load" @@ -54,7 +54,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting First Workload" @@ -82,7 +82,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting First Workload" @@ -92,7 +92,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Complete" @@ -103,16 +103,16 @@ - block: - name: Add the workload list length to redis - command: "redis-cli set {{ meta.name }}-{{ uuid }}-ycsb {{ workload_args.workloads|length }}" + command: "redis-cli set {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb {{ workload_args.workloads|length }}" - name: Add the first workload index to redis - command: "redis-cli set {{ meta.name }}-{{ uuid }}-ycsb-current 0" + command: "redis-cli set {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current 0" - name: Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Workload" @@ -122,7 +122,7 @@ - block: - name: Get current workload index - command: "redis-cli get {{ meta.name }}-{{ uuid }}-ycsb-current" + command: "redis-cli get {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current" register: wrkload - name: set ycsb_workload variable @@ -141,7 +141,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Running Workload" @@ -151,7 +151,7 @@ - block: - name: Get current workload from redis - command: "redis-cli get {{ meta.name }}-{{ uuid }}-ycsb-current" + command: "redis-cli get {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current" register: wrkload - name: set ycsb_workload variable @@ -172,7 +172,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Workload Complete" @@ -183,11 +183,11 @@ - block: - name: Get current workload from redis - command: "redis-cli get {{ meta.name }}-{{ uuid }}-ycsb-current" + command: "redis-cli get {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current" register: current_workload - name: Get list length of workloads from redis - command: "redis-cli get {{ meta.name }}-{{ uuid }}-ycsb" + command: "redis-cli get {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb" register: workload_list - name: Iterate index @@ -195,13 +195,13 @@ new_workload_index: "{{ current_workload.stdout|int + 1 }}" - name: Update current workload item in redis - command: "redis-cli set {{ meta.name }}-{{ uuid }}-ycsb-current {{ new_workload_index }}" + command: "redis-cli set {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current {{ new_workload_index }}" - name: Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Workload" @@ -211,7 +211,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Complete" diff --git a/roles/ycsb/templates/ycsb_load.yaml b/roles/ycsb/templates/ycsb_load.yaml index 54093a481..556510d6a 100644 --- a/roles/ycsb/templates/ycsb_load.yaml +++ b/roles/ycsb/templates/ycsb_load.yaml @@ -7,7 +7,7 @@ metadata: spec: template: metadata: - name: '{{ meta.name }}-ycsb-data-load' + name: '{{ ansible_operator_meta.name }}-ycsb-data-load' namespace: '{{ operator_namespace }}' labels: name: 'ycsb-load-{{ trunc_uuid }}' diff --git a/roles/ycsb/templates/ycsb_run.yaml b/roles/ycsb/templates/ycsb_run.yaml index 7f094e44c..c7523cf18 100644 --- a/roles/ycsb/templates/ycsb_run.yaml +++ b/roles/ycsb/templates/ycsb_run.yaml @@ -7,7 +7,7 @@ metadata: spec: template: metadata: - name: '{{ meta.name }}-ycsb-bench' + name: '{{ ansible_operator_meta.name }}-ycsb-bench' namespace: '{{ operator_namespace }}' labels: name: 'ycsb-run-{{ trunc_uuid }}' From e6f09352a58cad871687827fe2afaac6fc409350 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 31 Mar 2021 11:13:29 +0200 Subject: [PATCH 059/249] Update references in docs Signed-off-by: Raul Sevilla --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4490a9814..d32703a51 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -129,7 +129,7 @@ case of failure or when disabled. This ensures no interference with subsequent w ### The operator container image Any changes to the [roles](roles/) tree or to the [playbook](playbook.yml) file will necessitate a new build of the operator container image. -The container is built using the [Operator SDK](https://github.com/operator-framework/operator-sdk) (v.0.10.0 and before) and pushed to a public repository. +The container is built using the [Operator SDK](https://github.com/operator-framework/operator-sdk) (v0.1.5) and pushed to a public repository. The public repository could be [quay](https://quay.io) in which case you'll need to: ```bash From 923e156ac2c032298e423867b7def64b0da6263d Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 31 Mar 2021 11:31:17 +0200 Subject: [PATCH 060/249] Don't dump ansible container logs Signed-off-by: Raul Sevilla --- tests/common.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/common.sh b/tests/common.sh index ef93c038a..ea7df463f 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -266,9 +266,7 @@ function error { echo "Error caught. Dumping logs before exiting" echo "Benchmark operator Logs" - kubectl -n my-ripsaw logs --tail=40 -l name=benchmark-operator -c benchmark-operator - echo "Ansible sidecar Logs" - kubectl -n my-ripsaw logs -l name=benchmark-operator -c ansible + kubectl -n my-ripsaw logs --tail=200 -l name=benchmark-operator -c benchmark-operator } function wait_for_backpack() { From 6b07bcc6604fd1d7faa34cef18b2968cd3c6ba25 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 31 Mar 2021 11:52:49 +0200 Subject: [PATCH 061/249] Remove underscores from stockpile vars Signed-off-by: Raul Sevilla --- docs/metadata.md | 8 ++++---- roles/backpack/templates/backpack.yml | 6 +++--- templates/metadata.yml.j2 | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/metadata.md b/docs/metadata.md index 108c6c0d7..9e2d2282b 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -56,11 +56,11 @@ the workload template with an init container section that looks like: {% if metadata.force is sameas true %} --force {% endif %} -{% if metadata.stockpile_tags|length > 0 %} - --tags={{ metadata.stockpile_tags|join(",") }} +{% if metadata.stockpileTags|length > 0 %} + --tags={{ metadata.stockpileTags|join(",") }} {% endif %} -{% if metadata.stockpile_skip_tags|length > 0 %} - --skip-tags={{ metadata.stockpile_skip_tags|join(",") }} +{% if metadata.stockpileSkipTags|length > 0 %} + --skip-tags={{ metadata.stockpileSkipTags|join(",") }} {% endif %} {% if metadata.ssl is sameas true %} --sslskipverify diff --git a/roles/backpack/templates/backpack.yml b/roles/backpack/templates/backpack.yml index 48382e738..8d4f74e6c 100755 --- a/roles/backpack/templates/backpack.yml +++ b/roles/backpack/templates/backpack.yml @@ -50,9 +50,9 @@ spec: {% if metadata.force is sameas true %} --force {% endif %} - --tags={{ metadata.stockpile_tags|default(["common", "k8s", "openshift"])|join(",") }} -{% if metadata.stockpile_skip_tags|length > 0 %} - --skip-tags={{ metadata.stockpile_skip_tags|join(",") }} + --tags={{ metadata.stockpileTags|default(["common", "k8s", "openshift"])|join(",") }} +{% if metadata.stockpileSkipTags|length > 0 %} + --skip-tags={{ metadata.stockpileSkipTags|join(",") }} {% endif %} {% if metadata.ssl is sameas true %} --sslskipverify True diff --git a/templates/metadata.yml.j2 b/templates/metadata.yml.j2 index 5a7fd1381..e4fc6a7c9 100644 --- a/templates/metadata.yml.j2 +++ b/templates/metadata.yml.j2 @@ -19,9 +19,9 @@ {% if metadata.force is sameas true %} --force {% endif %} - --tags={{ metadata.stockpile_tags|default(["common", "k8s", "openshift"])|join(",") }} -{% if metadata.stockpile_skip_tags|length > 0 %} - --skip-tags={{ metadata.stockpile_skip_tags|join(",") }} + --tags={{ metadata.stockpileTags|default(["common", "k8s", "openshift"])|join(",") }} +{% if metadata.stockpileSkipTags|length > 0 %} + --skip-tags={{ metadata.stockpileSkipTags|join(",") }} {% endif %} {% if metadata.ssl is sameas true %} --sslskipverify True From 9d158f3c933fba79f5595f7da6c0251a3e3746bc Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 31 Mar 2021 12:55:14 +0200 Subject: [PATCH 062/249] Replace operator-sdk by podman Signed-off-by: Raul Sevilla --- CONTRIBUTING.md | 4 +--- tests/common.sh | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d32703a51..fcd65f68f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -129,12 +129,10 @@ case of failure or when disabled. This ensures no interference with subsequent w ### The operator container image Any changes to the [roles](roles/) tree or to the [playbook](playbook.yml) file will necessitate a new build of the operator container image. -The container is built using the [Operator SDK](https://github.com/operator-framework/operator-sdk) (v0.1.5) and pushed to a public repository. +The benchmark-operator container can be built using [podman](https://podman.io/) and pushed to a public repository. The public repository could be [quay](https://quay.io) in which case you'll need to: ```bash -$ operator-sdk build quay.io//benchmark-operator:testing --image-builder podman -OR $ podman build -f build/Dockerfile -t quay.io//benchmark-operator:testing . $ podman push quay.io//benchmark-operator:testing diff --git a/tests/common.sh b/tests/common.sh index ea7df463f..cb9c92bb1 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -213,7 +213,7 @@ function cleanup_operator_resources { function update_operator_image { tag_name="${NODE_NAME:-master}" - if operator-sdk build $image_location/$image_account/benchmark-operator:$tag_name --image-builder podman; then + if podman build -f build/Dockerfile -t $image_location/$image_account/benchmark-operator:$tag_name .; then # In case we have issues uploading to quay we will retry a few times for i in {1..3}; do podman push $image_location/$image_account/benchmark-operator:$tag_name && break From c84908d07080a5a261b6fa6c06aba521733eb347 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 7 Apr 2021 13:07:14 +0200 Subject: [PATCH 063/249] Update meta to ansible_operator_meta Signed-off-by: Raul Sevilla --- roles/log_generator/tasks/main.yml | 12 ++++++------ roles/system-metrics/tasks/main.yml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/log_generator/tasks/main.yml b/roles/log_generator/tasks/main.yml index f9cdc66f5..e3f6e1c85 100644 --- a/roles/log_generator/tasks/main.yml +++ b/roles/log_generator/tasks/main.yml @@ -3,14 +3,14 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: benchmark_state - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Starting Log Generator Pods" @@ -21,7 +21,7 @@ k8s_facts: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: benchmark_state @@ -35,7 +35,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Building Pods" @@ -57,7 +57,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Log Generator Running" @@ -79,7 +79,7 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete diff --git a/roles/system-metrics/tasks/main.yml b/roles/system-metrics/tasks/main.yml index 9d07d3685..361499297 100644 --- a/roles/system-metrics/tasks/main.yml +++ b/roles/system-metrics/tasks/main.yml @@ -30,7 +30,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: system_metrics: Collecting @@ -49,7 +49,7 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: system_metrics: Collected From e4c405fcbbf1b0edbb7254039c2fdccf73e4e281 Mon Sep 17 00:00:00 2001 From: Asis Patra Date: Wed, 17 Mar 2021 16:48:42 +0530 Subject: [PATCH 064/249] Suppot for any options under spec and container --- roles/byowl/templates/workload.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/roles/byowl/templates/workload.yml b/roles/byowl/templates/workload.yml index e66dafebc..effc34d31 100755 --- a/roles/byowl/templates/workload.yml +++ b/roles/byowl/templates/workload.yml @@ -13,6 +13,11 @@ spec: spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.specoptions is defined %} +{% for label, value in workload_args.specoptions.items() %} + {{ label }}: {{ value | to_json }} +{% endfor %} {% endif %} containers: - name: byowl @@ -20,6 +25,11 @@ spec: args: ["{{ workload_args.commands }}"] image: "{{ workload_args.image }}" imagePullPolicy: Always +{% if workload_args.containeroptions is defined %} +{% for label, value in workload_args.containeroptions.items() %} + {{ label }}: {{ value | to_json }} +{% endfor %} +{% endif %} restartPolicy: OnFailure {% if workload_args.nodeselector is defined %} nodeSelector: From 0fa5f20d885d25b375258ce032eba9fcdcfd3529 Mon Sep 17 00:00:00 2001 From: Asis Patra Date: Thu, 25 Mar 2021 12:25:55 +0530 Subject: [PATCH 065/249] BYOWL: Generic section under spec and containers --- docs/byowl.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/byowl.md b/docs/byowl.md index a9c31d0d8..5bc775e6c 100644 --- a/docs/byowl.md +++ b/docs/byowl.md @@ -44,3 +44,11 @@ spec: This will launch the uperf container, and simply print the messages above into the log of the container. + +### A generic support: +You can add any section under `specoptions` and `containeroptions` in `byowl` CRD, which will go under POD's `spec` and `containers` respectively as below: + + +![BYOWL_DOC](https://user-images.githubusercontent.com/4022122/112431010-f31de200-8d64-11eb-9179-e6ae7eb0e2cd.png) + + From 969026dec3c82e728c06fa9618a07c952e4b58ee Mon Sep 17 00:00:00 2001 From: prubenda Date: Thu, 18 Mar 2021 16:03:48 -0400 Subject: [PATCH 066/249] Adding concurrent builds to benchmark operator --- docs/kube-burner.md | 7 ++ .../ripsaw_v1alpha1_concurrent_builds_cr.yaml | 77 +++++++++++++++++++ .../crds/ripsaw_v1alpha1_kube-burner_cr.yaml | 14 +++- .../files/buildconfig_triggers.yml | 39 ++++++++++ roles/kube-burner/tasks/main.yml | 17 ++++ .../templates/concurrent-builds.yml.j2 | 51 ++++++++++++ tests/test_crs/valid_kube-burner.yaml | 8 ++ tests/test_kubeburner.sh | 1 + 8 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 resources/crds/ripsaw_v1alpha1_concurrent_builds_cr.yaml create mode 100644 roles/kube-burner/files/buildconfig_triggers.yml create mode 100644 roles/kube-burner/templates/concurrent-builds.yml.j2 diff --git a/docs/kube-burner.md b/docs/kube-burner.md index 3fd8acc35..a2a0ad604 100644 --- a/docs/kube-burner.md +++ b/docs/kube-burner.md @@ -63,6 +63,13 @@ Each iteration of this workload creates the following objects: Each iteration of this workload creates the following object: - 1 pod. (sleep) +- **concurrent-builds**: Creates a buildconfig, imagestream and corresponding build for a set application. **This will create as many namespaces with these objects as the configured job_iterations**. +See https://github.com/cloud-bulldozer/e2e-benchmarking/tree/master/workloads/kube-burner/builds for example parameters for each application +Each iteration of this workload creates the following object: + - 1 imagestream (dependent on application type set) + - 1 buildconfig (also dependent on application type set) + - 1 build created from buildconfig + The workload type is specified by the parameter `workload` from the `args` object of the configuration. Each workload supports several configuration parameters, detailed in the [configuration section](#configuration) ## Configuration diff --git a/resources/crds/ripsaw_v1alpha1_concurrent_builds_cr.yaml b/resources/crds/ripsaw_v1alpha1_concurrent_builds_cr.yaml new file mode 100644 index 000000000..d45129462 --- /dev/null +++ b/resources/crds/ripsaw_v1alpha1_concurrent_builds_cr.yaml @@ -0,0 +1,77 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-concurrent-builds-example + namespace: my-ripsaw +spec: + # Metadata information + elasticsearch: + # Elastic search instance with full URL format. https://elastic.apps.org:9200 + url: http://my.elasticsearch.server:80 + prometheus: + # Elastic search instance with full URL format. https://elastic.apps.org:9200 + es_url: http://my.elasticsearch.server:80 + # Prometheus bearer token + prom_token: PROMETHEUS_BEARER_TOKEN + # Prometheus URL with full URL format. https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + # Workload name + workload: concurrent-builds + # Kube-burner Job timeout + job_timeout: 7200 + # ES index name + default_index: ripsaw-kube-burner + # Number of job iterations + job_iterations: 100 + # Pin kube-burner to a node using the value of the label kubernetes.io/hostname + pin_server: {"node-role.kubernetes.io/worker": ""} + # Wait for pods to be runnig before finishing kube-burner workload + wait_when_finished: true + # Wait for all pods to be running before moving forward to the next job iteration + pod_wait: false + # Use a custom kube-burner image + image: quay.io/cloud-bulldozer/kube-burner:latest + # Queries per second + qps: 25 + # Max number of burst queries to perform + burst: 25 + # Log level. Allowed, info and debug + log_level: info + # Delete old namespaces for the selected workload before starting benchmark + cleanup: true + # Verify object creation + verify_objects: true + # Exit w/o indexing if a verify error happened + error_on_verify: true + # Prometheus step size + step: 30s + # kube-burner metrics profile + metrics_profile: metrics.yaml + # Remote configuration file + # remote_config: http://yourdomain/kube-burner.yml + # Remote metrics profile + # remote_metrics_profile: http://yourdomain/metrics-profile.yml + # kube-burner pod tolerations + # Application to build + app: django + source_strat_env: PIP_INDEX_URL + source_strat_from: python + source_strat_from_version: latest + # Script to run after build + post_commit_script: "./manage.py test" + # Build Image name + build_image_stream: django-psql-example + # Git url for application + git_url: https://github.com/sclorg/django-ex.git + build_image: image-registry.openshift-image-registry.svc:5000/svt-django/django-psql-example + tolerations: + - key: role + value: workload + effect: NoSchedule + # Pod nodeSelector + node_selector: + key: node-role.kubernetes.io/worker + value: diff --git a/resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml b/resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml index 6e47d1ed1..66983f8e9 100644 --- a/resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml +++ b/resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml @@ -55,6 +55,18 @@ spec: # Remote metrics profile # remote_metrics_profile: http://yourdomain/metrics-profile.yml # kube-burner pod tolerations + # Application to build + #app: django + #source_strat_env: PIP_INDEX_URL + #source_strat_from: python + #source_strat_from_version: latest + # Script to run after build + #post_commit_script: "./manage.py test" + # Build Image name + #build_image_stream: django-psql-example + # Git url for application + #git_url: https://github.com/sclorg/django-ex.git + #build_image: image-registry.openshift-image-registry.svc:5000/svt-django/django-psql-example tolerations: - key: role value: workload @@ -62,4 +74,4 @@ spec: # Pod nodeSelector node_selector: key: node-role.kubernetes.io/worker - value: + value: \ No newline at end of file diff --git a/roles/kube-burner/files/buildconfig_triggers.yml b/roles/kube-burner/files/buildconfig_triggers.yml new file mode 100644 index 000000000..44c836469 --- /dev/null +++ b/roles/kube-burner/files/buildconfig_triggers.yml @@ -0,0 +1,39 @@ +--- +kind: BuildConfig +apiVersion: build.openshift.io/v1 +metadata: + name: {{.JobName}}-{{.Replica}} + annotations: + template.alpha.openshift.io/wait-for-ready: 'true' +spec: + nodeSelector: + {{.nodeSelectorKey}}: {{.nodeSelectorValue}} + triggers: + - type: GitHub + github: + secret: {{.JobName}}-{{.Replica}} + - type: ImageChange + - type: ConfigChange + source: + git: + uri: {{.gitUri}} + type: Git + strategy: + type: Source + sourceStrategy: +{{ if index . "sourceStratEnv" }} + env: + - name: {{.sourceStratEnv}} +{{ end }} + from: + kind: ImageStreamTag + name: {{.fromSource}}:{{.fromSourceVersion}} + namespace: 'openshift' +{{ if index . "postCommitScript" }} + postCommit: + script: {{.postCommitScript}} +{{ end }} + output: + to: + kind: ImageStreamTag + name: {{.imageStream}}-{{.Replica}}:latest diff --git a/roles/kube-burner/tasks/main.yml b/roles/kube-burner/tasks/main.yml index fac7173c3..4066c6ab9 100644 --- a/roles/kube-burner/tasks/main.yml +++ b/roles/kube-burner/tasks/main.yml @@ -134,8 +134,25 @@ metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" simple-deployment.yml: "{{ lookup('file', 'simple-deployment.yml')}}" service.yml: "{{ lookup('file', 'service.yml')}}" + when: workload_args.workload == "max-services" + - name: Create concurrent builds + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'concurrent-builds.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" + buildconfig_triggers.yml: "{{ lookup('file', 'buildconfig_triggers.yml')}}" + configmap.yml: "{{ lookup('file', 'configmap.yml')}}" + imagestream.yml: "{{ lookup('file', 'imagestream.yml')}}" + when: workload_args.workload == "concurrent-builds" + - name: Launching kube-burner job k8s: definition: "{{ lookup('template', 'kube-burner.yml.j2') | from_yaml }}" diff --git a/roles/kube-burner/templates/concurrent-builds.yml.j2 b/roles/kube-burner/templates/concurrent-builds.yml.j2 new file mode 100644 index 000000000..09eb564c2 --- /dev/null +++ b/roles/kube-burner/templates/concurrent-builds.yml.j2 @@ -0,0 +1,51 @@ +--- +global: + writeToFile: false +{% if prometheus is defined and prometheus.prom_url is defined and prometheus.es_url != "" %} + indexerConfig: + enabled: true + esServers: ["{{ prometheus.es_url }}"] + insecureSkipVerify: true + defaultIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} + type: elastic + measurements: + - name: podLatency + esIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} +{% endif %} + +jobs: + - name: concurrent-builds + jobIterations: {{ workload_args.job_iterations }} + qps: {{ workload_args.qps|default(5) }} + burst: {{ workload_args.burst|default(10) }} + namespacedIterations: true + namespace: svt-{{ uuid }} + cleanup: {{ workload_args.cleanup|default(true) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + podWait: {{ workload_args.pod_wait|default(false) }} +{% if wait_for is defined %} + waitFor: {{ wait_for|default(["Build"]) }} +{% endif %} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + objects: + - objectTemplate: imagestream.yml + replicas: 1 + inputVars: + prefix: {{ workload_args.build_image_stream|default("") }} + image: {{ workload_args.build_image|default("") }} + - objectTemplate: buildconfig_triggers.yml + replicas: 1 + inputVars: + imageStream: {{ workload_args.build_image_stream|default("") }} + gitUri: {{ workload_args.git_url|default("") }} + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker")}} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" +{% if source_strat_env is defined and source_strat_env != "" %} + sourceStratEnv: {{ workload_args.source_strat_env|default("") }} +{% endif %} + fromSource: {{ workload_args.source_strat_from|default("") }} + fromSourceVersion: {{ workload_args.source_strat_from_version|default("latest") }} +{% if post_commit_script is defined and post_commit_script != "" %} + postCommitScript: {{ workload_args.post_commit_script|default("") }} +{% endif %} diff --git a/tests/test_crs/valid_kube-burner.yaml b/tests/test_crs/valid_kube-burner.yaml index d6d19172a..839dac7cc 100644 --- a/tests/test_crs/valid_kube-burner.yaml +++ b/tests/test_crs/valid_kube-burner.yaml @@ -32,6 +32,14 @@ spec: error_on_verify: true step: 30s metrics_profile: METRICS_PROFILE + app: django + source_strat_env: PIP_INDEX_URL + source_strat_from: python + source_strat_from_version: latest + post_commit_script: "./manage.py test" + build_image_stream: django-psql-example + git_url: https://github.com/sclorg/django-ex.git + build_image: image-registry.openshift-image-registry.svc:5000/svt-django/django-psql-example node_selector: key: node-role.kubernetes.io/worker value: diff --git a/tests/test_kubeburner.sh b/tests/test_kubeburner.sh index 536f36576..94f015591 100755 --- a/tests/test_kubeburner.sh +++ b/tests/test_kubeburner.sh @@ -55,3 +55,4 @@ functional_test_kubeburner node-density metrics.yaml functional_test_kubeburner node-density-heavy metrics.yaml functional_test_kubeburner max-namespaces metrics-aggregated.yaml functional_test_kubeburner max-services metrics-aggregated.yaml +functional_test_kubeburner concurrent-builds metrics-aggregated.yaml \ No newline at end of file From 36d02b4bd7ef702a5ba7b3dc3e309dc47807a9fe Mon Sep 17 00:00:00 2001 From: dry923 Date: Thu, 8 Apr 2021 15:34:57 -0400 Subject: [PATCH 067/249] Adding Image Pull workload --- README.md | 1 + docs/image_pull.md | 50 +++++++++++ roles/image_pull/tasks/main.yml | 89 +++++++++++++++++++ roles/image_pull/templates/image_pull.yml | 101 ++++++++++++++++++++++ tests/common.sh | 2 + tests/test_crs/valid_image_pull.yaml | 18 ++++ tests/test_image_pull.sh | 45 ++++++++++ 7 files changed, 306 insertions(+) create mode 100644 docs/image_pull.md create mode 100644 roles/image_pull/tasks/main.yml create mode 100644 roles/image_pull/templates/image_pull.yml create mode 100644 tests/test_crs/valid_image_pull.yaml create mode 100755 tests/test_image_pull.sh diff --git a/README.md b/README.md index 2790e28de..37b158ccd 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ helm delete benchmark-operator -n my-ripsaw --purge | [testpmd](docs/testpmd.md) | TestPMD DPDK App | No | Used | Not Supported | Preview | No | | [Flent](docs/flent.md) | Network Performance | Yes | Used, default : 3second | Not Supported | Not Supported | Yes | | [Log-Generator](docs/log_generator.md) | Log Throughput to Backend | Yes | Used, default : 3second | Not Supported | Yes | Yes | +| [Image-Pull](docs/image_pull.md) | Time to Pull Image from Container Repo | Yes | Used, default : 3second | Not Supported | Yes | Yes | ### Reconciliation diff --git a/docs/image_pull.md b/docs/image_pull.md new file mode 100644 index 000000000..12b4a7908 --- /dev/null +++ b/docs/image_pull.md @@ -0,0 +1,50 @@ +# Image Pull + +## What does it do? + +The Image Pull workload will take a list of images that exist in a container repository (e.x. Quay) +and copy them to the containers local working directory via [skopeo](https://github.com/containers/skopeo). +It will then display and/or index relevant data regarding the amount of time it took as well as any retries +or failures that occurred. + +Additioanlly, it can be configured to run multiple pods at the same time, each reporting its own relevant data. +This allows the user to test concurrency of image pulls on a container image. + +## Variables + +### Required variables: + +`image_list` a list of images in the format [image_transport]://[image_location] + +### Optional variables: + +`pod_count` total number of concurrent pods/tests to run (default: 1) + +`timeout` how long, in seconds, to wait for the image to copy (default: 600) + +`retries` how many times to retry failed copies (default: 0) + +### Example CR + +Your resource file may look like this when running 10 concurrent pods with 1 retry: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: image-pull + namespace: my-ripsaw +spec: + elasticsearch: + url: "http://es-instance.com:9200" + index_name: "image-pull" + workload: + name: image_pull + args: + pod_count: 2 + timeout: 600 + retries: 1 + image_list: + - docker://quay.io/cloud-bulldozer/backpack + - docker://quay.io/cloud-bulldozer/fio +``` diff --git a/roles/image_pull/tasks/main.yml b/roles/image_pull/tasks/main.yml new file mode 100644 index 000000000..c34eb4bcf --- /dev/null +++ b/roles/image_pull/tasks/main.yml @@ -0,0 +1,89 @@ +--- +- name: Get benchmark state + k8s_facts: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + register: benchmark_state + +- operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Image Pull Pods" + complete: false + when: benchmark_state.resources[0].status.state is not defined + +- name: Get benchmark state + k8s_facts: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + register: benchmark_state + +- block: + + - name: Image pull pods + k8s: + state: present + definition: "{{ lookup('template', 'image_pull.yml') }}" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Building Pods" + + when: benchmark_state.resources[0].status.state == "Starting Image Pull Pods" + +- block: + + - name: Get server pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = image-pull-{{ trunc_uuid }} + register: image_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Image Pull Pods Running" + when: "workload_args.pod_count|default('1')|int == image_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + + when: benchmark_state.resources[0].status.state == "Building Pods" + +- block: + + - name: Get server pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = image-pull-{{ trunc_uuid }} + register: image_pods + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + when: "workload_args.pod_count|default('1')|int == (image_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + + when: benchmark_state.resources[0].status.state == "Image Pull Pods Running" diff --git a/roles/image_pull/templates/image_pull.yml b/roles/image_pull/templates/image_pull.yml new file mode 100644 index 000000000..5299761e0 --- /dev/null +++ b/roles/image_pull/templates/image_pull.yml @@ -0,0 +1,101 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: 'image-pull-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' +spec: + parallelism: {{ workload_args.pod_count | default(1) | int }} + template: + metadata: + labels: + app: image-pull-{{ trunc_uuid }} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.tolerations is defined %} + tolerations: + - key: {{ workload_args.tolerations.key }} + value: {{ workload_args.tolerations.value }} + effect: {{ workload_args.tolerations.effect }} +{% endif %} +{% if workload_args.label is defined %} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ workload_args.label.key }} + operator: In + values: + - {{ workload_args.label.value }} +{% endif %} + containers: + - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/image_pull:latest') }} + name: pull-image + env: + - name: my_node_name + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: my_pod_name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: uuid + value: "{{ uuid }}" + - name: test_user + value: "{{ test_user | default("ripsaw") }}" + - name: clustername + value: "{{ clustername }}" +{% if elasticsearch is defined %} + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("image-pull") }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(true) }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" +{% endif %} +{% if prometheus is defined %} + - name: prom_es + value: "{{ prometheus.es_url }}" + - name: prom_parallel + value: "{{ prometheus.es_parallel | default(false) }}" + - name: prom_token + value: "{{ prometheus.prom_token | default() }}" + - name: prom_url + value: "{{ prometheus.prom_url | default() }}" +{% endif %} + command: ["/bin/sh", "-c"] + args: + - > +{% for my_image in workload_args.image_list %} + echo "Testing image: {{ my_image }}"; + echo "Waiting for all pods to be Ready"; + redis-cli -h {{ bo.resources[0].status.podIP }} INCR "image-pull-{{ trunc_uuid }}" > /dev/null 2>&1; + pods=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "image-pull-{{ trunc_uuid }}"`; + while [[ $pods != {{ workload_args.pod_count | default(1) | string }} ]]; do + pods=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "image-pull-{{ trunc_uuid }}"`; + sleep .5; + done; + sleep 2; + redis-cli -h {{ bo.resources[0].status.podIP }} DECR "image-pull-{{ trunc_uuid }}" > /dev/null 2>&1; + date; + run_snafu + --tool image_pull +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + -u "{{ uuid }}" + --pod-name ${my_pod_name} + --timeout "{{ workload_args.timeout | default(600) }}" + --pod-count {{ workload_args.pod_count | default(1) | int }} + --retries {{ workload_args.retries | default(0) | int }} + --image "{{ my_image }}"; +{% endfor %} + imagePullPolicy: Always + restartPolicy: Never +{% include "metadata.yml.j2" %} diff --git a/tests/common.sh b/tests/common.sh index cb9c92bb1..57c4953db 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -29,6 +29,7 @@ function populate_test_list { if [[ $(echo ${item} | grep 'roles/kube-burner') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/flent') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/log_generator') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/image_pull') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi # Check for changes in cr files @@ -49,6 +50,7 @@ function populate_test_list { if [[ $(echo ${item} | grep 'valid_kube-burner*') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_flent*') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_log_generator*') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_image_pull*') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi # Check for changes in test scripts diff --git a/tests/test_crs/valid_image_pull.yaml b/tests/test_crs/valid_image_pull.yaml new file mode 100644 index 000000000..ffdc92ec7 --- /dev/null +++ b/tests/test_crs/valid_image_pull.yaml @@ -0,0 +1,18 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: image-pull + namespace: my-ripsaw +spec: + elasticsearch: + url: ES_SERVER + index_name: "image-pull" + workload: + name: image_pull + args: + pod_count: 2 + timeout: 100 + retries: 1 + image_list: + - docker://quay.io/cloud-bulldozer/backpack + - docker://quay.io/cloud-bulldozer/fio diff --git a/tests/test_image_pull.sh b/tests/test_image_pull.sh new file mode 100755 index 000000000..70f7259fd --- /dev/null +++ b/tests/test_image_pull.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up Image Pull Test" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_image_pull { + wait_clean + apply_operator + test_name=$1 + cr=$2 + + echo "Performing: ${test_name}" + kubectl apply -f ${cr} + long_uuid=$(get_uuid 20) + uuid=${long_uuid:0:8} + + pod_count "app=image-pull-$uuid" 2 300 + wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=image-pull-$uuid jobs --timeout=500s" "500s" + + index="image-pull-results" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + exit 1 + fi + kubectl delete -f ${cr} +} + +figlet $(basename $0) +functional_test_image_pull "Image Pull" tests/test_crs/valid_image_pull.yaml From a2fbc9522b95769d006efce50de67681ececcf57 Mon Sep 17 00:00:00 2001 From: Kedar Vijay Kulkarni Date: Mon, 10 May 2021 11:21:13 -0400 Subject: [PATCH 068/249] adding cerberus validate certs parameter Signed-off-by: Kedar Vijay Kulkarni --- resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml | 2 ++ roles/cerberus/tasks/main.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml b/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml index d2c6f1426..c9c0ba7dd 100644 --- a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml +++ b/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml @@ -123,6 +123,8 @@ spec: type: object cerberus_url: type: string + cerberus_validate_certs: + type: boolean cleanup: type: boolean test_user: diff --git a/roles/cerberus/tasks/main.yml b/roles/cerberus/tasks/main.yml index 9a53a2c41..852af1617 100644 --- a/roles/cerberus/tasks/main.yml +++ b/roles/cerberus/tasks/main.yml @@ -2,6 +2,7 @@ - name: Get status from Cerberus uri: url: "{{ cerberus_url }}" + validate_certs: "{{ cerberus_validate_certs | default('False') }}" return_content: yes register: result From af5fe685d2c016b01bcb5b0ea0541357bb0df0e8 Mon Sep 17 00:00:00 2001 From: Kedar Vijay Kulkarni Date: Tue, 11 May 2021 15:20:39 -0400 Subject: [PATCH 069/249] Remove magzine section from CONTRIBUTING.md Signed-off-by: Kedar Vijay Kulkarni --- CONTRIBUTING.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fcd65f68f..eeec3769d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,14 +37,6 @@ In the interactive rebase screen, set the first commit to `pick` and all others Push your rebased commits (you may need to force), then issue your PR. -## Container Images - -Custom container image definitions are maintained in [magazine](https://github.com/cloud-bulldozer/magazine). -We use quay for all storing all our custom container images, and if you're adding a new -workload and not sure of where to add/maintain the container image. We highly recommend, to -add the Dockerfile to magazine, as we've automation setup for updating images in Quay, when -a git push happens to magazine. - ## Add workload Adding new workloads are always welcome, but before you submit PR: From 019cba9dc84d66fe125c3c400e0dbfc245bec35d Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Sun, 9 May 2021 15:21:05 -0500 Subject: [PATCH 070/249] Add support for kafka as log backend for verification Signed-off-by: Sai Sindhur Malleni --- docs/log_generator.md | 38 ++++++++++++++++++- roles/log_generator/tasks/main.yml | 2 +- .../log_generator/templates/log_generator.yml | 6 ++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/docs/log_generator.md b/docs/log_generator.md index 329aff9df..f69d229a3 100644 --- a/docs/log_generator.md +++ b/docs/log_generator.md @@ -49,6 +49,12 @@ AWS CloudWatch: `aws_secret_key` the secret key to use to query cloudwatch +Kafka: + +`kafka_bootstrap_server` the connection details to kafka + +`kafka_topic` the topic where logs are stored + Your resource file may look like this when using an Elasticsearch Backend: ```yaml @@ -73,7 +79,7 @@ spec: timeout: 600 label: key: foo - value: " + value: "" ``` Your resource file may look like this when using an AWS CloudWatch Backend: @@ -102,5 +108,33 @@ spec: timeout: 800 label: key: bar - value: " + value: "" ``` + +Your resource file may look like this when using a Kafka Backend: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: my-ripsaw +spec: + elasticsearch: + url: "http://es-instance.com:9200" + index_name: log-generator + workload: + name: log_generator + args: + pod_count: 2 + size: 512 + messages_per_second: 10 + duration: 1 + kafka_bootstrap_server: "my-cluster-kafka-bootstrap.amq:9092" + kafka_topic: "topic-logging-app" + timeout: 600 + label: + key: foo + value: "" +``` + diff --git a/roles/log_generator/tasks/main.yml b/roles/log_generator/tasks/main.yml index e3f6e1c85..17eac1060 100644 --- a/roles/log_generator/tasks/main.yml +++ b/roles/log_generator/tasks/main.yml @@ -26,7 +26,7 @@ register: benchmark_state - block: - + - name: Create log generator pods k8s: state: present diff --git a/roles/log_generator/templates/log_generator.yml b/roles/log_generator/templates/log_generator.yml index 94137b441..885e27bdb 100644 --- a/roles/log_generator/templates/log_generator.yml +++ b/roles/log_generator/templates/log_generator.yml @@ -106,10 +106,14 @@ spec: --aws-region "{{ workload_args.aws_region }}" --aws-access-key "{{ workload_args.aws_access_key }}" --aws-secret-key "{{ workload_args.aws_secret_key }}" +{% endif %} +{% if workload_args.kafka_bootstrap_server is defined and workload_args.kafka_topic is defined %} + --kafka-bootstrap-server "{{ workload_args.kafka_bootstrap_server }}" + --kafka-topic "{{ workload_args.kafka_topic }}" {% endif %} --pod-name ${my_pod_name} --timeout "{{ workload_args.timeout | default(600) }}" --pod-count {{ workload_args.pod_count | default(1) | int }} - imagePullPolicy: Always + imagePullPolicy: Always restartPolicy: Never {% include "metadata.yml.j2" %} From 3c866a5b3ddfd2bf43d7f596bd32cd50dd998cb5 Mon Sep 17 00:00:00 2001 From: Kedar Vijay Kulkarni Date: Wed, 12 May 2021 08:29:13 -0400 Subject: [PATCH 071/249] Expand README Signed-off-by: Kedar Vijay Kulkarni --- docs/cerberus.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/cerberus.md b/docs/cerberus.md index 5bdc0a05a..4bf99e8aa 100644 --- a/docs/cerberus.md +++ b/docs/cerberus.md @@ -1,9 +1,10 @@ # Cerberus Integration How To: -* [What is it](#what-is-it) -* [How it Works](#how-it-works) -* [How to enable Cerberus](#how-to-enable-Cerberus) +- [Cerberus Integration](#cerberus-integration) +- [What is it](#what-is-it) +- [How it Works](#how-it-works) +- [How to enable Cerberus](#how-to-enable-cerberus) # What is it @@ -42,12 +43,13 @@ spec: elasticsearch: url: "http://foo.bar.com:9200" cerberus_url: "http://1.2.3.4:8080" + cerberus_validate_certs: true workload: name: byowl ... ``` -NOTE: The cerberus url MUST BE in the format http://[address]:[port] +NOTE: The cerberus url MUST BE in the format http://[address]:[port] . If the cerberus is not running with proper ssl certificates, set `cerberus_validate_certs` to `false`. Once Cerberus is enabled the connection status can be viewed by getting the benchmark status. If Cerberus is not enabled the connection status will simply be "not connected" From 307d514445e3dbff5d8df8791d5c2d3ff0d1114b Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Wed, 19 May 2021 18:16:42 -0500 Subject: [PATCH 072/249] Quiesce logging in pod for log generator workload This along with https://github.com/cloud-bulldozer/benchmark-wrapper/pull/273 helps suppress any unneeded logging in the pod, so that we can accurately and easily count the number of log emssages recevied in a backend like kafka merely by using the offsets. The plan is to deploy the log generator pods in a separate namesapce and forward those logs to a topic in kafka. That way we would be able to reliably count the messages received just by looking at kafka topic offset. Otherwise there would be other logs from the log generator pods as well as benchmark-operator pod that would make it hard to reliably count logs received just by kafka offset. Signed-off-by: Sai Sindhur Malleni --- docs/log_generator.md | 2 ++ roles/log_generator/templates/log_generator.yml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/log_generator.md b/docs/log_generator.md index f69d229a3..23a07d11c 100644 --- a/docs/log_generator.md +++ b/docs/log_generator.md @@ -26,6 +26,8 @@ This data will also be indexed if Elasticsearch information is provided. `timeout` how long, in seconds, after have been sent to allow the backend service to receive all the messages (default: 600) +`snafu_disable_logs` Disable all logging in the pod from the snafu logger, thereby only leaving the generated log messages on stdout (default: False) + ### Verification variables: To verify your messages have been received by the backend aggregator you must provide information for ONLY ONE of the supported diff --git a/roles/log_generator/templates/log_generator.yml b/roles/log_generator/templates/log_generator.yml index 885e27bdb..330041d08 100644 --- a/roles/log_generator/templates/log_generator.yml +++ b/roles/log_generator/templates/log_generator.yml @@ -49,6 +49,8 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" + - name: snafu_disable_logs + value: "{{ workload_args.snafu_disable_logs | default(false) }}" {% if elasticsearch is defined %} - name: es value: "{{ elasticsearch.url }}" @@ -73,7 +75,7 @@ spec: args: - > {% if workload_args.pod_count | default(1) | int > 1 %} - echo "Waiting for all pods to be Ready"; + if [[ "${snafu_disable_logs}" == "False" ]]; then echo "Waiting for all pods to be Ready"; fi; redis-cli -h {{ bo.resources[0].status.podIP }} INCR "log-generator-{{ trunc_uuid }}" > /dev/null 2>&1; pods=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "log-generator-{{ trunc_uuid }}"`; while [[ $pods != {{ workload_args.pod_count | int }} ]]; do From ebf0f1f78547c538b303c95ee938d841452eb65a Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy Date: Thu, 20 May 2021 17:55:50 -0400 Subject: [PATCH 073/249] removed line breaks for trex tasks only --- roles/testpmd/templates/trex.yml.j2 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/roles/testpmd/templates/trex.yml.j2 b/roles/testpmd/templates/trex.yml.j2 index d6ca79275..a04fd82f0 100644 --- a/roles/testpmd/templates/trex.yml.j2 +++ b/roles/testpmd/templates/trex.yml.j2 @@ -49,11 +49,7 @@ spec: - name: trex command: ["/bin/sh", "-c"] args: - - run_snafu --tool trex --uuid {{ trunc_uuid }} --user {{ test_user | default('snafu') }} \ -{% if workload_args.debug is defined and workload_args.debug %} - -v \ -{% endif %} - ; + - run_snafu --tool trex --uuid {{ trunc_uuid }} --user {{ test_user | default('snafu') }} {% if workload_args.debug is defined and workload_args.debug %} -v {% endif %}; image: "{{ workload_args.image_trex | default('quay.io/cloud-bulldozer/trex:latest') }}" imagePullPolicy: "{{ workload_args.image_pull_policy | default('Always') }}" securityContext: From 49da13e97826139ba16305aded4ad11b76069b59 Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy Date: Sun, 23 May 2021 14:18:46 -0400 Subject: [PATCH 074/249] mounting module path for mlnx --- roles/testpmd/templates/trex.yml.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/testpmd/templates/trex.yml.j2 b/roles/testpmd/templates/trex.yml.j2 index a04fd82f0..fa30a941d 100644 --- a/roles/testpmd/templates/trex.yml.j2 +++ b/roles/testpmd/templates/trex.yml.j2 @@ -77,6 +77,8 @@ spec: volumeMounts: - name: hugepage mountPath: /dev/hugepages + - name: modules + mountPath: /lib/modules env: - name: CLUSTERNAME value: "{{ clustername }}" @@ -132,6 +134,9 @@ spec: - name: hugepage emptyDir: medium: HugePages + - name: modules + hostPath: + path: /lib/modules restartPolicy: OnFailure serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} From 3b213835da096e7308e768014c087070eb9fce0d Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy Date: Mon, 24 May 2021 09:50:51 -0400 Subject: [PATCH 075/249] updated doc for mlnx sriov policy --- docs/testpmd.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/testpmd.md b/docs/testpmd.md index bc2e9874e..cbafff86b 100644 --- a/docs/testpmd.md +++ b/docs/testpmd.md @@ -13,6 +13,8 @@ This workload assumes a healthy cluster is runing with SR-IOV and Performance Ad These are used to create VFs and SRIOV networks, interface might vary. Creating `SriovNetworkNodePolicy` will reboot your worker nodes one by one. +Below node policy is for Intel XXV710 NICs, for a Mellanox NIC to work in Data Plane Development Kit (DPDK) mode, use the `netdevice` driver type and include additional spec `isRdma: true`. + ```yaml apiVersion: sriovnetwork.openshift.io/v1 kind: SriovNetworkNodePolicy From 4aa5de0d58042f328857f1e5040c02b4f5adc710 Mon Sep 17 00:00:00 2001 From: Ayesha Vijay Kumar <84931574+Ayesha279@users.noreply.github.com> Date: Thu, 3 Jun 2021 16:20:06 -0400 Subject: [PATCH 076/249] Update installation.md --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 5ad67650a..5c6e4887e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -51,7 +51,7 @@ Note: If you're on a vanilla k8s distribution, then you can also deploy Ripsaw t First we'll need to clone the operator: ```bash -# git clone https://github.com/cloud-bulldozer/ripsaw +# git clone https://github.com/cloud-bulldozer/benchmark-operator.git # cd ripsaw # export KUBECONFIG= # if not already done ``` From 265a3845db3b1864cd4d42bda5cbaa7afce1b7b9 Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Tue, 25 May 2021 12:01:25 -0500 Subject: [PATCH 077/249] Make sink verification optional for kafka Signed-off-by: Sai Sindhur Malleni --- docs/log_generator.md | 2 ++ roles/log_generator/templates/log_generator.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/docs/log_generator.md b/docs/log_generator.md index 23a07d11c..be6332964 100644 --- a/docs/log_generator.md +++ b/docs/log_generator.md @@ -57,6 +57,8 @@ Kafka: `kafka_topic` the topic where logs are stored +`kafka_check` if you want to verify that log messages made it to kafka sink (requires a high timeout) + Your resource file may look like this when using an Elasticsearch Backend: ```yaml diff --git a/roles/log_generator/templates/log_generator.yml b/roles/log_generator/templates/log_generator.yml index 330041d08..e622be8ac 100644 --- a/roles/log_generator/templates/log_generator.yml +++ b/roles/log_generator/templates/log_generator.yml @@ -112,6 +112,9 @@ spec: {% if workload_args.kafka_bootstrap_server is defined and workload_args.kafka_topic is defined %} --kafka-bootstrap-server "{{ workload_args.kafka_bootstrap_server }}" --kafka-topic "{{ workload_args.kafka_topic }}" +{% if workload_args.kafka_check is defined and workload_args.kafka_check %} + --kafka-check +{% endif %} {% endif %} --pod-name ${my_pod_name} --timeout "{{ workload_args.timeout | default(600) }}" From cfb2b0a846c37d3eacfa64b3b00438c1754a1634 Mon Sep 17 00:00:00 2001 From: Ben England Date: Tue, 8 Jun 2021 17:23:43 -0400 Subject: [PATCH 078/249] Auto osd cache drop (#570) only do ceph osd cache dropping if user requests it default to openshift for benchmark-operator add option to drop Ceph OSD cache to CR document Ceph OSD cache dropping user must start cache dropper and ceph toolbox pod test both OSD cache dropping and kernel cache dropping at same time only if openshift-storage namespace is defined --- docs/cache_dropping.md | 38 ++++- group_vars/all.yml | 6 + playbook.yml | 4 + .../crds/ripsaw_v1alpha1_ripsaw_crd.yaml | 2 + .../rook_ceph_drop_cache_pod.yaml | 22 ++- roles/ceph_osd_cache_drop/tasks/main.yml | 130 ++++++++++++++++++ roles/ceph_osd_cache_drop/vars/main.yml | 3 + roles/ceph_osd_cache_drop/wait_for_pod.py | 75 ++++++++++ roles/fio_distributed/templates/client.yaml | 10 +- roles/kernel_cache_drop/tasks/main.yml | 4 +- roles/smallfile/templates/workload_job.yml.j2 | 12 +- tests/test_crs/valid_fiod_ocs_cache_drop.yaml | 85 ++++++++++++ tests/test_fiod.sh | 14 +- 13 files changed, 392 insertions(+), 13 deletions(-) rename {resources => roles/ceph_osd_cache_drop}/rook_ceph_drop_cache_pod.yaml (58%) create mode 100644 roles/ceph_osd_cache_drop/tasks/main.yml create mode 100644 roles/ceph_osd_cache_drop/vars/main.yml create mode 100644 roles/ceph_osd_cache_drop/wait_for_pod.py create mode 100644 tests/test_crs/valid_fiod_ocs_cache_drop.yaml diff --git a/docs/cache_dropping.md b/docs/cache_dropping.md index 360ecf57e..9d2836c4c 100644 --- a/docs/cache_dropping.md +++ b/docs/cache_dropping.md @@ -23,12 +23,15 @@ There are different types of caching that occur in the system - (Ceph OCS) OSD caching (not yet supported fully) you can control which type of cache dropping -is done using these CR fields in the workload args section: +is done using one or both of these CR fields in the workload args section: ``` drop_cache_kernel: true +drop_cache_rook_ceph: true ``` +## how to drop kernel cache + For this to work, you must **label** the nodes that you want to drop kernel cache, for example: ``` @@ -53,6 +56,24 @@ Benchmarks supported for kernel cache dropping at present are: - fio - smallfile +## how to drop Ceph OSD cache + +for this to work with OpenShift Container Storage, you must start both the Ceph toolbox pod in OCS +and the cache dropper pod. You can do this with: + +``` +oc patch OCSInitialization ocsinit -n openshift-storage --type json --patch \ + '[{ "op": "replace", "path": "/spec/enableCephTools", "value": true }]' + +oc create -f roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml + +oc -n openshift-storage get pod | awk '/tool/||/drop/' +``` + +when you see both of these pods in the running state, then you can use benchmark operator. The reason that +you have to manually start these two pods running is that the benchmark-operator does not have authorization +to run them in the openshift-storage namespace and get access to the secrets needed to do this. + # implementation notes For benchmark developers... @@ -84,3 +105,18 @@ For example, in your workload.yml.j2 where it creates the environment variables - name: KCACHE_DROP_PORT_NUM value: "{{ kernel_cache_drop_svc_port }}" ``` + +similarly, for Ceph OSD cache dropping, you must add this to one of your workload pods' environment variables: +``` + +{% if ceph_osd_cache_drop_pod_ip is defined %} + - name: ceph_osd_cache_drop_pod_ip + value: "{{ ceph_osd_cache_drop_pod_ip }}" + - name: CEPH_CACHE_DROP_PORT_NUM + value: "{{ ceph_cache_drop_svc_port }}" +{% endif %} + +``` +The ansible playbook for benchmark-operator will look up the ceph_osd_cache_drop_pod_ip IP address and fill in this var, +all you have to do is pass it in. See the ceph_osd_cache_drop ansible role for details. + diff --git a/group_vars/all.yml b/group_vars/all.yml index 60cc68ab0..d14efd36a 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -1,4 +1,10 @@ --- operator_namespace: '{{ ansible_operator_meta.namespace }}' + +# for upstream kubernetes this might be rook-ceph +# FIXME: how do we automatically determine that we are in openshift? +rook_ceph_namespace: 'openshift-storage' + clustername: 'myk8scluster' kernel_cache_drop_svc_port: 9222 +ceph_cache_drop_svc_port: 9457 diff --git a/playbook.yml b/playbook.yml index 915451755..3236e2b92 100644 --- a/playbook.yml +++ b/playbook.yml @@ -4,6 +4,10 @@ gather_facts: no tasks: + - include_role: + name: "ceph_osd_cache_drop" + when: workload.args.drop_cache_rook_ceph is defined + - include_role: name: "kernel_cache_drop" when: workload.args.drop_cache_kernel is defined diff --git a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml b/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml index c9c0ba7dd..94a0244dc 100644 --- a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml +++ b/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml @@ -135,6 +135,8 @@ spec: type: string drop_cache_kernel: type: boolean + drop_cache_rook_ceph: + type: boolean status: type: object properties: diff --git a/resources/rook_ceph_drop_cache_pod.yaml b/roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml similarity index 58% rename from resources/rook_ceph_drop_cache_pod.yaml rename to roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml index a4ec8d5c1..b7ffaec7f 100644 --- a/resources/rook_ceph_drop_cache_pod.yaml +++ b/roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml @@ -2,21 +2,33 @@ apiVersion: v1 kind: Pod metadata: name: rook-ceph-osd-cache-drop - namespace: rook-ceph + # if you are using upstream rook-ceph, change the next line + #namespace: "{{ rook_ceph_namespace }}" + namespace: "openshift-storage" + labels: + app: rook-ceph-osd-cache-drop spec: containers: - name: rook-ceph-osd-cache-drop - image: quay.io/cloud-bulldozer/ceph-cache-dropper:latest + image: "quay.io/cloud-bulldozer/ceph-cache-dropper:latest" imagePullPolicy: Always command: [/bin/sh, -c] args: - - cd /opt/bohica/ceph-cache-dropper; python3 ./osd-cache-drop-websvc.py + - cd /opt/ceph_cache_drop; python3 ./osd-cache-drop-websvc.py env: - - name: ROOK_ADMIN_SECRET + - name: ROOK_CEPH_USERNAME valueFrom: secretKeyRef: - key: admin-secret name: rook-ceph-mon + key: ceph-username + - name: ROOK_CEPH_SECRET + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-secret + - name: ceph_cache_drop_port + #value: "{{ ceph_cache_drop_svc_port }}" + value: "9457" securityContext: privileged: true volumeMounts: diff --git a/roles/ceph_osd_cache_drop/tasks/main.yml b/roles/ceph_osd_cache_drop/tasks/main.yml new file mode 100644 index 000000000..aecda4722 --- /dev/null +++ b/roles/ceph_osd_cache_drop/tasks/main.yml @@ -0,0 +1,130 @@ +--- +- debug: + msg: "in ceph OSD cache dropper role" + +# First ensure that toolbox pod is running + +- name: check for existing rook-ceph cluster + k8s_info: + kind: Pod + namespace: "{{ rook_ceph_namespace }}" + label_selectors: + - app = rook-ceph-mgr + register: rook_ceph_mgr_exists + +- name: generate boolean for namespace existence + set_fact: + rook_ceph_cluster_exists: "{{ rook_ceph_mgr_exists.resources | length > 0 }}" + +- debug: + var: rook_ceph_cluster_exists + +- fail: + msg: "You are asking for Ceph cache with drop_cache_rook_ceph: true in CR, but there is no ceph cluster!" + when: not rook_ceph_cluster_exists + +#- name: get pod info of pre-existing ceph toolbox pod +# k8s_info: +# kind: Pod +# label_selectors: +# - app = rook-ceph-tools +# namespace: "{{ rook_ceph_namespace }}" +# register: ceph_toolbox_already_exists + +#- debug: +# var: ceph_toolbox_already_exists + +# FIXME: next commented-out task gets 403 unprivileged error +# workaround is to issue oc command (with admin privs) to do it +# oc patch OCSInitialization ocsinit -n openshift-storage --type json --patch +# '[{ "op": "replace", "path": "/spec/enableCephTools", "value": true }]' + +#- name: ensure Ceph OSD toolbox pod is started +# k8s: +# namespace: "{{ rook_ceph_namespace }}" +# name: ocsinit +# kind: OCSInitialization +# state: present +# apply: true +# resource_definition: +# spec: +# enableCephTools: true +# when: ceph_toolbox_already_exists.resources | length == 0 + +#- name: wait for toolbox pod to start +# shell: "python3 /opt/ansible/roles/ceph_osd_cache_drop/wait_for_pod.py 30 {{ rook_ceph_namespace }} ceph-toolbox" +# when: ceph_toolbox_already_exists.resources | length == 0 + +- name: get pod name of running ceph toolbox pod + k8s_info: + kind: Pod + label_selectors: + - app = rook-ceph-tools + namespace: "{{ rook_ceph_namespace }}" + register: ceph_toolbox + +#- debug: +# var: ceph_toolbox + +# this next var is referenced by the YAML that creates the cache dropper pod + +- name: put pod id into a var + set_fact: + rook_ceph_toolbox_pod: "{{ ceph_toolbox.resources[0].metadata.name }}" + +- debug: + var: rook_ceph_toolbox_pod + + +# now ensure that ceph cache dropper pod is started and we have its IP + +#- name: get pre-existing ceph cache dropper pod +# k8s_info: +# kind: Pod +# label_selectors: +# - app = rook-ceph-osd-cache-drop +# namespace: "{{ rook_ceph_namespace }}" +# register: drop_pod_already_exists + +#- debug: +# var: drop_pod_already_exists + +# FIXME: this gets 403 unprivileged error +# workaround is to issue oc create -f command (with admin privs) to do it +# you must substitute a real value for the jinja2 var in this template first + +#- name: start ceph OSD cache dropper +# k8s: +# definition: "{{ lookup('template', '/opt/ansible/roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml') | from_yaml }}" +# register: ceph_cache_dropper_start +# when: drop_pod_already_exists.resources | length == 0 +# +#- name: wait for cache dropper to start +# shell: "python3 /opt/ansible/roles/ceph_osd_cache_drop/wait_for_pod.py 30 {{ rook_ceph_namespace }} rook-ceph-osd-cache-drop" +# when: rook_ceph_cluster_exists and drop_pod_already_exists.resources | length == 0 + +- name: get cache dropper pod + k8s_info: + kind: Pod + label_selectors: + - app = rook-ceph-osd-cache-drop + namespace: "{{ rook_ceph_namespace }}" + register: ceph_osd_cache_drop_pod + +#- debug: +# var: ceph_osd_cache_drop_pod + +- name: put ip into a var + set_fact: + ceph_osd_cache_drop_pod_ip: "{{ ceph_osd_cache_drop_pod.resources[0].status.podIP }}" + +- debug: + var: ceph_osd_cache_drop_pod_ip + +- name: test IP + shell: "curl http://{{ ceph_osd_cache_drop_pod_ip }}:{{ ceph_cache_drop_svc_port }}/" + register: drop_pod_test + +#- debug: +# var: drop_pod_test + diff --git a/roles/ceph_osd_cache_drop/vars/main.yml b/roles/ceph_osd_cache_drop/vars/main.yml new file mode 100644 index 000000000..517af5d01 --- /dev/null +++ b/roles/ceph_osd_cache_drop/vars/main.yml @@ -0,0 +1,3 @@ +# when debugging set this var, for example: +#ceph_cache_drop_image: quay.io/bengland2/ceph-cache-dropper:debug + diff --git a/roles/ceph_osd_cache_drop/wait_for_pod.py b/roles/ceph_osd_cache_drop/wait_for_pod.py new file mode 100644 index 000000000..297631cd4 --- /dev/null +++ b/roles/ceph_osd_cache_drop/wait_for_pod.py @@ -0,0 +1,75 @@ +#!/usr/bin/python3 +# wait_for_pod.py - +# wait until toolbox pod and web service pod is in Running state for specified deploy +# see usage() for input params + +from kubernetes import client, config +from sys import argv, exit +from os import getenv +import time + +NOTOK = 1 +# may have to adjust this based on number of pods +poll_interval = 2.0 + +def usage(msg): + print('ERROR: %s' % msg) + print('usage: wait_for_pod.py timeout namespace pod-name-pattern') + exit(NOTOK) + + +# parse command line + +if len(argv) < 4: + usage('too few parameters') + +timeout_str = argv[1] +ns = argv[2] +pod_name_pattern = argv[3] + +# show 'em what we parsed + +print('timeout if pods not seen in %s sec' % timeout_str) +print('namespace: %s' % ns) +print('pod name pattern: %s' % pod_name_pattern) + +timeout = int(timeout_str) +if timeout <= poll_interval: + usage('timeout %d must be greater than poll interval %d' % + (timeout, poll_interval)) + +# wait for pods + +# cannot do this from inside cluster pod: config.load_kube_config() +if getenv('KUBECONFIG'): + config.load_kube_config() +else: + config.load_incluster_config() +v1 = client.CoreV1Api() + +print('waiting for pod...') +start_time = time.time() +matching_pods = 0 + +while True: + time.sleep(poll_interval) + now_time = time.time() + delta_time = now_time - start_time + if delta_time > timeout: + break + + ret = v1.list_pod_for_all_namespaces(watch=False) + matching_pods = 0 + for i in ret.items: + if i.metadata.namespace == ns and \ + i.metadata.generate_name.__contains__ (pod_name_pattern) and \ + i.status.phase == 'Running': + matching_pods += 1 + if matching_pods >= 1: + break + +if delta_time > timeout: + usage('timeout waiting for pods to reach running state') + +if matching_pods != 1: + usage('expected 1 pod, found %d pods' % matching_pods) diff --git a/roles/fio_distributed/templates/client.yaml b/roles/fio_distributed/templates/client.yaml index ecef56843..372cca38e 100644 --- a/roles/fio_distributed/templates/client.yaml +++ b/roles/fio_distributed/templates/client.yaml @@ -22,7 +22,7 @@ spec: - name: uuid value: "{{ uuid }}" - name: test_user - value: "{{ test_user | default("ripsaw") }}" + value: "{{ test_user | default('ripsaw') }}" - name: clustername value: "{{ clustername }}" - name: ceph_cache_drop_pod_ip @@ -31,11 +31,17 @@ spec: value: "{{ kcache_drop_pod_ips | default() }}" - name: KCACHE_DROP_PORT_NUM value: "{{ kernel_cache_drop_svc_port }}" +{% if ceph_osd_cache_drop_pod_ip is defined %} + - name: ceph_osd_cache_drop_pod_ip + value: "{{ ceph_osd_cache_drop_pod_ip }}" + - name: CEPH_CACHE_DROP_PORT_NUM + value: "{{ ceph_cache_drop_svc_port }}" +{% endif %} {% if elasticsearch is defined %} - name: es value: "{{ elasticsearch.url }}" - name: es_index - value: "{{ elasticsearch.index_name | default("ripsaw-fio") }}" + value: "{{ elasticsearch.index_name | default('ripsaw-fio') }}" - name: es_verify_cert value: "{{ elasticsearch.verify_cert | default(true) }}" - name: parallel diff --git a/roles/kernel_cache_drop/tasks/main.yml b/roles/kernel_cache_drop/tasks/main.yml index f8b486b2f..de1a3a94f 100644 --- a/roles/kernel_cache_drop/tasks/main.yml +++ b/roles/kernel_cache_drop/tasks/main.yml @@ -17,8 +17,8 @@ namespace: "{{ operator_namespace }}" register: kcache_drop_pod_list -- debug: - var: kcache_drop_pod_list +#- debug: +# var: kcache_drop_pod_list - name: put ip list into a var set_fact: diff --git a/roles/smallfile/templates/workload_job.yml.j2 b/roles/smallfile/templates/workload_job.yml.j2 index 96880328b..8d28b82ec 100644 --- a/roles/smallfile/templates/workload_job.yml.j2 +++ b/roles/smallfile/templates/workload_job.yml.j2 @@ -45,12 +45,20 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if kcache_drop_pod_ips is defined and item | int == 1 %} +{% if item | int == 1 %} +{% if kcache_drop_pod_ips is defined %} - name: kcache_drop_pod_ips - value: "{{ kcache_drop_pod_ips | default() }}" + value: "{{ kcache_drop_pod_ips }}" - name: KCACHE_DROP_PORT_NUM value: "{{ kernel_cache_drop_svc_port }}" {% endif %} +{% if ceph_osd_cache_drop_pod_ip is defined %} + - name: ceph_osd_cache_drop_pod_ip + value: "{{ ceph_osd_cache_drop_pod_ip }}" + - name: CEPH_CACHE_DROP_PORT_NUM + value: "{{ ceph_cache_drop_svc_port }}" +{% endif %} +{% endif %} {% if elasticsearch is defined %} - name: es value: "{{ elasticsearch.url }}" diff --git a/tests/test_crs/valid_fiod_ocs_cache_drop.yaml b/tests/test_crs/valid_fiod_ocs_cache_drop.yaml new file mode 100644 index 000000000..47f3d0fcc --- /dev/null +++ b/tests/test_crs/valid_fiod_ocs_cache_drop.yaml @@ -0,0 +1,85 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: example-benchmark + namespace: my-ripsaw +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-fio + metadata: + collection: true + cleanup: false + workload: + name: "fio_distributed" + args: + drop_cache_rook_ceph: true + drop_cache_kernel: true + samples: 2 + servers: 2 + jobs: + - randwrite + - randread + - randrw + bs: + - 4KiB + numjobs: + - 2 + iodepth: 1 + read_runtime: 10 + write_runtime: 10 + read_ramp_time: 1 + write_ramp_time: 1 + filesize: 10MiB + log_sample_rate: 2000 + storagesize: 16Mi + debug: false +####################################### +# EXPERT AREA - MODIFY WITH CAUTION # +####################################### + job_params: + - jobname_match: write + params: + - time_based=1 + - fsync_on_close=1 + - create_on_open=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: read + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: rw + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: readwrite + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randread + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randwrite + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: randrw + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} diff --git a/tests/test_fiod.sh b/tests/test_fiod.sh index cdb5d33e5..042a764dc 100755 --- a/tests/test_fiod.sh +++ b/tests/test_fiod.sh @@ -35,7 +35,7 @@ function functional_test_fio { fio_pod=$(get_pod "app=fiod-client-$uuid" 300) wait_for "kubectl wait --for=condition=Initialized pods/$fio_pod -n my-ripsaw --timeout=500s" "500s" $fio_pod wait_for "kubectl wait --for=condition=complete -l app=fiod-client-$uuid jobs -n my-ripsaw --timeout=700s" "700s" $fio_pod - + kubectl -n my-ripsaw logs $fio_pod > /tmp/$fio_pod.log indexes="ripsaw-fio-results ripsaw-fio-log ripsaw-fio-analyzed-result" if check_es "${long_uuid}" "${indexes}" then @@ -50,5 +50,17 @@ function functional_test_fio { figlet $(basename $0) kubectl label nodes -l node-role.kubernetes.io/worker= kernel-cache-dropper=yes --overwrite functional_test_fio "Fio distributed" tests/test_crs/valid_fiod.yaml +openshift_storage_present=$(oc get namespace | awk '/openshift-storage/' | wc -l) +if [ $openshift_storage_present -gt 0 ] ; then + oc patch OCSInitialization ocsinit -n openshift-storage --type json --patch \ + '[{ "op": "replace", "path": "/spec/enableCephTools", "value": true }]' + drop_cache_pods=$(oc -n openshift-storage get pod | awk '/drop/' | awk '/unning/' | wc -l) + if [ $drop_cache_pods -eq 0 ] ; then + oc create -f roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml + kubectl wait --for=condition=Initialized pods/rook-ceph-osd-cache-drop -n openshift-storage --timeout=100s + fi + sleep 5 + functional_test_fio "Fio cache drop" tests/test_crs/valid_fiod_ocs_cache_drop.yaml +fi functional_test_fio "Fio distributed - bsrange" tests/test_crs/valid_fiod_bsrange.yaml functional_test_fio "Fio hostpath distributed" tests/test_crs/valid_fiod_hostpath.yaml From 66f82ee406933d2b689f340f3c259d46a572dfcb Mon Sep 17 00:00:00 2001 From: dry923 Date: Mon, 7 Jun 2021 10:08:34 -0400 Subject: [PATCH 079/249] Shorten max-services to max-serv to avoid line length issue --- roles/kube-burner/templates/max-services.yml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/kube-burner/templates/max-services.yml.j2 b/roles/kube-burner/templates/max-services.yml.j2 index 92d29851d..c3841aea6 100644 --- a/roles/kube-burner/templates/max-services.yml.j2 +++ b/roles/kube-burner/templates/max-services.yml.j2 @@ -14,7 +14,7 @@ global: {% endif %} jobs: - - name: max-services + - name: max-serv jobIterations: {{ workload_args.job_iterations }} qps: {{ workload_args.qps|default(5) }} burst: {{ workload_args.burst|default(10) }} @@ -30,11 +30,11 @@ jobs: - objectTemplate: simple-deployment.yml replicas: 1 inputVars: - name: max-services + name: max-serv nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" - objectTemplate: service.yml replicas: 1 inputVars: - name: max-services + name: max-serv From d7c36cd33673abec61d8ff9fa9fc7c8731e2ea16 Mon Sep 17 00:00:00 2001 From: ebattat <73884315+ebattat@users.noreply.github.com> Date: Fri, 11 Jun 2021 17:46:47 +0300 Subject: [PATCH 080/249] Hammerdb: add VM support (#569) * add hammerdb vm support CNV-6501 and pod support for mariadb and postgres * add generic hammerdb cr * add hammerdb vm example * change hammerdb crds hirarchy according to database type * fixes after review * fix hammerdb mssql test * revert sql server namespace * revert transactions number * update transactions number to 500k * update transactions to 100000 * update transactions to 100000 * update transactions to 10000 for fast run * fix hammer workload name * add creator pod wait * add debug true * revert app label to hammerdb_workload * fix type name * temporary fix in common.sh * revert my common.sh changes * change db init to false * change db init to true * update changes to support operator-sdk version 1.5.0 * update changes to support operator-sdk version 1.5.0 * enlarge the timeout from 500 to 800 * increase timeout to 1000 * revert timout to 500s * add pin and resources support * add mssql 2019 image and creator pod * revert it back to legacy mssql test * add es custom fields support * fix image example name * add hammerdb vm support CNV-6501 and pod support for mariadb and postgres * add generic hammerdb cr * add hammerdb vm example * change hammerdb crds hirarchy according to database type * fixes after review * fix hammerdb mssql test * revert sql server namespace * revert transactions number * update transactions number to 500k * update transactions to 100000 * update transactions to 100000 * update transactions to 10000 for fast run * fix hammer workload name * add creator pod wait * add debug true * revert app label to hammerdb_workload * fix type name * temporary fix in common.sh * revert my common.sh changes * change db init to false * change db init to true * update changes to support operator-sdk version 1.5.0 * enlarge the timeout from 500 to 800 * increase timeout to 1000 * revert timout to 500s * add pin and resources support * add mssql 2019 image and creator pod * revert it back to legacy mssql test * add es custom fields support * fix image example name * update changes to support operator-sdk version 1.5.0 * add latest changes * update changes to support operator-sdk version 1.5.0 * fix operator-sdk version 1.5.0 * add os version * fixes after changes * add es_os_version * update changes to support operator-sdk version 1.5.0 * fix hammer doc - database per the CR file * fix hammedb doc * fix hammedb doc * add es_kind * add es_kind to cr and fix merge conflict * remove .idea * update changes to support operator-sdk version 1.5.0 * adding cerberus validate certs parameter Signed-off-by: Kedar Vijay Kulkarni * Remove magzine section from CONTRIBUTING.md Signed-off-by: Kedar Vijay Kulkarni * Add support for kafka as log backend for verification Signed-off-by: Sai Sindhur Malleni * Expand README Signed-off-by: Kedar Vijay Kulkarni * Quiesce logging in pod for log generator workload This along with https://github.com/cloud-bulldozer/benchmark-wrapper/pull/273 helps suppress any unneeded logging in the pod, so that we can accurately and easily count the number of log emssages recevied in a backend like kafka merely by using the offsets. The plan is to deploy the log generator pods in a separate namesapce and forward those logs to a topic in kafka. That way we would be able to reliably count the messages received just by looking at kafka topic offset. Otherwise there would be other logs from the log generator pods as well as benchmark-operator pod that would make it hard to reliably count logs received just by kafka offset. Signed-off-by: Sai Sindhur Malleni * removed line breaks for trex tasks only * mounting module path for mlnx * updated doc for mlnx sriov policy * Update installation.md * Make sink verification optional for kafka Signed-off-by: Sai Sindhur Malleni * Auto osd cache drop (#570) only do ceph osd cache dropping if user requests it default to openshift for benchmark-operator add option to drop Ceph OSD cache to CR document Ceph OSD cache dropping user must start cache dropper and ceph toolbox pod test both OSD cache dropping and kernel cache dropping at same time only if openshift-storage namespace is defined * replace preview by working in hammerdb doc * update changes to support operator-sdk version 1.5.0 * replace preview by working in hammerdb doc * remove stressng fixes Co-authored-by: Kedar Vijay Kulkarni Co-authored-by: Sai Sindhur Malleni Co-authored-by: Murali Krishnasamy Co-authored-by: Ayesha Vijay Kumar <84931574+Ayesha279@users.noreply.github.com> Co-authored-by: Ben England --- README.md | 2 +- docs/hammerdb.md | 49 +++- docs/stressng.md | 2 +- .../ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml | 78 ++++++ ...psaw_v1alpha1_hammerdb_mariadb_server.yaml | 86 ++++++ ..._v1alpha1_hammerdb_mariadb_server_pvc.yaml | 100 +++++++ .../ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml | 99 +++++++ .../ripsaw_v1alpha1_hammerdb_mssql_cr.yaml | 78 ++++++ ...ripsaw_v1alpha1_hammerdb_mssql_server.yaml | 60 ++++ ...aw_v1alpha1_hammerdb_mssql_server_pvc.yaml | 74 +++++ .../ripsaw_v1alpha1_hammerdb_mssql_vm.yaml | 99 +++++++ .../ripsaw_v1alpha1_hammerdb_postgres_cr.yaml | 78 ++++++ ...saw_v1alpha1_hammerdb_postgres_server.yaml | 81 ++++++ ...v1alpha1_hammerdb_postgres_server_pvc.yaml | 94 +++++++ .../ripsaw_v1alpha1_hammerdb_postgres_vm.yaml | 101 +++++++ .../ripsaw_v1alpha1_hammerdb_cr.yaml | 78 ++++++ .../ripsaw_v1alpha1_hammerdb_vm.yaml | 93 +++++++ .../crds/ripsaw_v1alpha1_hammerdb_cr.yaml | 37 --- .../ripsaw_v1alpha1_hammerdb_index_cr.yaml | 23 -- .../crds/ripsaw_v1alpha1_stressng_cr.yaml | 10 +- roles/hammerdb/tasks/main.yaml | 261 +++++++++++++----- ...b_mysql.tcl.j2 => createdb_mariadb.tcl.j2} | 2 +- roles/hammerdb/templates/createdb_pg.tcl.j2 | 3 - roles/hammerdb/templates/db_creation.yml | 15 + .../templates/db_creation_mariadb_vm.yml | 121 ++++++++ .../templates/db_creation_mssql_vm.yml | 122 ++++++++ .../templates/db_creation_postgres_vm.yml | 121 ++++++++ ...load.yml.j2 => db_mariadb_workload.yml.j2} | 33 ++- .../templates/db_mariadb_workload_vm.sh.j2 | 62 +++++ .../templates/db_mssql_workload.yml.j2 | 30 +- .../templates/db_mssql_workload_vm.sh.j2 | 66 +++++ .../templates/db_postgres_workload.yml.j2 | 32 ++- .../templates/db_postgres_workload_vm.sh.j2 | 56 ++++ ...ql.tcl.j2 => tpcc-workload-mariadb.tcl.j2} | 2 +- .../templates/stressng_workload.yml.j2 | 3 +- tests/mssql.yaml | 3 +- tests/test_crs/valid_hammerdb.yaml | 7 +- 37 files changed, 2096 insertions(+), 165 deletions(-) create mode 100644 resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml create mode 100644 resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server.yaml create mode 100644 resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml create mode 100644 resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml create mode 100644 resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml create mode 100644 resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml create mode 100644 resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml create mode 100644 resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml create mode 100644 resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml create mode 100644 resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml create mode 100644 resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml create mode 100644 resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml create mode 100644 resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml create mode 100644 resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml delete mode 100644 resources/crds/ripsaw_v1alpha1_hammerdb_cr.yaml delete mode 100644 resources/crds/ripsaw_v1alpha1_hammerdb_index_cr.yaml rename roles/hammerdb/templates/{createdb_mysql.tcl.j2 => createdb_mariadb.tcl.j2} (92%) create mode 100644 roles/hammerdb/templates/db_creation_mariadb_vm.yml create mode 100644 roles/hammerdb/templates/db_creation_mssql_vm.yml create mode 100644 roles/hammerdb/templates/db_creation_postgres_vm.yml rename roles/hammerdb/templates/{db_mysql_workload.yml.j2 => db_mariadb_workload.yml.j2} (75%) create mode 100644 roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 create mode 100644 roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 create mode 100644 roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 rename roles/hammerdb/templates/{tpcc-workload-mysql.tcl.j2 => tpcc-workload-mariadb.tcl.j2} (97%) diff --git a/README.md b/README.md index 37b158ccd..5b791ad8b 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ helm delete benchmark-operator -n my-ripsaw --purge | [Pgbench](docs/pgbench.md) | Postgres Performance | Yes | Used, default : 3second | Not Supported | Preview | Yes | | [Smallfile](docs/smallfile.md) | Storage IO Performance | Yes | Used, default : 3second | Not Supported | Preview | Yes | | [fs-drift](docs/fs-drift.md) | Storage IO Longevity | Yes | Not used | Not Supported | Preview | Yes | -| [hammerdb](docs/hammerdb.md) | Database Performance | Yes | Used, default : 3second | Not Supported | Preview | Yes | +| [hammerdb](docs/hammerdb.md) | Database Performance | Yes | Used, default : 3second | Working | Preview | Yes | | [Service Mesh](docs/servicemesh.md) | Microservices | No | Used, default : 3second | Not Supported | Preview | No | | [Vegeta](docs/vegeta.md) | HTTP Performance | Yes | Used, default : 3second | Not Supported | Preview | Yes | | [Scale Openshift](docs/scale_openshift.md) | Scale Openshift Cluster | Yes | Used, default : 3second | Not Supported | Preview | Yes | diff --git a/docs/hammerdb.md b/docs/hammerdb.md index 8af4140c5..d067edc74 100644 --- a/docs/hammerdb.md +++ b/docs/hammerdb.md @@ -5,9 +5,9 @@ ## Running hammerdb Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_hammerdb_cr.yaml) to your needs. +you can modify [cr.yaml](../resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml) to your needs. -The pgbench workload needs to be pointed at an existing MS-SQL databases via the CR file. +The hammerdb workload needs to be pointed at an existing [MSSQL](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml), [MARIADB](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml), [POSTGRES](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml) database via the CR file. An example CR might look like this @@ -57,3 +57,48 @@ Once done creating/editing the resource file, you can run it by: # kubectl apply -f ``` +## Running hammerdb in VMs through kubevirt/cnv [Working] + + +### changes to cr file + +```yaml + kind: vm + client_vm: + dedicatedcpuplacement: false + sockets: 1 + cores: 2 + threads: 1 + image: kubevirt/fedora-cloud-container-disk-demo:latest + limits: + memory: 4Gi + requests: + memory: 4Gi + network: + front_end: bridge # or masquerade + multiqueue: + enabled: false # if set to true, highly recommend to set selinux to permissive on the nodes where the vms would be scheduled + queues: 0 # must be given if enabled is set to true and ideally should be set to vcpus ideally so sockets*threads*cores, your image must've ethtool installed + extra_options: + - none + #- hostpassthrough +``` + +The above is the additional changes required to run hammerdb in vms. +Currently, we only support images that can be used as [containerDisk](https://docs.openshift.com/container-platform/4.6/virt/virtual_machines/virtual_disks/virt-using-container-disks-with-vms.html#virt-preparing-container-disk-for-vms_virt-using-container-disks-with-vms). + +You can easily make your own container-disk-image as follows by downloading your qcow2 image of choice. +You can then make changes to your qcow2 image as needed using virt-customize. + +```bash +cat << END > Dockerfile +FROM scratch +ADD .qcow2 /disk/ +END + +podman build -t . +podman push +``` + +You can either access results by indexing them directly or by accessing the console. +The results are stored in /tmp/ directory \ No newline at end of file diff --git a/docs/stressng.md b/docs/stressng.md index 421218ec5..0f055a0ad 100644 --- a/docs/stressng.md +++ b/docs/stressng.md @@ -94,7 +94,7 @@ Note: this is currently in preview mode. ``` The above is the additional changes required to run stressng in vms. -Currently, we only support images that can be used as [containerDisk](https://kubevirt.io/user-guide/docs/latest/creating-virtual-machines/disks-and-volumes.html#containerdisk). +Currently, we only support images that can be used as [containerDisk](https://docs.openshift.com/container-platform/4.6/virt/virtual_machines/virtual_disks/virt-using-container-disks-with-vms.html#virt-preparing-container-disk-for-vms_virt-using-container-disks-with-vms). You can easily make your own container-disk-image as follows by downloading your qcow2 image of choice. You can then make changes to your qcow2 image as needed using virt-customize. diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml new file mode 100644 index 000000000..0b5c9b09b --- /dev/null +++ b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml @@ -0,0 +1,78 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-benchmark + namespace: my-ripsaw +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/test/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_client: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 4 + limits_memory: 16Gi + db_type: "mariadb" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "mariadb-deployment.mariadb-db" + db_port: "3306" + db_warehouses: 1 + db_num_workers: 1 + db_user: "root" + db_pass: "mysql" + db_name: "tpcc" + transactions: 500000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # es custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "10.3.0" + es_os_version: "centos8" + es_kind: "pod" diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server.yaml b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server.yaml new file mode 100644 index 000000000..ceed5fb1f --- /dev/null +++ b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mariadb-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-custom-config + namespace: mariadb-db +data: + custom.conf: | + # This group is read both both by the client and the server + # use it for options that affect everything + # + [client-server] + # + # include all files from the config directory + # + !includedir /etc/my.cnf.d + port=3306 + ssl=0 + [mysqld] + innodb_file_per_table=1 + innodb_flush_method=O_DIRECT + innodb_adaptive_flushing=1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + matchLabels: + app: mariadb + template: + metadata: + labels: + app: mariadb + type: mariadb-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "node1" + terminationGracePeriodSeconds: 10 + containers: + - name: mariadb + image: centos/mariadb-103-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 3306 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: MYSQL_USER + value: "test" + - name: MYSQL_PASSWORD + value: "test" + - name: MYSQL_ROOT_PASSWORD + value: "mysql" + volumeMounts: + - name: mariadb-custom-config + mountPath: /etc/my.cnf + subPath: custom.conf #should be the name used in the ConfigMap + volumes: + - name: mariadb-custom-config + configMap: + name: mariadb-custom-config +--- +apiVersion: v1 +kind: Service +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + app: mariadb + ports: + - protocol: TCP + port: 3306 + targetPort: 3306 diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml new file mode 100644 index 000000000..410550aaa --- /dev/null +++ b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml @@ -0,0 +1,100 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mariadb-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-custom-config + namespace: mariadb-db +data: + custom.conf: | + # This group is read both both by the client and the server + # use it for options that affect everything + # + [client-server] + # + # include all files from the config directory + # + !includedir /etc/my.cnf.d + port=3306 + ssl=0 + [mysqld] + innodb_file_per_table=1 + innodb_flush_method=O_DIRECT + innodb_adaptive_flushing=1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + matchLabels: + app: mariadb + template: + metadata: + labels: + app: mariadb + type: mariadb-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "worker-0" + terminationGracePeriodSeconds: 10 + containers: + - name: mariadb + image: centos/mariadb-103-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 3306 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 14 + memory: 16Gi + env: + - name: MYSQL_USER + value: "test" + - name: MYSQL_PASSWORD + value: "test" + - name: MYSQL_ROOT_PASSWORD + value: "mysql" + volumeMounts: + - name: mariadb-custom-config + mountPath: /etc/my.cnf + subPath: custom.conf #should be the name used in the ConfigMap + - name: maria-persistent-storage + mountPath: /var/lib/mysql + readOnly: false + volumes: + - name: mariadb-custom-config + configMap: + name: mariadb-custom-config + - name: maria-persistent-storage + volumeClaimTemplates: + - metadata: + name: mysql-ocs-data + spec: + storageClassName: ocs-storagecluster-ceph-rbd + accessModes: [ "ReadWriteOnce" ] + volumeMode: Filesystem + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + app: mariadb + ports: + - protocol: TCP + port: 3306 + targetPort: 3306 diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml new file mode 100644 index 000000000..cad64ee45 --- /dev/null +++ b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml @@ -0,0 +1,99 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-vm-benchmark + namespace: my-ripsaw +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/test/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_client: "node1" + db_type: "mariadb" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "127.0.0.1" + db_port: "3306" + db_warehouses: 1 + db_num_workers: 1 + db_user: "root" + db_pass: "mysql" + db_name: "tpcc" + transactions: 500000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # es custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "10.3.0" + es_os_version: "centos8" + es_kind: "vm" + kind: vm + client_vm: + dedicatedcpuplacement: false + sockets: 1 + cores: 4 + threads: 1 + image: quay.io/ebattat/centos8-mariadb10-3-container-disk:latest + limits: + memory: 16Gi + requests: + memory: 100Mi + network: + front_end: bridge # or masquerade + multiqueue: + enabled: false # if set to true, highly recommend to set selinux to permissive on the nodes where the vms would be scheduled + queues: 0 # must be given if enabled is set to true and ideally should be set to vcpus ideally so sockets*threads*cores, your image must've ethtool installed + extra_options: + - none + #- hostpassthrough + # PVC + #storageclass: ocs-storagecluster-ceph-rbd + # Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce + #pvcaccessmode: ReadWriteMany + # Can be one of Filesystem,Block Default: Filesystem + #pvcvolumemode: Block + #storagesize: 5Gi diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml new file mode 100644 index 000000000..29b9cd245 --- /dev/null +++ b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml @@ -0,0 +1,78 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-benchmark + namespace: my-ripsaw +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/test/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_client: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 4 + limits_memory: 16Gi + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "mssql-deployment.mssql-db" + db_port: "1433" + db_warehouses: 1 + db_num_workers: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 500000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # es custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "2019" + es_os_version: "centos8" + es_kind: "pod" \ No newline at end of file diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml new file mode 100644 index 000000000..aecc2e12f --- /dev/null +++ b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: mssql-db +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + replicas: 1 + selector: + matchLabels: + app: mssql + template: + metadata: + labels: + app: mssql + type: mssql-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "node1" + terminationGracePeriodSeconds: 10 + containers: + - name: mssql + image: mcr.microsoft.com/mssql/rhel/server:2019-latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 1433 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: MSSQL_PID + value: "Enterprise" + - name: ACCEPT_EULA + value: "Y" + - name: MSSQL_SA_PASSWORD + value: "s3curePasswordString" +--- +apiVersion: v1 +kind: Service +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + selector: + app: mssql + ports: + - protocol: TCP + port: 1433 + targetPort: 1433 + + diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml new file mode 100644 index 000000000..e63a90435 --- /dev/null +++ b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml @@ -0,0 +1,74 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: mssql-db +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + replicas: 1 + selector: + matchLabels: + app: mssql + template: + metadata: + labels: + app: mssql + type: mssql-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "worker-0" + terminationGracePeriodSeconds: 10 + containers: + - name: mssql + image: mcr.microsoft.com/mssql/rhel/server:2019-latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 1433 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 6 + memory: 16Gi + env: + - name: MSSQL_PID + value: "Enterprise" + - name: ACCEPT_EULA + value: "Y" + - name: MSSQL_SA_PASSWORD + value: "s3curePasswordString" + volumeMounts: + - name: mssql-persistent-storage + mountPath: /var/opt/mssql + readOnly: false + volumes: + - name: mssql-persistent-storage + volumeClaimTemplates: + - metadata: + name: mssql-persistent-storage + spec: + storageClassName: ocs-storagecluster-ceph-rbd + accessModes: [ "ReadWriteOnce" ] + volumeMode: Filesystem + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + selector: + app: mssql + ports: + - protocol: TCP + port: 1433 + targetPort: 1433 diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml new file mode 100644 index 000000000..90ad3382c --- /dev/null +++ b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml @@ -0,0 +1,99 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-vm-benchmark + namespace: my-ripsaw +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/test/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_client: "node1" + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "127.0.0.1" + db_port: "1433" + db_warehouses: 1 + db_num_workers: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 500000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # es custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "2019" + es_os_version: "centos8" + es_kind: "vm" + kind: vm + client_vm: + dedicatedcpuplacement: false + sockets: 1 + cores: 4 + threads: 1 + image: quay.io/ebattat/centos8-mssql2019-container-disk:latest + limits: + memory: 16Gi + requests: + memory: 100Mi + network: + front_end: bridge # or masquerade + multiqueue: + enabled: false # if set to true, highly recommend to set selinux to permissive on the nodes where the vms would be scheduled + queues: 0 # must be given if enabled is set to true and ideally should be set to vcpus ideally so sockets*threads*cores, your image must've ethtool installed + extra_options: + - none + #- hostpassthrough + # PVC + #storageclass: ocs-storagecluster-ceph-rbd + # Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce + #pvcaccessmode: ReadWriteMany + # Can be one of Filesystem,Block Default: Filesystem + #pvcvolumemode: Block + #storagesize: 5Gi diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml new file mode 100644 index 000000000..4e2b2ccf7 --- /dev/null +++ b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml @@ -0,0 +1,78 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-benchmark + namespace: my-ripsaw +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: # add custom hammerdb image + pin: false # true for nodeSelector + pin_client: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 4 + limits_memory: 16Gi + db_type: "pg" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "postgres-deployment.postgres-db" + db_port: "5432" + db_warehouses: 1 + db_num_workers: 1 + db_user: "postgres" + db_pass: "postgres" + db_name: "tpcc" + transactions: 500000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "postgres" + db_postgresql_superuser_pass: "postgres" + db_postgresql_defaultdbase: "tpcc" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # es custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "10" + es_os_version: "centos8" + es_kind: "pod" \ No newline at end of file diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml new file mode 100644 index 000000000..888ac33a4 --- /dev/null +++ b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: postgres-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-custom-config + namespace: postgres-db +data: + custom.pg_hba.conf: | + # TYPE DATABASE USER ADDRESS METHOD + # "local" is for Unix domain socket connections only + local all all trust + # IPv4 local connections: + host all all all trust +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + matchLabels: + app: postgres + replicas: 1 + template: + metadata: + labels: + app: postgres + type: postgres-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "node1" + terminationGracePeriodSeconds: 10 + containers: + - name: postgres + image: centos/postgresql-10-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5432 + resources: + limits: + cpu: 200m + memory: 100Mi + requests: + cpu: 4 + memory: 16Gi + env: + - name: POSTGRESQL_USER + value: "test" + - name: POSTGRESQL_PASSWORD + value: "test" + - name: POSTGRESQL_DATABASE + value: "test" + - name: POSTGRESQL_ADMIN_PASSWORD + value: "postgres" + volumeMounts: + - name: postgres-custom-config + mountPath: /var/lib/pgsql/data/pg_hba.conf + subPath: custom.pg_hba.conf #should be the name used in the ConfigMap + volumes: + - name: postgres-custom-config + configMap: + name: postgres-custom-config + +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + app: postgres + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml new file mode 100644 index 000000000..93bdff7dc --- /dev/null +++ b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: postgres-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-custom-config + namespace: postgres-db +data: + custom.pg_hba.conf: | + # TYPE DATABASE USER ADDRESS METHOD + # "local" is for Unix domain socket connections only + local all all trust + # IPv4 local connections: + host all all all trust +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + matchLabels: + app: postgres + replicas: 1 + template: + metadata: + labels: + app: postgres + type: postgres-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "worker-0" + terminationGracePeriodSeconds: 10 + containers: + - name: postgres + image: centos/postgresql-10-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5432 + resources: + limits: + cpu: 200m + memory: 100Mi + requests: + cpu: 14 + memory: 16Gi + env: + - name: POSTGRESQL_USER + value: "test" + - name: POSTGRESQL_PASSWORD + value: "test" + - name: POSTGRESQL_DATABASE + value: "test" + - name: POSTGRESQL_ADMIN_PASSWORD + value: "postgres" + volumeMounts: + - name: postgres-custom-config + mountPath: /var/lib/pgsql/data/pg_hba.conf + subPath: custom.pg_hba.conf #should be the name used in the ConfigMap + - name: postgres-persistent-storage + mountPath: /var/lib/pgsql/data + readOnly: false + volumes: + - name: postgres-custom-config + configMap: + name: postgres-custom-config + - name: postgres-persistent-storage + volumeClaimTemplates: + - metadata: + name: postgres-persistent-storage + spec: + storageClassName: ocs-storagecluster-ceph-rbd + accessModes: [ "ReadWriteOnce" ] + volumeMode: Filesystem + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + app: postgres + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml new file mode 100644 index 000000000..d1686d774 --- /dev/null +++ b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml @@ -0,0 +1,101 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-vm-benchmark + namespace: my-ripsaw +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: # add custom hammerdb image + pin: false # true for nodeSelector + pin_client: "node1" + db_type: "pg" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "127.0.0.1" + db_port: "5432" + db_warehouses: 1 + db_num_workers: 1 + db_user: "postgres" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 500000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "postgres" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "tpcc" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # es custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "10" + es_os_version: "centos8" + es_kind: "vm" + kind: vm + client_vm: + dedicatedcpuplacement: false + sockets: 1 + cores: 4 + threads: 1 + #image: kubevirt/fedora-cloud-container-disk-demo:latest + image: quay.io/ebattat/centos8-postgres10-container-disk:latest + limits: + memory: 16Gi + requests: + memory: 16Gi + network: + front_end: bridge # or masquerade + multiqueue: + enabled: false # if set to true, highly recommend to set selinux to permissive on the nodes where the vms would be scheduled + queues: 0 # must be given if enabled is set to true and ideally should be set to vcpus ideally so sockets*threads*cores, your image must've ethtool installed + extra_options: + - none + #- hostpassthrough + # PVC + #storageclass: ocs-storagecluster-ceph-rbd + # Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce + #pvcaccessmode: ReadWriteMany + # Can be one of Filesystem,Block Default: Filesystem + #pvcvolumemode: Block + #storagesize: 5Gi + diff --git a/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml b/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml new file mode 100644 index 000000000..29b9cd245 --- /dev/null +++ b/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml @@ -0,0 +1,78 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-benchmark + namespace: my-ripsaw +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/test/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_client: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 4 + limits_memory: 16Gi + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "mssql-deployment.mssql-db" + db_port: "1433" + db_warehouses: 1 + db_num_workers: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 500000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # es custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "2019" + es_os_version: "centos8" + es_kind: "pod" \ No newline at end of file diff --git a/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml b/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml new file mode 100644 index 000000000..520373c0b --- /dev/null +++ b/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml @@ -0,0 +1,93 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-vm-benchmark + namespace: my-ripsaw +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/test/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_client: "node1" + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + # true only for first run to build schema + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "127.0.0.1" + db_port: "1433" + db_warehouses: 1 + db_num_workers: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 500000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # es custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "2019" + es_os_version: "centos8" + es_kind: "vm" + kind: vm + client_vm: + dedicatedcpuplacement: false + sockets: 1 + cores: 4 + threads: 1 + image: quay.io/ebattat/centos8-mssql2019-container-disk:latest + limits: + memory: 16Gi + requests: + memory: 100Mi + network: + front_end: bridge # or masquerade + multiqueue: + enabled: false # if set to true, highly recommend to set selinux to permissive on the nodes where the vms would be scheduled + queues: 0 # must be given if enabled is set to true and ideally should be set to vcpus ideally so sockets*threads*cores, your image must've ethtool installed + extra_options: + - none + #- hostpassthrough diff --git a/resources/crds/ripsaw_v1alpha1_hammerdb_cr.yaml b/resources/crds/ripsaw_v1alpha1_hammerdb_cr.yaml deleted file mode 100644 index 8be445f15..000000000 --- a/resources/crds/ripsaw_v1alpha1_hammerdb_cr.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: hammerdb-benchmark-example - namespace: my-ripsaw -spec: - # where elastic search is running - elasticsearch: - url: http://my.elasticsearch.server:80 - verify_cert: false - parallel: false - # clustername: myk8scluster - # test_user: ripsaw - workload: - name: hammerdb - args: - ## 'initialize' enables the schema creation in the db if it's not done already - ## 'benchmark' enables the actual benchmarking using the parameters further down below - db_init: true - db_benchmark: true - ## Standard options used for all operations, initialization and benchmark - db_server: "mssql-deployment.sql-server" - db_port: "1443" - db_user: "SA" - db_pass: "s3curePasswordString" - db_warehouses: 1 - db_num_workers: 1 - db_tcp: "true" - - ## Workload specific settings - ## runtime and transactions are mutually exclusive, transactions will override runtime - transactions: 100000 - runtime: 30 # corresponds to runtimer in the tcl script (in minutes) - rampup: 1 # corresponds to mssqls_rampup in the tcl script (minutes) - samples: 3 # needs to be implemented - #clients: 4 # corresponds to virtual users (vu) in the tcl script, this should be the maximum number of clients which we approach logarithmically (sp?) - diff --git a/resources/crds/ripsaw_v1alpha1_hammerdb_index_cr.yaml b/resources/crds/ripsaw_v1alpha1_hammerdb_index_cr.yaml deleted file mode 100644 index 93b743cca..000000000 --- a/resources/crds/ripsaw_v1alpha1_hammerdb_index_cr.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: hammerdb-benchmark-example - namespace: my-ripsaw -spec: - # where elastic search is running - elasticsearch: - url: http://my.elasticsearch.server:80 - verify_cert: false - parallel: false - # clustername: myk8scluster - # test_user: ripsaw - workload: - # cleanup: true - name: hammerdb - args: - db_server: false - db_port: false - warehouses: 1 - workers: 1 - protocol: "tcp" - runtime: 30 diff --git a/resources/crds/ripsaw_v1alpha1_stressng_cr.yaml b/resources/crds/ripsaw_v1alpha1_stressng_cr.yaml index b8cd8c549..cd7a213dd 100644 --- a/resources/crds/ripsaw_v1alpha1_stressng_cr.yaml +++ b/resources/crds/ripsaw_v1alpha1_stressng_cr.yaml @@ -15,9 +15,13 @@ spec: runtype: "parallel" timeout: "30" instances: 1 - # nodeselector: - # key1: value1 - # key2: value2 + pin: false # true for nodeSelector + pin_client: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 2 + limits_memory: 4Gi # cpu stressor options cpu_stressors: "1" cpu_percentage: "100" diff --git a/roles/hammerdb/tasks/main.yaml b/roles/hammerdb/tasks/main.yaml index 617dc2bf8..902587034 100644 --- a/roles/hammerdb/tasks/main.yaml +++ b/roles/hammerdb/tasks/main.yaml @@ -25,19 +25,37 @@ namespace: '{{ operator_namespace }}' register: resource_state +- name: Create PVC(s) + k8s: + definition: + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: "claim-{{ trunc_uuid }}" + namespace: '{{ operator_namespace }}' + annotations: + volume.beta.kubernetes.io/storage-class: "{{ workload_args.client_vm.storageclass }}" + spec: + accessModes: + - "{{ workload_args.client_vm.pvcaccessmode | default('ReadWriteOnce') }}" + volumeMode: "{{ workload_args.client_vm.pvcvolumemode | default('Filesystem') }}" + resources: + requests: + storage: "{{ workload_args.client_vm.storagesize }}" + when: workload_args.client_vm.storageclass is defined - block: - - name: template mysql db creation script - k8s: + - name: template mariadb db creation script + k8s: definition: apiVersion: v1 kind: ConfigMap - metadata: + metadata: name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' - data: - createdb.tcl: "{{ lookup('template', 'createdb_mysql.tcl.j2') }}" - when: (workload_args.db_type == "mysql") + data: + createdb.tcl: "{{ lookup('template', 'createdb_mariadb.tcl.j2') }}" + when: (workload_args.db_type == "mariadb") - name: template mssql db creation script k8s: @@ -49,7 +67,7 @@ namespace: '{{ operator_namespace }}' data: createdb.tcl: "{{ lookup('template', 'createdb_mssql.tcl.j2') }}" - when: (workload_args.db_type == "mssql") + when: (workload_args.db_type == "mssql") - name: template postgresql db creation script k8s: @@ -61,9 +79,9 @@ namespace: '{{ operator_namespace }}' data: createdb.tcl: "{{ lookup('template', 'createdb_pg.tcl.j2') }}" - when: (workload_args.db_type == "pg") + when: (workload_args.db_type == "pg") - - name: template mysql db workload script + - name: template mariadb db workload script k8s: definition: apiVersion: v1 @@ -72,8 +90,8 @@ name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: - tpcc-workload-mysql.tcl: "{{ lookup('template', 'tpcc-workload-mysql.tcl.j2') }}" - when: (workload_args.db_type == "mysql") + tpcc-workload-mariadb.tcl: "{{ lookup('template', 'tpcc-workload-mariadb.tcl.j2') }}" + when: (workload_args.db_type == "mariadb") - name: template mssql db workload script k8s: @@ -85,7 +103,7 @@ namespace: '{{ operator_namespace }}' data: tpcc-workload-mssql.tcl: "{{ lookup('template', 'tpcc-workload-mssql.tcl.j2') }}" - when: (workload_args.db_type == "mssql") + when: (workload_args.db_type == "mssql") - name: template postgresql db workload script k8s: @@ -97,7 +115,43 @@ namespace: '{{ operator_namespace }}' data: tpcc-workload-pg.tcl: "{{ lookup('template', 'tpcc-workload-pg.tcl.j2') }}" - when: (workload_args.db_type == "pg") + when: (workload_args.db_type == "pg") + + - name: template mssql workload vm + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: '{{ ansible_operator_meta.name }}-mssql-workload-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' + data: + run_mssql_script.sh: "{{ lookup('template', 'db_mssql_workload_vm.sh.j2') }}" + when: workload_args.db_type == "mssql" and (workload.args.kind | default('pod')) == "vm" + + - name: template mariadb workload vm + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: '{{ ansible_operator_meta.name }}-mariadb-workload-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' + data: + run_mariadb_script.sh: "{{ lookup('template', 'db_mariadb_workload_vm.sh.j2') }}" + when: workload_args.db_type == "mariadb" and (workload.args.kind | default('pod')) == "vm" + + - name: template postgresql workload vm + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: '{{ ansible_operator_meta.name }}-postgres-workload-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' + data: + run_postgres_script.sh: "{{ lookup('template', 'db_postgres_workload_vm.sh.j2') }}" + when: workload_args.db_type == "pg" and (workload.args.kind | default('pod')) == "vm" - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 @@ -108,7 +162,7 @@ state: Starting DB complete: false when: workload_args.db_init is defined and workload_args.db_init - + - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -132,11 +186,38 @@ when: resource_state.resources[0].status.state == "Building" - block: - - name: start db creation job - k8s: - state: present - definition: "{{ lookup('template', 'db_creation.yml') | from_yaml }}" - + - block: + - name: start db creation job + k8s: + state: present + definition: "{{ lookup('template', 'db_creation.yml') | from_yaml }}" + + when: (workload.args.kind | default('pod')) == "pod" + + - block: + - name: set complete to false + command: "redis-cli set db-creation-{{trunc_uuid}} false" + + - name: start mssql db creation vm job + k8s: + state: present + definition: "{{ lookup('template', 'db_creation_mssql_vm.yml') | from_yaml }}" + when: (workload_args.db_type == "mssql") + + - name: start mariadb db creation vm job + k8s: + state: present + definition: "{{ lookup('template', 'db_creation_mariadb_vm.yml') | from_yaml }}" + when: (workload_args.db_type == "mariadb") + + - name: start postgres db creation vm job + k8s: + state: present + definition: "{{ lookup('template', 'db_creation_postgres_vm.yml') | from_yaml }}" + when: (workload_args.db_type == "pg") + + when: (workload.args.kind | default('pod')) == "vm" + - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -150,54 +231,84 @@ - block: - - name: wait for db creation job to finish - k8s_facts: - kind: Job - api_version: batch/v1 - name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' - namespace: "{{ operator_namespace }}" - register: hammerdb_creator_pod + - block: + - name: wait for db creation job to finish + k8s_facts: + kind: Job + api_version: batch/v1 + name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' + namespace: "{{ operator_namespace }}" + register: hammerdb_creator_pod - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting DB Workload - complete: false - when: hammerdb_creator_pod | json_query('resources[].status.succeeded') and workload_args.db_benchmark is defined and workload_args.db_benchmark + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Starting DB Workload + complete: false + when: hammerdb_creator_pod | json_query('resources[].status.succeeded') and workload_args.db_benchmark is defined and workload_args.db_benchmark - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: DB Created - complete: true - when: hammerdb_creator_pod | json_query('resources[].status.succeeded') and (workload_args.db_benchmark is not defined or not workload_args.db_benchmark) + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: DB Created + complete: true + when: hammerdb_creator_pod | json_query('resources[].status.succeeded') and (workload_args.db_benchmark is not defined or not workload_args.db_benchmark) + when: (workload.args.kind | default('pod')) == "pod" + + - block: + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Starting DB Workload + complete: false + - name: get complete + command: "redis-cli get db-creation-{{trunc_uuid}}" + register: db_creation_status + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: DB Created + complete: false + when: db_creation_status.stdout == "true" + when: (workload.args.kind | default('pod')) == "vm" when: resource_state.resources[0].status.state == "DB Creating" - + - block: + - name: set complete to false + command: "redis-cli set complete false" + when: (workload.args.kind | default('pod')) == "vm" + - name: start mssql db workload job k8s: state: present definition: "{{ lookup('template', 'db_mssql_workload.yml.j2') | from_yaml }}" - when: (workload_args.db_type == "mssql") + when: workload_args.db_type == "mssql" and (workload.args.kind | default('pod')) == "pod" - - name: start mysql db workload job + - name: start mariadb db workload job k8s: state: present - definition: "{{ lookup('template', 'db_mysql_workload.yml.j2') | from_yaml }}" - when: (workload_args.db_type == "mysql") + definition: "{{ lookup('template', 'db_mariadb_workload.yml.j2') | from_yaml }}" + when: workload_args.db_type == "mariadb" and (workload.args.kind | default('pod')) == "pod" - name: start postgresql db workload job k8s: state: present definition: "{{ lookup ('template', 'db_postgres_workload.yml.j2') | from_yaml }}" - when: (workload_args.db_type == "pg") + when: workload_args.db_type == "pg" and (workload.args.kind | default('pod')) == "pod" - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 @@ -211,22 +322,40 @@ when: resource_state.resources[0].status.state == "Starting DB Workload" - block: - - name: wait for db workload job to finish - k8s_facts: - kind: Job - api_version: batch/v1 - name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' - namespace: "{{ operator_namespace }}" - register: hammerdb_workload_pod + - block: + - name: wait for db workload job to finish + k8s_facts: + kind: Job + api_version: batch/v1 + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' + namespace: "{{ operator_namespace }}" + register: hammerdb_workload_pod - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: DB Workload Complete - complete: true - when: "hammerdb_workload_pod | json_query('resources[].status.succeeded')" + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: DB Workload Complete + complete: true + when: "hammerdb_workload_pod | json_query('resources[].status.succeeded')" + when: (workload.args.kind | default('pod')) == "pod" + + - block: + - name: get complete + command: "redis-cli get complete" + register: complete_status + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: DB Workload Complete + complete: true + when: complete_status.stdout == "true" + when: (workload.args.kind | default('pod')) == "vm" when: resource_state.resources[0].status.state == "DB workload running" diff --git a/roles/hammerdb/templates/createdb_mysql.tcl.j2 b/roles/hammerdb/templates/createdb_mariadb.tcl.j2 similarity index 92% rename from roles/hammerdb/templates/createdb_mysql.tcl.j2 rename to roles/hammerdb/templates/createdb_mariadb.tcl.j2 index 0c08a45eb..9cbc3e170 100644 --- a/roles/hammerdb/templates/createdb_mysql.tcl.j2 +++ b/roles/hammerdb/templates/createdb_mariadb.tcl.j2 @@ -11,6 +11,7 @@ dbset db mysql dbset bm TPC-C diset connection mysql_host {{workload_args.db_server}} diset connection mysql_port {{workload_args.db_port}} +diset connection mysql_socket {{workload_args.db_mysql_socket}} diset tpcc mysql_user {{workload_args.db_user}} diset tpcc mysql_pass {{workload_args.db_pass}} diset tpcc mysql_dbase {{workload_args.db_name}} @@ -22,4 +23,3 @@ puts "CREATING SCHEMA" buildschema wait_to_complete vwait forever - diff --git a/roles/hammerdb/templates/createdb_pg.tcl.j2 b/roles/hammerdb/templates/createdb_pg.tcl.j2 index 2cb6265d9..fed3d4273 100644 --- a/roles/hammerdb/templates/createdb_pg.tcl.j2 +++ b/roles/hammerdb/templates/createdb_pg.tcl.j2 @@ -11,8 +11,6 @@ dbset db pg dbset bm TPC-C diset connection pg_host {{workload_args.db_server}} diset connection pg_port {{workload_args.db_port}} -diset tpcc pg_user {{workload_args.db_user}} -diset tpcc pg_pass {{workload_args.db_pass}} diset tpcc pg_dbase {{workload_args.db_postgresql_defaultdbase}} diset tpcc pg_count_ware {{workload_args.db_warehouses}} diset tpcc pg_num_vu {{workload_args.db_num_workers}} @@ -21,4 +19,3 @@ puts "CREATING SCHEMA" buildschema wait_to_complete vwait forever - diff --git a/roles/hammerdb/templates/db_creation.yml b/roles/hammerdb/templates/db_creation.yml index 83edfc0ab..750eab2bd 100644 --- a/roles/hammerdb/templates/db_creation.yml +++ b/roles/hammerdb/templates/db_creation.yml @@ -9,12 +9,27 @@ spec: metadata: labels: app: hammerdb_creator-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-creator-{{ trunc_uuid }} + benchmark-operator-uuid: {{ uuid }} spec: +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} containers: - name: hammerdb +{% if workload_args.resources is sameas true %} + resources: + requests: + cpu: {{ workload_args.requests_cpu }} + memory: {{ workload_args.requests_memory }} + limits: + cpu: {{ workload_args.limits_cpu }} + memory: {{ workload_args.limits_memory }} +{% endif %} command: ["/bin/sh", "-c"] args: ["/usr/local/bin/uid_entrypoint; cd /hammer; ./hammerdbcli auto /creator/createdb.tcl"] image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:latest') }} diff --git a/roles/hammerdb/templates/db_creation_mariadb_vm.yml b/roles/hammerdb/templates/db_creation_mariadb_vm.yml new file mode 100644 index 000000000..e6b1949b1 --- /dev/null +++ b/roles/hammerdb/templates/db_creation_mariadb_vm.yml @@ -0,0 +1,121 @@ +--- +apiVersion: kubevirt.io/v1alpha3 +kind: VirtualMachineInstance +metadata: + name: {{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + labels: + app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-operator-uuid: {{ uuid }} +spec: + domain: + cpu: + sockets: {{ workload_args.client_vm.sockets }} + cores: {{ workload_args.client_vm.cores }} + threads: {{ workload_args.client_vm.threads }} + dedicatedCpuPlacement: {{ workload_args.client_vm.dedicatedcpuplacement }} +{% if 'hostpassthrough' in workload_args.client_vm.extra_options %} + model: host-passthrough +{% endif %} + devices: + disks: + - disk: + bus: virtio + name: containerdisk + - disk: + bus: virtio + name: cloudinitdisk + - disk: {} + name: hammerdb-creator-volume + # set serial + serial: CVLY623300HK240C + - disk: {} + name: hammerdb-workload-volume + # set serial + serial: CVLY623300HK240D + - disk: {} + name: hammerdb-mariadb-workload-volume + # set serial + serial: CVLY623300HK240E +{% if workload_args.client_vm.storageclass is defined + or workload_args.client_vm.hostpath is defined %} + - disk: + bus: virtio + name: data-volume + serial: data +{% endif %} + interfaces: + - name: default + {{ workload_args.client_vm.network.front_end }}: {} + networkInterfaceMultiqueue: {{ workload_args.client_vm.network.multiqueue.enabled }} + machine: + type: "" + resources: + requests: + memory: {{ workload_args.client_vm.requests.memory }} + limits: + memory: {{ workload_args.client_vm.limits.memory }} + terminationGracePeriodSeconds: 0 +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client }}' +{% endif %} + networks: + - name: default + pod: {} + volumes: + - name: containerdisk + containerDisk: + image: {{ workload_args.client_vm.image }} + - cloudInitNoCloud: + userData: |- + #cloud-config + password: centos + chpasswd: { expire: False } + bootcmd: + # mount the ConfigMap + - "mkdir /creator" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240C | cut -f1 -d' ') /creator" + - "mkdir /workload" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240D | cut -f1 -d' ') /workload" + - "mkdir /tmp/hammerdb-mariadb-test" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240E | cut -f1 -d' ') /tmp/hammerdb-mariadb-test" + runcmd: +{% if workload_args.client_vm.storageclass is defined + or workload_args.client_vm.hostpath is defined %} + - "mkdir -p /var/lib/mysql || true" + - mkfs.ext4 /dev/disk/by-id/virtio-data + - "mount /dev/disk/by-id/virtio-data /var/lib/mysql" + - "chown -R mysql:mysql /var/lib/mysql" +{% endif %} +{% if workload_args.client_vm.network.multiqueue.enabled %} + - dnf install -y ethtool + - ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }} +{% endif %} + - dnf install -y redis + - systemctl start redis + - systemctl enable redis + - bash /tmp/hammerdb-mariadb-test/run_mariadb_script.sh + name: cloudinitdisk + - configMap: + name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" + name: hammerdb-creator-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + name: hammerdb-workload-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-mariadb-workload-{{ trunc_uuid }}" + name: hammerdb-mariadb-workload-volume +{% if workload_args.client_vm.storageclass is defined %} + - name: data-volume + persistentVolumeClaim: + claimName: claim-{{ trunc_uuid }} +{% elif workload_args.client_vm.hostpath is defined %} + - name: data-volume + hostDisk: + path: {{ workload_args.hostpath }} + capacity: {{ workload_args.storagesize | default("5Gi") }} + type: DiskOrCreate +{% endif %} +status: {} diff --git a/roles/hammerdb/templates/db_creation_mssql_vm.yml b/roles/hammerdb/templates/db_creation_mssql_vm.yml new file mode 100644 index 000000000..f62e7d504 --- /dev/null +++ b/roles/hammerdb/templates/db_creation_mssql_vm.yml @@ -0,0 +1,122 @@ +--- +apiVersion: kubevirt.io/v1alpha3 +kind: VirtualMachineInstance +metadata: + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + namespace: "{{ operator_namespace }}" + labels: + app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-operator-uuid: {{ uuid }} +spec: + domain: + cpu: + sockets: {{ workload_args.client_vm.sockets }} + cores: {{ workload_args.client_vm.cores }} + threads: {{ workload_args.client_vm.threads }} + dedicatedCpuPlacement: {{ workload_args.client_vm.dedicatedcpuplacement }} +{% if 'hostpassthrough' in workload_args.client_vm.extra_options %} + model: host-passthrough +{% endif %} + devices: + disks: + - disk: + bus: virtio + name: containerdisk + - disk: + bus: virtio + name: cloudinitdisk + - disk: {} + name: hammerdb-creator-volume + # set serial + serial: CVLY623300HK240C + - disk: {} + name: hammerdb-workload-volume + # set serial + serial: CVLY623300HK240D + - disk: {} + name: hammerdb-mssql-workload-volume + # set serial + serial: CVLY623300HK240E +{% if workload_args.client_vm.storageclass is defined + or workload_args.client_vm.hostpath is defined %} + - disk: + bus: virtio + name: data-volume + serial: data +{% endif %} + interfaces: + - name: default + {{ workload_args.client_vm.network.front_end }}: {} + networkInterfaceMultiqueue: {{ workload_args.client_vm.network.multiqueue.enabled }} + machine: + type: "" + resources: + requests: + memory: {{ workload_args.client_vm.requests.memory }} + limits: + memory: {{ workload_args.client_vm.limits.memory }} + terminationGracePeriodSeconds: 0 +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client }}' +{% endif %} + networks: + - name: default + pod: {} + volumes: + - name: containerdisk + containerDisk: + image: {{ workload_args.client_vm.image }} + - cloudInitNoCloud: + userData: |- + #cloud-config + password: centos + chpasswd: { expire: False } + bootcmd: + # mount the ConfigMap + - "mkdir /creator" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240C | cut -f1 -d' ') /creator" + - "mkdir /workload" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240D | cut -f1 -d' ') /workload" + - "mkdir /tmp/hammerdb-mssql-test" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240E | cut -f1 -d' ') /tmp/hammerdb-mssql-test" + runcmd: +{% if workload_args.client_vm.storageclass is defined + or workload_args.client_vm.hostpath is defined %} + - "mkdir -p /var/opt/mssql || true" + - mkfs.ext4 /dev/disk/by-id/virtio-data + - "mount /dev/disk/by-id/virtio-data /var/opt/mssql" + - "chown -R mssql:mssql /var/opt/mssql" + - "chgrp mssql /var/opt/mssql" +{% endif %} +{% if workload_args.client_vm.network.multiqueue.enabled %} + - dnf install -y ethtool + - ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }} +{% endif %} + - dnf install -y redis + - systemctl start redis + - systemctl enable redis + - bash /tmp/hammerdb-mssql-test/run_mssql_script.sh + name: cloudinitdisk + - configMap: + name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" + name: hammerdb-creator-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + name: hammerdb-workload-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-mssql-workload-{{ trunc_uuid }}" + name: hammerdb-mssql-workload-volume +{% if workload_args.client_vm.storageclass is defined %} + - name: data-volume + persistentVolumeClaim: + claimName: claim-{{ trunc_uuid }} +{% elif workload_args.client_vm.hostpath is defined %} + - name: data-volume + hostDisk: + path: {{ workload_args.hostpath }} + capacity: {{ workload_args.storagesize | default("5Gi") }} + type: DiskOrCreate +{% endif %} +status: {} diff --git a/roles/hammerdb/templates/db_creation_postgres_vm.yml b/roles/hammerdb/templates/db_creation_postgres_vm.yml new file mode 100644 index 000000000..ae7faebe3 --- /dev/null +++ b/roles/hammerdb/templates/db_creation_postgres_vm.yml @@ -0,0 +1,121 @@ +--- +apiVersion: kubevirt.io/v1alpha3 +kind: VirtualMachineInstance +metadata: + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + namespace: "{{ operator_namespace }}" + labels: + app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-operator-uuid: {{ uuid }} +spec: + domain: + cpu: + sockets: {{ workload_args.client_vm.sockets }} + cores: {{ workload_args.client_vm.cores }} + threads: {{ workload_args.client_vm.threads }} + dedicatedCpuPlacement: {{ workload_args.client_vm.dedicatedcpuplacement }} +{% if 'hostpassthrough' in workload_args.client_vm.extra_options %} + model: host-passthrough +{% endif %} + devices: + disks: + - disk: + bus: virtio + name: containerdisk + - disk: + bus: virtio + name: cloudinitdisk + - disk: {} + name: hammerdb-creator-volume + # set serial + serial: CVLY623300HK240C + - disk: {} + name: hammerdb-workload-volume + # set serial + serial: CVLY623300HK240D + - disk: {} + name: hammerdb-postgres-workload-volume + # set serial + serial: CVLY623300HK240E +{% if workload_args.client_vm.storageclass is defined + or workload_args.client_vm.hostpath is defined %} + - disk: + bus: virtio + name: data-volume + serial: data +{% endif %} + interfaces: + - name: default + {{ workload_args.client_vm.network.front_end }}: {} + networkInterfaceMultiqueue: {{ workload_args.client_vm.network.multiqueue.enabled }} + machine: + type: "" + resources: + requests: + memory: {{ workload_args.client_vm.requests.memory }} + limits: + memory: {{ workload_args.client_vm.limits.memory }} + terminationGracePeriodSeconds: 0 +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client }}' +{% endif %} + networks: + - name: default + pod: {} + volumes: + - name: containerdisk + containerDisk: + image: {{ workload_args.client_vm.image }} + - cloudInitNoCloud: + userData: |- + #cloud-config + password: centos + chpasswd: { expire: False } + bootcmd: + # mount the ConfigMap + - "mkdir /creator" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240C | cut -f1 -d' ') /creator" + - "mkdir /workload" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240D | cut -f1 -d' ') /workload" + - "mkdir /tmp/hammerdb-postgres-test" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240E | cut -f1 -d' ') /tmp/hammerdb-postgres-test" +{% if workload_args.client_vm.storageclass is defined + or workload_args.client_vm.hostpath is defined %} + - "mkdir -p /var/lib/pgsql || true" + - mkfs.ext4 /dev/disk/by-id/virtio-data + - "mount /dev/disk/by-id/virtio-data /var/lib/pgsql" + - "chown -R postgres:postgres /var/lib/pgsql" +{% endif %} + runcmd: +{% if workload_args.client_vm.network.multiqueue.enabled %} + - dnf install -y ethtool + - ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }} +{% endif %} + - dnf install -y redis + - systemctl start redis + - systemctl enable redis + - bash /tmp/hammerdb-postgres-test/run_postgres_script.sh + name: cloudinitdisk + - configMap: + name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" + name: hammerdb-creator-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + name: hammerdb-workload-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-postgres-workload-{{ trunc_uuid }}" + name: hammerdb-postgres-workload-volume +{% if workload_args.client_vm.storageclass is defined %} + - name: data-volume + persistentVolumeClaim: + claimName: claim-{{ trunc_uuid }} +{% elif workload_args.client_vm.hostpath is defined %} + - name: data-volume + hostDisk: + path: {{ workload_args.hostpath }} + capacity: {{ workload_args.storagesize | default("5Gi") }} + type: DiskOrCreate +{% endif %} +status: {} diff --git a/roles/hammerdb/templates/db_mysql_workload.yml.j2 b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 similarity index 75% rename from roles/hammerdb/templates/db_mysql_workload.yml.j2 rename to roles/hammerdb/templates/db_mariadb_workload.yml.j2 index c1a6f1467..f2c02a194 100644 --- a/roles/hammerdb/templates/db_mysql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 @@ -10,12 +10,27 @@ spec: metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-operator-uuid: {{ uuid }} spec: +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} containers: - name: hammerdb +{% if workload_args.resources is sameas true %} + resources: + requests: + cpu: {{ workload_args.requests_cpu }} + memory: {{ workload_args.requests_memory }} + limits: + cpu: {{ workload_args.limits_cpu }} + memory: {{ workload_args.limits_memory }} +{% endif %} image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:master') }} imagePullPolicy: Always env: @@ -44,9 +59,9 @@ spec: value: "{{ prometheus.prom_url | default() }}" {% endif %} command: ["/bin/sh", "-c"] - args: + args: - "/usr/local/bin/uid_entrypoint; - export db_type={{workload_args.db_type}}; + export db_type={{workload_args.db_type}}; export timed_test={{workload_args.timed_test}}; export db_server={{workload_args.db_server}}; export db_port={{workload_args.db_port}}; @@ -70,12 +85,15 @@ spec: export samples={{workload_args.samples}}; export db_mysql_storage_engine={{workload_args.db_mysql_storage_engine}}; export db_mysql_partition={{workload_args.db_mysql_partition}}; - cd /hammer; - run_snafu --tool hammerdb -u {{ uuid }} -{% if workload_args.debug is defined and workload_args.debug %} - -v \ +{% if workload_args.es_custom_field is sameas true %} + export es_ocp_version={{workload_args.es_ocp_version}}; + export es_cnv_version={{workload_args.es_cnv_version}}; + export es_db_version={{workload_args.es_db_version}}; + export es_os_version={{workload_args.es_os_version}}; + export es_kind={{workload_args.es_kind}}; {% endif %} - ;" + cd /hammer; + run_snafu --tool hammerdb -u {{ uuid }}" volumeMounts: - name: hammerdb-workload-volume mountPath: "/workload" @@ -87,3 +105,4 @@ spec: defaultMode: 0640 restartPolicy: OnFailure {% include "metadata.yml.j2" %} + diff --git a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 new file mode 100644 index 000000000..59fc4c362 --- /dev/null +++ b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 @@ -0,0 +1,62 @@ +{% if elasticsearch is defined %} +{% if elasticsearch.url is defined %} +export es={{ elasticsearch.url }}; +export es_index={{ elasticsearch.index_name | default("ripsaw-hammerdb") }}; +export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; +export parallel={{ elasticsearch.parallel | default("false") }}; +export uuid={{uuid}}; +{% endif %} +{% endif %} +{% if workload_args.client_vm.network.multiqueue.enabled %} +dnf install -y ethtool; +ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; +{% endif %} +dnf install -y epel-release git python3-pip; +pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper; +export db_type={{workload_args.db_type}}; +export timed_test={{workload_args.timed_test}}; +export db_server={{workload_args.db_server}}; +export db_port={{workload_args.db_port}}; +export db_warehouses={{workload_args.db_warehouses}}; +export db_num_workers={{workload_args.db_num_workers}}; +export db_user={{workload_args.db_user}}; +export db_pass={{workload_args.db_pass}}; +export db_name={{workload_args.db_name}}; +export transactions={{workload_args.transactions}}; +export raiseerror={{workload_args.raiseerror}}; +export keyandthink={{workload_args.keyandthink}}; +export driver={{workload_args.driver}}; +export rampup={{workload_args.rampup}}; +export runtime={{workload_args.runtime}}; +export allwarehouse={{workload_args.allwarehouse}}; +export timeprofile={{workload_args.timeprofile}}; +export async_scale={{workload_args.async_scale}}; +export async_client={{workload_args.async_client}}; +export async_verbose={{workload_args.async_verbose}}; +export async_delay={{workload_args.async_delay}}; +export samples={{workload_args.samples}}; +export db_mysql_storage_engine={{workload_args.db_mysql_storage_engine}}; +export db_mysql_partition={{workload_args.db_mysql_partition}}; +{% if workload_args.es_custom_field is sameas true %} +export es_ocp_version={{workload_args.es_ocp_version}}; +export es_cnv_version={{workload_args.es_cnv_version}}; +export es_db_version={{workload_args.es_db_version}}; +export es_os_version={{workload_args.es_os_version}}; +export es_kind={{workload_args.es_kind}}; +{% endif %} +export HOME=/root; +{% if workload_args.client_vm.storageclass is defined + or workload_args.client_vm.hostpath is defined %} +sudo systemctl stop mariadb.service; +echo "UPDATE mysql.user SET Password=PASSWORD('mysql') WHERE User='root';" > input; +echo "GRANT ALL ON *.* to root@'%' IDENTIFIED BY 'mysql';" >> input; +echo "flush privileges;" >> input; +echo "exit" >> input; +sudo systemctl start mariadb.service; +mysql -u root < input; +{% endif %} +cd /hammer; +./hammerdbcli auto /creator/createdb.tcl; +redis-cli set db-creation-{{trunc_uuid}} true; +run_snafu --tool hammerdb -u {{ uuid }}; +redis-cli set complete true; diff --git a/roles/hammerdb/templates/db_mssql_workload.yml.j2 b/roles/hammerdb/templates/db_mssql_workload.yml.j2 index 1a366a976..cfc6ed8e6 100644 --- a/roles/hammerdb/templates/db_mssql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mssql_workload.yml.j2 @@ -10,12 +10,27 @@ spec: metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-operator-uuid: {{ uuid }} spec: +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} containers: - name: hammerdb +{% if workload_args.resources is sameas true %} + resources: + requests: + cpu: {{ workload_args.requests_cpu }} + memory: {{ workload_args.requests_memory }} + limits: + cpu: {{ workload_args.limits_cpu }} + memory: {{ workload_args.limits_memory }} +{% endif %} image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:master') }} imagePullPolicy: Always env: @@ -40,7 +55,7 @@ spec: value: "{{ prometheus.es_parallel | default(false) }}" - name: prom_token value: "{{ prometheus.prom_token | default() }}" - - name: prom_url + - name: prom_url value: "{{ prometheus.prom_url | default() }}" {% endif %} command: ["/bin/sh", "-c"] @@ -76,12 +91,15 @@ spec: export db_mssql_bucket={{workload_args.db_mssql_bucket}}; export db_mssql_durability={{workload_args.db_mssql_durability}}; export db_mssql_checkpoint={{workload_args.db_mssql_checkpoint}}; - cd /hammer; - run_snafu --tool hammerdb -u {{ uuid }} \ -{% if workload_args.debug is defined and workload_args.debug %} - -v \ +{% if workload_args.es_custom_field is sameas true %} + export es_ocp_version={{workload_args.es_ocp_version}}; + export es_cnv_version={{workload_args.es_cnv_version}}; + export es_db_version={{workload_args.es_db_version}}; + export es_os_version={{workload_args.es_os_version}}; + export es_kind={{workload_args.es_kind}}; {% endif %} - ;" + cd /hammer; + run_snafu --tool hammerdb -u {{ uuid }}" volumeMounts: - name: hammerdb-workload-volume mountPath: "/workload" diff --git a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 new file mode 100644 index 000000000..7ac6924ec --- /dev/null +++ b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 @@ -0,0 +1,66 @@ +{% if elasticsearch is defined %} +{% if elasticsearch.url is defined %} +export es={{ elasticsearch.url }}; +export es_index={{ elasticsearch.index_name | default("ripsaw-hammerdb") }}; +export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; +export parallel={{ elasticsearch.parallel | default("false") }}; +export uuid={{uuid}}; +{% endif %} +{% endif %} +{% if workload_args.client_vm.network.multiqueue.enabled %} +dnf install -y ethtool; +ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; +{% endif %} +dnf install -y epel-release git python3-pip; +pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper; +export db_type={{workload_args.db_type}}; +export timed_test={{workload_args.timed_test}}; +export db_server={{workload_args.db_server}}; +export db_port={{workload_args.db_port}}; +export db_warehouses={{workload_args.db_warehouses}}; +export db_num_workers={{workload_args.db_num_workers}}; +export db_user={{workload_args.db_user}}; +export transactions={{workload_args.transactions}}; +export raiseerror={{workload_args.raiseerror}}; +export keyandthink={{workload_args.keyandthink}}; +export db_driver={{workload_args.driver}}; +export rampup={{workload_args.rampup}}; +export runtime={{workload_args.runtime}}; +export allwarehouse={{workload_args.allwarehouse}}; +export timeprofile={{workload_args.timeprofile}}; +export async_scale={{workload_args.async_scale}}; +export async_client={{workload_args.async_client}}; +export async_verbose={{workload_args.async_verbose}}; +export async_delay={{workload_args.async_delay}}; +export samples={{workload_args.samples}}; +export db_mssql_tcp={{workload_args.db_mssql_tcp}}; +export db_mssql_azure={{workload_args.db_mssql_azure}}; +export db_mssql_authentication={{workload_args.db_mssql_authentication}}; +export db_mssql_linux_authent={{workload_args.db_mssql_linux_authent}}; +export db_mssql_odbc_driver='{{workload_args.db_mssql_odbc_driver}}'; +export db_mssql_linux_odbc='{{workload_args.db_mssql_linux_odbc}}'; +export db_mssql_imdb={{workload_args.db_mssql_imdb}}; +export db_mssql_bucket={{workload_args.db_mssql_bucket}}; +export db_mssql_durability={{workload_args.db_mssql_durability}}; +export db_mssql_checkpoint={{workload_args.db_mssql_checkpoint}}; +{% if workload_args.es_custom_field is sameas true %} +export es_ocp_version={{workload_args.es_ocp_version}}; +export es_cnv_version={{workload_args.es_cnv_version}}; +export es_db_version={{workload_args.es_db_version}}; +export es_os_version={{workload_args.es_os_version}}; +export es_kind={{workload_args.es_kind}}; +{% endif %} +export HOME=/root; +{% if workload_args.client_vm.storageclass is defined + or workload_args.client_vm.hostpath is defined %} +systemctl stop mssql-server +sudo ACCEPT_EULA='Y' MSSQL_PID='Enterprise' MSSQL_SA_PASSWORD='s3curePasswordString' /opt/mssql/bin/mssql-conf setup +sudo firewall-cmd --zone=public --add-port=1433/tcp --permanent +sudo firewall-cmd --reload +systemctl start mssql-server +{% endif %} +cd /hammer; +./hammerdbcli auto /creator/createdb.tcl; +redis-cli set db-creation-{{trunc_uuid}} true; +run_snafu --tool hammerdb -u {{ uuid }}; +redis-cli set complete true; diff --git a/roles/hammerdb/templates/db_postgres_workload.yml.j2 b/roles/hammerdb/templates/db_postgres_workload.yml.j2 index 4622808d8..f4ab3e90a 100644 --- a/roles/hammerdb/templates/db_postgres_workload.yml.j2 +++ b/roles/hammerdb/templates/db_postgres_workload.yml.j2 @@ -10,12 +10,27 @@ spec: metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-operator-uuid: {{ uuid }} spec: +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} containers: - name: hammerdb +{% if workload_args.resources is sameas true %} + resources: + requests: + cpu: {{ workload_args.requests_cpu }} + memory: {{ workload_args.requests_memory }} + limits: + cpu: {{ workload_args.limits_cpu }} + memory: {{ workload_args.limits_memory }} +{% endif %} image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:master') }} imagePullPolicy: Always env: @@ -44,8 +59,8 @@ spec: value: "{{ prometheus.prom_url | default() }}" {% endif %} command: ["/bin/sh", "-c"] - args: - - "/usr/local/bin/uid_entrypoint; + args: + - "/usr/local/bin/uid_entrypoint; export db_type={{workload_args.db_type}}; export timed_test={{workload_args.timed_test}}; export db_server={{workload_args.db_server}}; @@ -74,12 +89,15 @@ spec: export db_postgresql_dritasnap={{workload_args.db_postgresql_dritasnap}}; export db_postgresql_oracompat={{workload_args.db_postgresql_oracompat}}; export db_postgresql_storedprocs={{workload_args.db_postgresql_storedprocs}}; - cd /hammer; - run_snafu --tool hammerdb -u {{ uuid }} \ -{% if workload_args.debug is defined and workload_args.debug %} - -v \ +{% if workload_args.es_custom_field is sameas true %} + export es_ocp_version={{workload_args.es_ocp_version}}; + export es_cnv_version={{workload_args.es_cnv_version}}; + export es_db_version={{workload_args.es_db_version}}; + export es_os_version={{workload_args.es_os_version}}; + export es_kind={{workload_args.es_kind}}; {% endif %} - ;" + cd /hammer; + run_snafu --tool hammerdb -u {{ uuid }}" volumeMounts: - name: hammerdb-workload-volume mountPath: "/workload" diff --git a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 new file mode 100644 index 000000000..5571f9c91 --- /dev/null +++ b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 @@ -0,0 +1,56 @@ +{% if elasticsearch is defined %} +{% if elasticsearch.url is defined %} +export es={{ elasticsearch.url }}; +export es_index={{ elasticsearch.index_name | default("ripsaw-hammerdb") }}; +export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; +export parallel={{ elasticsearch.parallel | default("false") }}; +export uuid={{uuid}}; +{% endif %} +{% endif %} +{% if workload_args.client_vm.network.multiqueue.enabled %} +dnf install -y ethtool; +ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; +{% endif %} +dnf install -y epel-release git python3-pip; +pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper; +export db_type={{workload_args.db_type}}; +export timed_test={{workload_args.timed_test}}; +export db_server={{workload_args.db_server}}; +export db_port={{workload_args.db_port}}; +export db_warehouses={{workload_args.db_warehouses}}; +export db_num_workers={{workload_args.db_num_workers}}; +export db_user={{workload_args.db_user}}; +export db_name={{workload_args.db_name}}; +export transactions={{workload_args.transactions}}; +export raiseerror={{workload_args.raiseerror}}; +export keyandthink={{workload_args.keyandthink}}; +export driver={{workload_args.driver}}; +export rampup={{workload_args.rampup}}; +export runtime={{workload_args.runtime}}; +export allwarehouse={{workload_args.allwarehouse}}; +export timeprofile={{workload_args.timeprofile}}; +export async_scale={{workload_args.async_scale}}; +export async_client={{workload_args.async_client}}; +export async_verbose={{workload_args.async_verbose}}; +export async_delay={{workload_args.async_delay}}; +export samples={{workload_args.samples}}; +export db_postgresql_defaultdbase={{workload_args.db_postgresql_defaultdbase}}; +export db_postgresql_superuser={{workload_args.db_postgresql_superuser}}; +export db_postgresql_superuser_pass={{workload_args.db_postgresql_superuser_pass}}; +export db_postgresql_vacuum={{workload_args.db_postgresql_vacuum}}; +export db_postgresql_dritasnap={{workload_args.db_postgresql_dritasnap}}; +export db_postgresql_oracompat={{workload_args.db_postgresql_oracompat}}; +export db_postgresql_storedprocs={{workload_args.db_postgresql_storedprocs}}; +{% if workload_args.es_custom_field is sameas true %} +export es_ocp_version={{workload_args.es_ocp_version}}; +export es_cnv_version={{workload_args.es_cnv_version}}; +export es_db_version={{workload_args.es_db_version}}; +export es_os_version={{workload_args.es_os_version}}; +export es_kind={{workload_args.es_kind}}; +{% endif %} +export HOME=/root; +cd /hammer; +./hammerdbcli auto /creator/createdb.tcl; +redis-cli set db-creation-{{trunc_uuid}} true; +run_snafu --tool hammerdb -u {{ uuid }}; +redis-cli set complete true; diff --git a/roles/hammerdb/templates/tpcc-workload-mysql.tcl.j2 b/roles/hammerdb/templates/tpcc-workload-mariadb.tcl.j2 similarity index 97% rename from roles/hammerdb/templates/tpcc-workload-mysql.tcl.j2 rename to roles/hammerdb/templates/tpcc-workload-mariadb.tcl.j2 index 2dfae4066..4b1f0d15c 100644 --- a/roles/hammerdb/templates/tpcc-workload-mysql.tcl.j2 +++ b/roles/hammerdb/templates/tpcc-workload-mariadb.tcl.j2 @@ -43,7 +43,7 @@ diset tpcc mysql_async_scale {{workload_args.async_scale}} diset tpcc mysql_async_client {{workload_args.async_client}} diset tpcc mysql_async_verbose {{workload_args.async_verbose}} diset tpcc mysql_async_delay {{workload_args.async_delay}} - +diset connection mysql_socket {{workload_args.db_mysql_socket}} vuset logtotemp 1 loadscript diff --git a/roles/stressng/templates/stressng_workload.yml.j2 b/roles/stressng/templates/stressng_workload.yml.j2 index 4686d3dbf..2aaf3911b 100644 --- a/roles/stressng/templates/stressng_workload.yml.j2 +++ b/roles/stressng/templates/stressng_workload.yml.j2 @@ -74,7 +74,7 @@ spec: value: "{{ prometheus.prom_url | default() }}" {% endif %} command: ["/bin/sh", "-c"] - args: + args: - run_snafu --tool stressng -j /workload/jobfile -u {{ uuid }} {% if workload_args.debug is defined and workload_args.debug %} -v @@ -91,4 +91,3 @@ spec: defaultMode: 0660 restartPolicy: OnFailure {% include "metadata.yml.j2" %} - diff --git a/tests/mssql.yaml b/tests/mssql.yaml index e83de4e39..1649d7fad 100644 --- a/tests/mssql.yaml +++ b/tests/mssql.yaml @@ -11,7 +11,7 @@ metadata: namespace: sql-server spec: replicas: 1 - selector: + selector: matchLabels: app: mssql template: @@ -41,4 +41,3 @@ spec: - protocol: TCP port: 1433 targetPort: 1433 - diff --git a/tests/test_crs/valid_hammerdb.yaml b/tests/test_crs/valid_hammerdb.yaml index dff12f4a5..5fd145989 100644 --- a/tests/test_crs/valid_hammerdb.yaml +++ b/tests/test_crs/valid_hammerdb.yaml @@ -15,10 +15,11 @@ spec: index_name: ripsaw-hammerdb metadata: collection: true + cleanup: false workload: name: "hammerdb" args: - # image: "quay.io/mkarg/hammerdb:latest" + # image: quay.io/test/hammerdb:latest db_type: "mssql" timed_test: true test_type: "tpc-c" @@ -27,7 +28,7 @@ spec: db_server: "mssql-deployment.sql-server" db_port: "1433" db_warehouses: 1 - db_num_workers: 1 + db_num_workers: 1 db_user: "SA" db_pass: "s3curePasswordString" db_name: "tpcc" @@ -67,4 +68,4 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" - debug: true + debug: true \ No newline at end of file From 7e9abacf06d80159c368ec5d8cb4408d7c4017f0 Mon Sep 17 00:00:00 2001 From: Ben England Date: Thu, 10 Jun 2021 16:14:09 -0400 Subject: [PATCH 081/249] give benchmark-operator privs to look into openshift-storage namespace --- .../ocs-cache-drop-clusterrole.yaml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 roles/ceph_osd_cache_drop/ocs-cache-drop-clusterrole.yaml diff --git a/roles/ceph_osd_cache_drop/ocs-cache-drop-clusterrole.yaml b/roles/ceph_osd_cache_drop/ocs-cache-drop-clusterrole.yaml new file mode 100644 index 000000000..4d61914ec --- /dev/null +++ b/roles/ceph_osd_cache_drop/ocs-cache-drop-clusterrole.yaml @@ -0,0 +1,33 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: benchmark-operator-openshift-storage +rules: +- apiGroups: + - '' + resources: + - nodes + - pods + verbs: + - get + - list +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: benchmark-operator-openshift-storage +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: benchmark-operator-openshift-storage +subjects: +- kind: ServiceAccount + name: benchmark-operator + namespace: openshift-storage From 54dfffddc94913e213a63b67f98b27d4fe2123bc Mon Sep 17 00:00:00 2001 From: Ben England Date: Thu, 10 Jun 2021 16:20:38 -0400 Subject: [PATCH 082/249] add RBAC to let benchmark-operator detect openshift-storage pods --- docs/cache_dropping.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/cache_dropping.md b/docs/cache_dropping.md index 9d2836c4c..d36afe1d5 100644 --- a/docs/cache_dropping.md +++ b/docs/cache_dropping.md @@ -58,22 +58,36 @@ Benchmarks supported for kernel cache dropping at present are: ## how to drop Ceph OSD cache -for this to work with OpenShift Container Storage, you must start both the Ceph toolbox pod in OCS -and the cache dropper pod. You can do this with: +for this to work with OpenShift Container Storage, you must do these steps once the benchmark-operator is running: +and the cache dropper pod, and enable benchmark-operator to see into the openshift-storage namespace. +You can do this with: ``` + +# enable the benchmark operator to look for pods in the openshift-storage namespace +oc create -f roles/ceph_osd_cache_drop/ocs-cache-drop-clusterrole.yaml + +# start the Ceph toolbox pod in openshift-storage namespace oc patch OCSInitialization ocsinit -n openshift-storage --type json --patch \ '[{ "op": "replace", "path": "/spec/enableCephTools", "value": true }]' +# start the OSD cache dropper pod in openshift-storage namespace oc create -f roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml +# repeat until you see if the 2 pods are both running oc -n openshift-storage get pod | awk '/tool/||/drop/' + ``` when you see both of these pods in the running state, then you can use benchmark operator. The reason that you have to manually start these two pods running is that the benchmark-operator does not have authorization to run them in the openshift-storage namespace and get access to the secrets needed to do this. +Benchmarks supported for Ceph OSD cache dropping are: + +- fio +- smallfile + # implementation notes For benchmark developers... From c7576b9e870d6061cec419267571fb5d313ca96a Mon Sep 17 00:00:00 2001 From: Ben England Date: Fri, 11 Jun 2021 11:12:16 -0400 Subject: [PATCH 083/249] nit --- docs/cache_dropping.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cache_dropping.md b/docs/cache_dropping.md index d36afe1d5..e95093552 100644 --- a/docs/cache_dropping.md +++ b/docs/cache_dropping.md @@ -58,7 +58,7 @@ Benchmarks supported for kernel cache dropping at present are: ## how to drop Ceph OSD cache -for this to work with OpenShift Container Storage, you must do these steps once the benchmark-operator is running: +For this to work with OpenShift Container Storage, you must do these steps once the benchmark-operator is running: and the cache dropper pod, and enable benchmark-operator to see into the openshift-storage namespace. You can do this with: From dc2640cf4424b3c76fa8d7320bab58201bf742be Mon Sep 17 00:00:00 2001 From: ebattat <73884315+ebattat@users.noreply.github.com> Date: Wed, 16 Jun 2021 14:33:45 +0300 Subject: [PATCH 084/249] add stressng vm elasticsearch/nodeselector support (#590) --- .../crds/ripsaw_v1alpha1_stressng_vm.yaml | 5 ++--- .../templates/stressng_workload.yml.j2 | 13 ++++++++++++ .../templates/stressng_workload_vm.yml.j2 | 20 ++++++++++++------- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/resources/crds/ripsaw_v1alpha1_stressng_vm.yaml b/resources/crds/ripsaw_v1alpha1_stressng_vm.yaml index 1969bd7ae..d2442e8e5 100644 --- a/resources/crds/ripsaw_v1alpha1_stressng_vm.yaml +++ b/resources/crds/ripsaw_v1alpha1_stressng_vm.yaml @@ -15,9 +15,8 @@ spec: runtype: "parallel" timeout: "30" instances: 1 - # nodeselector: - # key1: value1 - # key2: value2 + pin: false # true for nodeSelector + pin_client: "node1" # cpu stressor options cpu_stressors: "1" cpu_percentage: "100" diff --git a/roles/stressng/templates/stressng_workload.yml.j2 b/roles/stressng/templates/stressng_workload.yml.j2 index 2aaf3911b..53e8e95de 100644 --- a/roles/stressng/templates/stressng_workload.yml.j2 +++ b/roles/stressng/templates/stressng_workload.yml.j2 @@ -12,6 +12,10 @@ spec: labels: app: stressng_workload-{{ trunc_uuid }} spec: +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} @@ -20,6 +24,15 @@ spec: {% endif %} containers: - name: stressng +{% if workload_args.resources is sameas true %} + resources: + requests: + cpu: {{ workload_args.requests_cpu }} + memory: {{ workload_args.requests_memory }} + limits: + cpu: {{ workload_args.limits_cpu }} + memory: {{ workload_args.limits_memory }} +{% endif %} image: {{ workload_args.image | default('quay.io/cloud-bulldozer/stressng:latest') }} imagePullPolicy: Always env: diff --git a/roles/stressng/templates/stressng_workload_vm.yml.j2 b/roles/stressng/templates/stressng_workload_vm.yml.j2 index 1fb6bb4ac..5618c70eb 100644 --- a/roles/stressng/templates/stressng_workload_vm.yml.j2 +++ b/roles/stressng/templates/stressng_workload_vm.yml.j2 @@ -60,17 +60,23 @@ spec: - "mkdir /tmp/stressng-test/jobfile" - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240D | cut -f1 -d' ') /tmp/stressng-test" runcmd: +{% if elasticsearch is defined %} +{% if elasticsearch.url is defined %} + - export es={{ elasticsearch.url }}; + - export es_index={{ elasticsearch.index_name | default("ripsaw-stressng") }}; + - export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; + - export parallel={{ elasticsearch.parallel | default("false") }}; + - export uuid={{uuid}}; +{% endif %} +{% endif %} {% if workload_args.client_vm.network.multiqueue.enabled %} - dnf install -y ethtool - ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }} {% endif %} - - dnf install -y stress-ng redis git - - pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper - - run_snafu --tool stressng -j /tmp/stressng-test/jobfile -u {{ uuid }} \ -{% if workload_args.debug is defined and workload_args.debug %} - -v \ -{% endif %} - ; + - dnf install -y python3-pip + - dnf install -y git redis stress-ng + - pip install git+https://github.com/cloud-bulldozer/benchmark-wrapper + - run_snafu --tool stressng -j /tmp/stressng-test/jobfile -u {{ uuid }} name: cloudinitdisk - configMap: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" From 97de39eb715b96a203a04d43d74361bb661d85fa Mon Sep 17 00:00:00 2001 From: Mohit Sheth Date: Thu, 1 Jul 2021 21:03:14 +0000 Subject: [PATCH 085/249] fix svc name to not include period --- roles/uperf/templates/service.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 89ae4b71a..d3e1305ac 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -7,7 +7,7 @@ items: - kind: Service apiVersion: v1 metadata: - name: uperf-service-{{ worker_node_list[ node_idx_item ] | truncate(27,true,'') }}-{{ item }}-{{ trunc_uuid }} + name: uperf-service-{{ item }}-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' labels: app: uperf-bench-server-{{ worker_node_list[ node_idx_item ] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} From 2f6a621c37b6f465709226fd9463e114a585aecf Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 5 Jul 2021 14:30:02 +0200 Subject: [PATCH 086/249] Update namespace count query Signed-off-by: Raul Sevilla --- roles/kube-burner/files/metrics-aggregated.yaml | 2 +- roles/kube-burner/files/metrics.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/kube-burner/files/metrics-aggregated.yaml b/roles/kube-burner/files/metrics-aggregated.yaml index e5a6d4e0a..fb67eb574 100644 --- a/roles/kube-burner/files/metrics-aggregated.yaml +++ b/roles/kube-burner/files/metrics-aggregated.yaml @@ -140,7 +140,7 @@ metrics: metricName: P99APIEtcdRequestLatency # Cluster metrics - - query: count(kube_namespace_created) + - query: sum(kube_namespace_status_phase) by (phase) > 0 metricName: namespaceCount - query: sum(kube_pod_status_phase{}) by (phase) diff --git a/roles/kube-burner/files/metrics.yaml b/roles/kube-burner/files/metrics.yaml index bdad2be8f..306f0c0c0 100644 --- a/roles/kube-burner/files/metrics.yaml +++ b/roles/kube-burner/files/metrics.yaml @@ -87,7 +87,7 @@ metrics: instant: true # Cluster metrics - - query: count(kube_namespace_created) + - query: sum(kube_namespace_status_phase) by (phase) > 0 metricName: namespaceCount - query: sum(kube_pod_status_phase{}) by (phase) From de9fa9097777a9c61a7f7f5e8619d9a53f962740 Mon Sep 17 00:00:00 2001 From: ebattat <73884315+ebattat@users.noreply.github.com> Date: Wed, 7 Jul 2021 21:26:08 +0300 Subject: [PATCH 087/249] Hammerdb (#596) * update changes to support operator-sdk version 1.5.0 * add PVC Hostpath and local support * fix conflict * upload database images container disk into quay.io cloud-bulldozer * delete unnecessary block * update memory limit to 16GB and cpu to 4 * update remark * update comments * fix es fields * update hammerdb documentation in hammerdb.md * update hammerdb documentation in hammerdb.md * add documentations for Hostpath Local and PVC on Pod and VM * add documentations for Hostpath Local and PVC on Pod and VM * add documentations for Hostpath Local and PVC on Pod and VM * add documentations for Hostpath Local and PVC on Pod and VM --- docs/hammerdb.md | 178 ++++++++++++++++-- .../ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml | 8 +- ...pha1_hammerdb_mariadb_server_hostpath.yaml | 123 ++++++++++++ ...1alpha1_hammerdb_mariadb_server_local.yaml | 133 +++++++++++++ ..._v1alpha1_hammerdb_mariadb_server_pvc.yaml | 32 ++-- .../ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml | 36 ++-- .../ripsaw_v1alpha1_hammerdb_mssql_cr.yaml | 8 +- ...ripsaw_v1alpha1_hammerdb_mssql_server.yaml | 1 - ...alpha1_hammerdb_mssql_server_hostpath.yaml | 96 ++++++++++ ..._v1alpha1_hammerdb_mssql_server_local.yaml | 105 +++++++++++ ...aw_v1alpha1_hammerdb_mssql_server_pvc.yaml | 30 +-- .../ripsaw_v1alpha1_hammerdb_mssql_vm.yaml | 32 ++-- .../ripsaw_v1alpha1_hammerdb_postgres_cr.yaml | 8 +- ...saw_v1alpha1_hammerdb_postgres_server.yaml | 4 +- ...ha1_hammerdb_postgres_server_hostpath.yaml | 117 ++++++++++++ ...alpha1_hammerdb_postgres_server_local.yaml | 126 +++++++++++++ ...v1alpha1_hammerdb_postgres_server_pvc.yaml | 35 ++-- .../ripsaw_v1alpha1_hammerdb_postgres_vm.yaml | 32 ++-- .../ripsaw_v1alpha1_hammerdb_cr.yaml | 8 +- .../ripsaw_v1alpha1_hammerdb_vm.yaml | 24 ++- roles/hammerdb/defaults/main.yml | 2 + roles/hammerdb/tasks/main.yaml | 37 ++-- roles/hammerdb/templates/db_creation.yml | 2 +- .../templates/db_creation_mariadb_vm.yml | 18 +- .../templates/db_creation_mssql_vm.yml | 18 +- .../templates/db_creation_postgres_vm.yml | 20 +- .../templates/db_mariadb_workload.yml.j2 | 6 +- .../templates/db_mariadb_workload_vm.sh.j2 | 4 +- .../templates/db_mssql_workload.yml.j2 | 6 +- .../templates/db_mssql_workload_vm.sh.j2 | 14 +- .../templates/db_postgres_workload.yml.j2 | 6 +- .../templates/db_postgres_workload_vm.sh.j2 | 15 ++ tests/test_crs/valid_hammerdb.yaml | 7 +- 33 files changed, 1095 insertions(+), 196 deletions(-) create mode 100644 resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_hostpath.yaml create mode 100644 resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_local.yaml create mode 100644 resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_hostpath.yaml create mode 100644 resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_local.yaml create mode 100644 resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_hostpath.yaml create mode 100644 resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_local.yaml create mode 100644 roles/hammerdb/defaults/main.yml diff --git a/docs/hammerdb.md b/docs/hammerdb.md index d067edc74..04322ada7 100644 --- a/docs/hammerdb.md +++ b/docs/hammerdb.md @@ -15,38 +15,137 @@ An example CR might look like this apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: hammerdb + name: hammerdb-benchmark namespace: my-ripsaw spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false workload: - name: "hammerdb" + name: hammerdb args: - db_init: false + #image: quay.io/user/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_client: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 4 + limits_memory: 16Gi + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema db_benchmark: true - db_server: "mssql-deployment.sql-server" - db_port: "1443" - db_tcp: "true" - db_user: "SA" - db_pass: "s3curePasswordString" + db_server: "mssql-deployment.mssql-db" + db_port: "1433" db_warehouses: 1 db_num_workers: 1 - transactions: 20000 - runtime: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 10000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # ElasticSearch custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "2019" + es_os_version: "centos8" + es_kind: "pod" ``` +The `pin` feature is `true` for node selector, `pin_node` is the node name + +The `resource` feature is `true` for resources configurations: `requests_cpu`, `requests_memory`, `limits_cpu`, `limits_memory` + +The `db_type` is the database type: mariadb, pg or mssql + +The `db_init` feature determines wether the database has already been initialized `false` or needs to be initialized `true`. If the DB has been used previously to run benchmarks against it, it needs to be set to `false`. + +The `db_benchmark` feature used to run the actual benchmark when set to `true`. -The `db_init` feature determines wether the database has already been initialized (false) or needs to be initialized (true). If the DB has been used previously to run benchmarks against it, it needs to be set to `false`. +The `db_server` either holds the name or the IP address of the DB server, -The `db_benchmark` feature is used to run the actual benchmark when set to true. `db_server` either holds the name or the IP address of the DB server, `db_port` the port on which the DB is accessible. If `db_tcp` is set to true the client will use a TCP connection, if it's set to `false` UDP will be used. +The `db_port` the port on which the DB is accessible. If `db_mssql_tcp` is set to `true` the client will use a TCP connection, if it's set to `false` UDP will be used. -`db_user` and `db_pass` need to be set identical to the settings on the DB server side. +The `db_user` and `db_pass` need to be set identical to the settings on the DB server side. -The tpcc benchmark which we use can set up an arbitrary number of warehouses between which goods will be transferred in order to simulate a real-world scenario. The higher the number of warehouses is, the more complex and load-heavy the benchmark can get. `db_warehouses` is used to define this number. -`db_num_workers` controls the number of virtual users, acting upon the warehouses and the goods in them. This number needs to lesser or equal to the number of warehouses. +The tpcc benchmark which we use can set up an arbitrary number of warehouses between which goods will be transferred in order to simulate a real-world scenario. The higher the number of warehouses is, the more complex and load-heavy the benchmark can get. +The `db_warehouses` is used to define this number. +The `db_num_workers` is used to controls the number of virtual users, acting upon the warehouses and the goods in them. This number needs to lesser or equal to the number of warehouses. +* db_warehouses >= db_num_workers: virtual users must be less than or equal to number of warehouses With `runtime`, `rampup` and `samples` the time for a single run, the rampup time per run and the number of runs can be controlled. +The `es_custom_field` feature is `true` to enable distribute the following fields to Elastic Search: `es_ocp_version`, `es_cnv_version`, `es_db_version`, `es_os_version`, `es_kind` + +There are several options to store database data on Pod, the default one is ephemeral: + +HostPath: The data will be stored on a local disk where the OS is placed. + +Local: The data will be stored on local disk on separate disk, separate disk from OS. + +PVC: The data will be stored on Container Storage, it's required a Pre-installed Container Storage + +MSSQL examples: + +[MSSQL ephemeral](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml), +[MSSQL HostPath](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_hostpath.yaml), +[MSSQL Local](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_local.yaml), +[MSSQL PVC](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml) + +Postgres examples: + +[Postgres ephemeral](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml), +[Postgres HostPath](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_hostpath.yaml), +[Postgres Local](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_local.yaml), +[Postgres PVC](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml) + +Mariadb examples: + +[Mariadb ephemeral](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server.yaml), +[Mariadb HostPath](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_hostpath.yaml), +[Mariadb Local](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_local.yaml), +[Mariadb PVC](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml) + + The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. @@ -70,10 +169,10 @@ Once done creating/editing the resource file, you can run it by: cores: 2 threads: 1 image: kubevirt/fedora-cloud-container-disk-demo:latest - limits: - memory: 4Gi requests: - memory: 4Gi + memory: 100Mi + limits: + memory: 16Gi # at least 16Gi for VM network: front_end: bridge # or masquerade multiqueue: @@ -82,9 +181,49 @@ Once done creating/editing the resource file, you can run it by: extra_options: - none #- hostpassthrough +- ## OCS PVC + pvc: false # enable for OCS PVC + pvc_storageclass: ocs-storagecluster-ceph-rbd + pvc_pvcaccessmode: ReadWriteMany + pvc_pvcvolumemode: Block + pvc_storagesize: 10Gi + ## HostPath - Configuring SELinux on cluster workers + hostpath: false # enable for hostPath + hostpath_path: /var/tmp/disk.img + hostpath_storagesize: 10Gi ``` The above is the additional changes required to run hammerdb in vms. + +There several options to store database data on VM, the default one is ephemeral: + +PVC: The data will be stored on Container Storage, it's required a Pre-installed Container Storage + +The `pvc` feature is `true` for enabling container storage PVC on VM, +there several parameters that must be configured: + +`pvc_storageclass` for pvc storage class (`kubectl get sc`) + +`pvc_pvcaccessmode` can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce + +`pvc_pvcvolumemode` can be one of Filesystem,Block Default: Filesystem + +`pvc_storagesize` the PVC storage size + +HostPath: The data will be stored on the local disk where the OS is placed. + +The `hostpath` feature is `true` for enabling HostPath on VM, there several parameters that must be configured: + +`hostpath_path` The image path to hold the hostPath + +`hostpath_storagesize` the HostPath storage size + +examples: +[MSSQL](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml), +[Postgres](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml), +[Mariadb](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml), + + Currently, we only support images that can be used as [containerDisk](https://docs.openshift.com/container-platform/4.6/virt/virtual_machines/virtual_disks/virt-using-container-disks-with-vms.html#virt-preparing-container-disk-for-vms_virt-using-container-disks-with-vms). You can easily make your own container-disk-image as follows by downloading your qcow2 image of choice. @@ -101,4 +240,5 @@ podman push ``` You can either access results by indexing them directly or by accessing the console. -The results are stored in /tmp/ directory \ No newline at end of file +The results are stored in /tmp/ directory +You can console into VM by running `virtctl console vmi_name` \ No newline at end of file diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml index 0b5c9b09b..2a7f0547e 100644 --- a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml +++ b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml @@ -12,9 +12,9 @@ spec: workload: name: hammerdb args: - #image: quay.io/test/hammerdb:latest # add custom hammerdb image + #image: quay.io/user/hammerdb:latest # add custom hammerdb image pin: false # true for nodeSelector - pin_client: "node1" + pin_node: "node1" resources: false # true for resources requests/limits requests_cpu: 200m requests_memory: 100Mi @@ -32,7 +32,7 @@ spec: db_user: "root" db_pass: "mysql" db_name: "tpcc" - transactions: 500000 + transactions: 10000 raiseerror: "false" keyandthink: "false" driver: "timed" @@ -69,7 +69,7 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" - # es custom fields + # ElasticSearch custom fields es_custom_field: false es_ocp_version: "4.7.0" es_cnv_version: "2.6.2" diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_hostpath.yaml b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_hostpath.yaml new file mode 100644 index 000000000..71dded4cf --- /dev/null +++ b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_hostpath.yaml @@ -0,0 +1,123 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mariadb-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-custom-config + namespace: mariadb-db +data: + custom.conf: | + # This group is read both both by the client and the server + # use it for options that affect everything + # + [client-server] + # + # include all files from the config directory + # + !includedir /etc/my.cnf.d + port=3306 + ssl=0 + [mysqld] + innodb_file_per_table=1 + innodb_flush_method=O_DIRECT + innodb_adaptive_flushing=1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + matchLabels: + app: mariadb + template: + metadata: + labels: + app: mariadb + type: mariadb-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "worker-0" + terminationGracePeriodSeconds: 10 + containers: + - name: mariadb + image: centos/mariadb-103-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 3306 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: MYSQL_USER + value: "test" + - name: MYSQL_PASSWORD + value: "test" + - name: MYSQL_ROOT_PASSWORD + value: "mysql" + securityContext: + privileged: true + volumeMounts: + - name: mariadb-custom-config + mountPath: /etc/my.cnf + subPath: custom.conf #should be the name used in the ConfigMap + - name: maria-persistent-storage + mountPath: /var/lib/mysql + readOnly: false + volumes: + - name: mariadb-custom-config + configMap: + name: mariadb-custom-config + - name: maria-persistent-storage + persistentVolumeClaim: + claimName: maria-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb + labels: + type: local +spec: + persistentVolumeReclaimPolicy: Delete + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /tmp +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: maria-persistent-storage + namespace: mariadb-db +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: manual +--- +apiVersion: v1 +kind: Service +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + app: mariadb + ports: + - protocol: TCP + port: 3306 + targetPort: 3306 diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_local.yaml b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_local.yaml new file mode 100644 index 000000000..794695bd6 --- /dev/null +++ b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_local.yaml @@ -0,0 +1,133 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mariadb-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-custom-config + namespace: mariadb-db +data: + custom.conf: | + # This group is read both both by the client and the server + # use it for options that affect everything + # + [client-server] + # + # include all files from the config directory + # + !includedir /etc/my.cnf.d + port=3306 + ssl=0 + [mysqld] + innodb_file_per_table=1 + innodb_flush_method=O_DIRECT + innodb_adaptive_flushing=1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + matchLabels: + app: mariadb + template: + metadata: + labels: + app: mariadb + type: mariadb-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "master-0" + terminationGracePeriodSeconds: 10 + containers: + - name: mariadb + image: centos/mariadb-103-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 3306 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: MYSQL_USER + value: "test" + - name: MYSQL_PASSWORD + value: "test" + - name: MYSQL_ROOT_PASSWORD + value: "mysql" + securityContext: + privileged: true + volumeMounts: + - name: mariadb-custom-config + mountPath: /etc/my.cnf + subPath: custom.conf #should be the name used in the ConfigMap + - name: maria-persistent-storage + mountPath: /var/lib/mysql + readOnly: false + volumes: + - name: mariadb-custom-config + configMap: + name: mariadb-custom-config + - name: maria-persistent-storage + persistentVolumeClaim: + claimName: maria-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: localblock + local: + # mkdir inside node + path: /var/lib/mysql + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master-0 +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: maria-persistent-storage + namespace: mariadb-db +spec: + storageClassName: localblock + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + app: mariadb + ports: + - protocol: TCP + port: 3306 + targetPort: 3306 +--- \ No newline at end of file diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml index 410550aaa..a5a633ab6 100644 --- a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml +++ b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml @@ -54,7 +54,7 @@ spec: cpu: 200m memory: 100Mi limits: - cpu: 14 + cpu: 4 memory: 16Gi env: - name: MYSQL_USER @@ -67,24 +67,29 @@ spec: - name: mariadb-custom-config mountPath: /etc/my.cnf subPath: custom.conf #should be the name used in the ConfigMap - - name: maria-persistent-storage + - name: mariadb-persistent-storage mountPath: /var/lib/mysql readOnly: false volumes: - name: mariadb-custom-config configMap: name: mariadb-custom-config - - name: maria-persistent-storage - volumeClaimTemplates: - - metadata: - name: mysql-ocs-data - spec: - storageClassName: ocs-storagecluster-ceph-rbd - accessModes: [ "ReadWriteOnce" ] - volumeMode: Filesystem - resources: - requests: - storage: 5Gi + - name: mariadb-persistent-storage + persistentVolumeClaim: + claimName: mariadb-persistent-storage +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mariadb-persistent-storage + namespace: mariadb-db +spec: + storageClassName: ocs-storagecluster-ceph-rbd + accessModes: [ "ReadWriteOnce" ] + volumeMode: Filesystem + resources: + requests: + storage: 10Gi --- apiVersion: v1 kind: Service @@ -98,3 +103,4 @@ spec: - protocol: TCP port: 3306 targetPort: 3306 +--- diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml index cad64ee45..5431ef132 100644 --- a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml +++ b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml @@ -12,13 +12,14 @@ spec: workload: name: hammerdb args: - #image: quay.io/test/hammerdb:latest # add custom hammerdb image - pin: false # true for nodeSelector - pin_client: "node1" + # image: "quay.io/test/hammerdb:latest" # add custom hammerdb image + pin: false # enable for nodeSelector + pin_node: "node1" db_type: "mariadb" timed_test: true test_type: "tpc-c" - db_init: true # true only for first run to build schema + # true only for first run to build schema + db_init: true db_benchmark: true db_server: "127.0.0.1" db_port: "3306" @@ -27,7 +28,7 @@ spec: db_user: "root" db_pass: "mysql" db_name: "tpcc" - transactions: 500000 + transactions: 10000 raiseerror: "false" keyandthink: "false" driver: "timed" @@ -64,7 +65,7 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" - # es custom fields + # ElasticSearch custom fields es_custom_field: false es_ocp_version: "4.7.0" es_cnv_version: "2.6.2" @@ -75,13 +76,13 @@ spec: client_vm: dedicatedcpuplacement: false sockets: 1 - cores: 4 + cores: 2 threads: 1 - image: quay.io/ebattat/centos8-mariadb10-3-container-disk:latest - limits: - memory: 16Gi + image: quay.io/cloud-bulldozer/centos8-mariadb103-container-disk:latest requests: memory: 100Mi + limits: + memory: 16Gi # at least 16Gi for VM network: front_end: bridge # or masquerade multiqueue: @@ -90,10 +91,15 @@ spec: extra_options: - none #- hostpassthrough - # PVC - #storageclass: ocs-storagecluster-ceph-rbd + ## OCS PVC + pvc: false # enable for OCS PVC + pvc_storageclass: ocs-storagecluster-ceph-rbd # Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce - #pvcaccessmode: ReadWriteMany + pvc_pvcaccessmode: ReadWriteMany # Can be one of Filesystem,Block Default: Filesystem - #pvcvolumemode: Block - #storagesize: 5Gi + pvc_pvcvolumemode: Block + pvc_storagesize: 10Gi + ## HostPath - Configuring SELinux on cluster workers + hostpath: false # enable for hostPath + hostpath_path: /var/tmp/disk.img + hostpath_storagesize: 10Gi diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml index 29b9cd245..34bdb856c 100644 --- a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml +++ b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml @@ -12,9 +12,9 @@ spec: workload: name: hammerdb args: - #image: quay.io/test/hammerdb:latest # add custom hammerdb image + #image: quay.io/user/hammerdb:latest # add custom hammerdb image pin: false # true for nodeSelector - pin_client: "node1" + pin_node: "node1" resources: false # true for resources requests/limits requests_cpu: 200m requests_memory: 100Mi @@ -32,7 +32,7 @@ spec: db_user: "SA" db_pass: "s3curePasswordString" db_name: "tpcc" - transactions: 500000 + transactions: 10000 raiseerror: "false" keyandthink: "false" driver: "timed" @@ -69,7 +69,7 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" - # es custom fields + # ElasticSearch custom fields es_custom_field: false es_ocp_version: "4.7.0" es_cnv_version: "2.6.2" diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml index aecc2e12f..d293a476c 100644 --- a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml +++ b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml @@ -1,4 +1,3 @@ ---- apiVersion: v1 kind: Namespace metadata: diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_hostpath.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_hostpath.yaml new file mode 100644 index 000000000..6f00b05af --- /dev/null +++ b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_hostpath.yaml @@ -0,0 +1,96 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mssql-db +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + replicas: 1 + selector: + matchLabels: + app: mssql + template: + metadata: + labels: + app: mssql + type: mssql-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "worker-0" + terminationGracePeriodSeconds: 10 + containers: + - name: mssql + image: mcr.microsoft.com/mssql/rhel/server:2019-latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 1433 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: MSSQL_PID + value: "Enterprise" + - name: ACCEPT_EULA + value: "Y" + - name: MSSQL_SA_PASSWORD + value: "s3curePasswordString" + securityContext: + privileged: true + volumeMounts: + - name: mssql-persistent-storage + mountPath: /var/opt/mssql + readOnly: false + volumes: + - name: mssql-persistent-storage + persistentVolumeClaim: + claimName: mssql-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb + labels: + type: local +spec: + persistentVolumeReclaimPolicy: Delete + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /tmp +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mssql-persistent-storage + namespace: mssql-db +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: manual +--- +apiVersion: v1 +kind: Service +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + selector: + app: mssql + ports: + - protocol: TCP + port: 1433 + targetPort: 1433 diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_local.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_local.yaml new file mode 100644 index 000000000..e147debbc --- /dev/null +++ b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_local.yaml @@ -0,0 +1,105 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mssql-db +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + replicas: 1 + selector: + matchLabels: + app: mssql + template: + metadata: + labels: + app: mssql + type: mssql-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "master-0" + terminationGracePeriodSeconds: 10 + containers: + - name: mssql + image: mcr.microsoft.com/mssql/rhel/server:2019-latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 1433 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: MSSQL_PID + value: "Enterprise" + - name: ACCEPT_EULA + value: "Y" + - name: MSSQL_SA_PASSWORD + value: "s3curePasswordString" + securityContext: + privileged: true + volumeMounts: + - name: mssql-persistent-storage + mountPath: /var/opt/mssql + readOnly: false + volumes: + - name: mssql-persistent-storage + persistentVolumeClaim: + claimName: mssql-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: localblock + local: + # mkdir inside node + path: /var/opt/mssql + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master-0 +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mssql-persistent-storage + namespace: mssql-db +spec: + storageClassName: localblock + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + selector: + app: mssql + ports: + - protocol: TCP + port: 1433 + targetPort: 1433 \ No newline at end of file diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml index e63a90435..828e885cf 100644 --- a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml +++ b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml @@ -1,4 +1,3 @@ ---- apiVersion: v1 kind: Namespace metadata: @@ -21,7 +20,7 @@ spec: type: mssql-database-server spec: nodeSelector: - kubernetes.io/hostname: "worker-0" + kubernetes.io/hostname: "worker-0" terminationGracePeriodSeconds: 10 containers: - name: mssql @@ -34,7 +33,7 @@ spec: cpu: 200m memory: 100Mi limits: - cpu: 6 + cpu: 4 memory: 16Gi env: - name: MSSQL_PID @@ -49,16 +48,21 @@ spec: readOnly: false volumes: - name: mssql-persistent-storage - volumeClaimTemplates: - - metadata: - name: mssql-persistent-storage - spec: - storageClassName: ocs-storagecluster-ceph-rbd - accessModes: [ "ReadWriteOnce" ] - volumeMode: Filesystem - resources: - requests: - storage: 5Gi + persistentVolumeClaim: + claimName: mssql-persistent-storage +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mssql-persistent-storage + namespace: mssql-db +spec: + storageClassName: ocs-storagecluster-ceph-rbd + accessModes: [ "ReadWriteOnce" ] + volumeMode: Filesystem + resources: + requests: + storage: 10Gi --- apiVersion: v1 kind: Service diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml index 90ad3382c..47609fb0b 100644 --- a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml +++ b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml @@ -12,9 +12,9 @@ spec: workload: name: hammerdb args: - #image: quay.io/test/hammerdb:latest # add custom hammerdb image + #image: quay.io/user/hammerdb:latest # add custom hammerdb image pin: false # true for nodeSelector - pin_client: "node1" + pin_node: "node1" db_type: "mssql" timed_test: true test_type: "tpc-c" @@ -27,7 +27,7 @@ spec: db_user: "SA" db_pass: "s3curePasswordString" db_name: "tpcc" - transactions: 500000 + transactions: 10000 raiseerror: "false" keyandthink: "false" driver: "timed" @@ -64,7 +64,7 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" - # es custom fields + # ElasticSearch custom fields es_custom_field: false es_ocp_version: "4.7.0" es_cnv_version: "2.6.2" @@ -75,13 +75,13 @@ spec: client_vm: dedicatedcpuplacement: false sockets: 1 - cores: 4 + cores: 2 threads: 1 - image: quay.io/ebattat/centos8-mssql2019-container-disk:latest - limits: - memory: 16Gi + image: quay.io/cloud-bulldozer/centos8-mssql2019-container-disk:latest requests: memory: 100Mi + limits: + memory: 16Gi # at least 16Gi for VM network: front_end: bridge # or masquerade multiqueue: @@ -90,10 +90,16 @@ spec: extra_options: - none #- hostpassthrough - # PVC - #storageclass: ocs-storagecluster-ceph-rbd + ## OCS PVC + pvc: false # enable for OCS PVC + pvc_storageclass: ocs-storagecluster-ceph-rbd # Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce - #pvcaccessmode: ReadWriteMany + pvc_pvcaccessmode: ReadWriteMany # Can be one of Filesystem,Block Default: Filesystem - #pvcvolumemode: Block - #storagesize: 5Gi + pvc_pvcvolumemode: Block + pvc_storagesize: 10Gi + ## HostPath - Configuring SELinux on cluster workers + hostpath: false # enable for hostPath + hostpath_path: /var/tmp/disk.img + hostpath_storagesize: 10Gi + diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml index 4e2b2ccf7..6a5f4bdbc 100644 --- a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml +++ b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml @@ -12,9 +12,9 @@ spec: workload: name: hammerdb args: - #image: # add custom hammerdb image + #image: quay.io/user/hammerdb:latest # add custom hammerdb image pin: false # true for nodeSelector - pin_client: "node1" + pin_node: "node1" resources: false # true for resources requests/limits requests_cpu: 200m requests_memory: 100Mi @@ -32,7 +32,7 @@ spec: db_user: "postgres" db_pass: "postgres" db_name: "tpcc" - transactions: 500000 + transactions: 10000 raiseerror: "false" keyandthink: "false" driver: "timed" @@ -69,7 +69,7 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" - # es custom fields + # ElasticSearch custom fields es_custom_field: false es_ocp_version: "4.7.0" es_cnv_version: "2.6.2" diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml index 888ac33a4..f996efec3 100644 --- a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml +++ b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml @@ -42,10 +42,10 @@ spec: ports: - containerPort: 5432 resources: - limits: + requests: cpu: 200m memory: 100Mi - requests: + limits: cpu: 4 memory: 16Gi env: diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_hostpath.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_hostpath.yaml new file mode 100644 index 000000000..15734c035 --- /dev/null +++ b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_hostpath.yaml @@ -0,0 +1,117 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: postgres-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-custom-config + namespace: postgres-db +data: + custom.pg_hba.conf: | + # TYPE DATABASE USER ADDRESS METHOD + # "local" is for Unix domain socket connections only + local all all trust + # IPv4 local connections: + host all all all trust +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + matchLabels: + app: postgres + replicas: 1 + template: + metadata: + labels: + app: postgres + type: postgres-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "master-0" + terminationGracePeriodSeconds: 10 + containers: + - name: postgres + image: centos/postgresql-10-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5432 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: POSTGRESQL_USER + value: "test" + - name: POSTGRESQL_PASSWORD + value: "test" + - name: POSTGRESQL_DATABASE + value: "test" + - name: POSTGRESQL_ADMIN_PASSWORD + value: "postgres" + securityContext: + privileged: true + volumeMounts: + - name: postgres-custom-config + mountPath: /var/lib/pgsql/data/pg_hba.conf + subPath: custom.pg_hba.conf #should be the name used in the ConfigMap + - name: postgres-persistent-storage + mountPath: /var/lib/pgsql/data + readOnly: false + volumes: + - name: postgres-custom-config + configMap: + name: postgres-custom-config + - name: postgres-persistent-storage + persistentVolumeClaim: + claimName: postgres-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb + labels: + type: local +spec: + persistentVolumeReclaimPolicy: Delete + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /tmp +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: postgres-persistent-storage + namespace: postgres-db +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: manual +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + app: postgres + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 \ No newline at end of file diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_local.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_local.yaml new file mode 100644 index 000000000..f0b15004c --- /dev/null +++ b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_local.yaml @@ -0,0 +1,126 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: postgres-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-custom-config + namespace: postgres-db +data: + custom.pg_hba.conf: | + # TYPE DATABASE USER ADDRESS METHOD + # "local" is for Unix domain socket connections only + local all all trust + # IPv4 local connections: + host all all all trust +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + matchLabels: + app: postgres + replicas: 1 + template: + metadata: + labels: + app: postgres + type: postgres-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "master-0" + terminationGracePeriodSeconds: 10 + containers: + - name: postgres + image: centos/postgresql-10-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5432 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: POSTGRESQL_USER + value: "test" + - name: POSTGRESQL_PASSWORD + value: "test" + - name: POSTGRESQL_DATABASE + value: "test" + - name: POSTGRESQL_ADMIN_PASSWORD + value: "postgres" + securityContext: + privileged: true + volumeMounts: + - name: postgres-custom-config + mountPath: /var/lib/pgsql/data/pg_hba.conf + subPath: custom.pg_hba.conf #should be the name used in the ConfigMap + - name: postgres-persistent-storage + mountPath: /var/lib/pgsql/data + readOnly: false + volumes: + - name: postgres-custom-config + configMap: + name: postgres-custom-config + - name: postgres-persistent-storage + persistentVolumeClaim: + claimName: postgres-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: localblock + local: + # mkdir inside node + path: /var/lib/pgsql/data + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master-0 +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: postgres-persistent-storage + namespace: postgres-db +spec: + storageClassName: localblock + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + app: postgres + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 \ No newline at end of file diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml index 93bdff7dc..1a215f4c4 100644 --- a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml +++ b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml @@ -42,11 +42,11 @@ spec: ports: - containerPort: 5432 resources: - limits: - cpu: 200m - memory: 100Mi requests: - cpu: 14 + cpu: 200m + memory: 100Mi + limits: + cpu: 4 memory: 16Gi env: - name: POSTGRESQL_USER @@ -69,16 +69,21 @@ spec: configMap: name: postgres-custom-config - name: postgres-persistent-storage - volumeClaimTemplates: - - metadata: - name: postgres-persistent-storage - spec: - storageClassName: ocs-storagecluster-ceph-rbd - accessModes: [ "ReadWriteOnce" ] - volumeMode: Filesystem - resources: - requests: - storage: 5Gi + persistentVolumeClaim: + claimName: postgres-persistent-storage +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: postgres-persistent-storage + namespace: postgres-db +spec: + storageClassName: ocs-storagecluster-ceph-rbd + accessModes: [ "ReadWriteOnce" ] + volumeMode: Filesystem + resources: + requests: + storage: 10Gi --- apiVersion: v1 kind: Service @@ -91,4 +96,4 @@ spec: ports: - protocol: TCP port: 5432 - targetPort: 5432 + targetPort: 5432 \ No newline at end of file diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml index d1686d774..f5366d850 100644 --- a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml +++ b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml @@ -12,9 +12,9 @@ spec: workload: name: hammerdb args: - #image: # add custom hammerdb image + #image: quay.io/user/hammerdb:latest # add custom hammerdb image pin: false # true for nodeSelector - pin_client: "node1" + pin_node: "node1" db_type: "pg" timed_test: true test_type: "tpc-c" @@ -64,7 +64,7 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" - # es custom fields + # ElasticSearch custom fields es_custom_field: false es_ocp_version: "4.7.0" es_cnv_version: "2.6.2" @@ -75,14 +75,13 @@ spec: client_vm: dedicatedcpuplacement: false sockets: 1 - cores: 4 + cores: 2 threads: 1 - #image: kubevirt/fedora-cloud-container-disk-demo:latest - image: quay.io/ebattat/centos8-postgres10-container-disk:latest - limits: - memory: 16Gi + image: quay.io/cloud-bulldozer/centos8-postgres10-container-disk:latest requests: - memory: 16Gi + memory: 100Mi + limits: + memory: 16Gi # at least 16Gi for VM network: front_end: bridge # or masquerade multiqueue: @@ -91,11 +90,16 @@ spec: extra_options: - none #- hostpassthrough - # PVC - #storageclass: ocs-storagecluster-ceph-rbd + ## OCS PVC + pvc: false # enable for OCS PVC + pvc_storageclass: ocs-storagecluster-ceph-rbd # Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce - #pvcaccessmode: ReadWriteMany + pvc_pvcaccessmode: ReadWriteMany # Can be one of Filesystem,Block Default: Filesystem - #pvcvolumemode: Block - #storagesize: 5Gi + pvc_pvcvolumemode: Block + pvc_storagesize: 10Gi + ## HostPath - Configuring SELinux on cluster workers + hostpath: false # enable for hostPath + hostpath_path: /var/tmp/disk.img + hostpath_storagesize: 10Gi diff --git a/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml b/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml index 29b9cd245..34bdb856c 100644 --- a/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml +++ b/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml @@ -12,9 +12,9 @@ spec: workload: name: hammerdb args: - #image: quay.io/test/hammerdb:latest # add custom hammerdb image + #image: quay.io/user/hammerdb:latest # add custom hammerdb image pin: false # true for nodeSelector - pin_client: "node1" + pin_node: "node1" resources: false # true for resources requests/limits requests_cpu: 200m requests_memory: 100Mi @@ -32,7 +32,7 @@ spec: db_user: "SA" db_pass: "s3curePasswordString" db_name: "tpcc" - transactions: 500000 + transactions: 10000 raiseerror: "false" keyandthink: "false" driver: "timed" @@ -69,7 +69,7 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" - # es custom fields + # ElasticSearch custom fields es_custom_field: false es_ocp_version: "4.7.0" es_cnv_version: "2.6.2" diff --git a/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml b/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml index 520373c0b..fbd607cc2 100644 --- a/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml +++ b/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml @@ -12,9 +12,9 @@ spec: workload: name: hammerdb args: - #image: quay.io/test/hammerdb:latest # add custom hammerdb image + #image: quay.io/user/hammerdb:latest # add custom hammerdb image pin: false # true for nodeSelector - pin_client: "node1" + pin_node: "node1" db_type: "mssql" timed_test: true test_type: "tpc-c" @@ -65,7 +65,7 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" - # es custom fields + # ElasticSearch custom fields es_custom_field: false es_ocp_version: "4.7.0" es_cnv_version: "2.6.2" @@ -76,11 +76,11 @@ spec: client_vm: dedicatedcpuplacement: false sockets: 1 - cores: 4 + cores: 2 threads: 1 - image: quay.io/ebattat/centos8-mssql2019-container-disk:latest + image: quay.io/cloud-bulldozer/centos8-mssql2019-container-disk:latest limits: - memory: 16Gi + memory: 16Gi # at least 16Gi for VM requests: memory: 100Mi network: @@ -91,3 +91,15 @@ spec: extra_options: - none #- hostpassthrough + ## OCS PVC + pvc: false # enable for OCS PVC + pvc_storageclass: ocs-storagecluster-ceph-rbd + # Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce + pvc_pvcaccessmode: ReadWriteMany + # Can be one of Filesystem,Block Default: Filesystem + pvc_pvcvolumemode: Block + pvc_storagesize: 10Gi + ## HostPath - Configuring SELinux on cluster workers + hostpath: false # enable for hostPath + hostpath_path: /var/tmp/disk.img + hostpath_storagesize: 10Gi diff --git a/roles/hammerdb/defaults/main.yml b/roles/hammerdb/defaults/main.yml new file mode 100644 index 000000000..c2be416b5 --- /dev/null +++ b/roles/hammerdb/defaults/main.yml @@ -0,0 +1,2 @@ +--- +resource_kind: "{{ workload.args.kind | default('pod') }}" diff --git a/roles/hammerdb/tasks/main.yaml b/roles/hammerdb/tasks/main.yaml index 902587034..c3046ca6b 100644 --- a/roles/hammerdb/tasks/main.yaml +++ b/roles/hammerdb/tasks/main.yaml @@ -34,15 +34,16 @@ name: "claim-{{ trunc_uuid }}" namespace: '{{ operator_namespace }}' annotations: - volume.beta.kubernetes.io/storage-class: "{{ workload_args.client_vm.storageclass }}" + volume.beta.kubernetes.io/storage-class: "{{ workload_args.client_vm.pvc_storageclass }}" spec: accessModes: - - "{{ workload_args.client_vm.pvcaccessmode | default('ReadWriteOnce') }}" - volumeMode: "{{ workload_args.client_vm.pvcvolumemode | default('Filesystem') }}" + - "{{ workload_args.client_vm.pvc_pvcaccessmode | default('ReadWriteOnce') }}" + volumeMode: "{{ workload_args.client_vm.pvc_pvcvolumemode | default('Filesystem') }}" resources: requests: - storage: "{{ workload_args.client_vm.storagesize }}" - when: workload_args.client_vm.storageclass is defined + storage: "{{ workload_args.client_vm.pvc_storagesize }}" + + when: workload_args.client_vm.pvc is sameas true - block: - name: template mariadb db creation script @@ -127,7 +128,7 @@ namespace: '{{ operator_namespace }}' data: run_mssql_script.sh: "{{ lookup('template', 'db_mssql_workload_vm.sh.j2') }}" - when: workload_args.db_type == "mssql" and (workload.args.kind | default('pod')) == "vm" + when: workload_args.db_type == "mssql" and resource_kind == "vm" - name: template mariadb workload vm k8s: @@ -139,7 +140,7 @@ namespace: '{{ operator_namespace }}' data: run_mariadb_script.sh: "{{ lookup('template', 'db_mariadb_workload_vm.sh.j2') }}" - when: workload_args.db_type == "mariadb" and (workload.args.kind | default('pod')) == "vm" + when: workload_args.db_type == "mariadb" and resource_kind == "vm" - name: template postgresql workload vm k8s: @@ -151,7 +152,7 @@ namespace: '{{ operator_namespace }}' data: run_postgres_script.sh: "{{ lookup('template', 'db_postgres_workload_vm.sh.j2') }}" - when: workload_args.db_type == "pg" and (workload.args.kind | default('pod')) == "vm" + when: workload_args.db_type == "pg" and resource_kind == "vm" - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 @@ -192,7 +193,7 @@ state: present definition: "{{ lookup('template', 'db_creation.yml') | from_yaml }}" - when: (workload.args.kind | default('pod')) == "pod" + when: resource_kind == "pod" - block: - name: set complete to false @@ -216,7 +217,7 @@ definition: "{{ lookup('template', 'db_creation_postgres_vm.yml') | from_yaml }}" when: (workload_args.db_type == "pg") - when: (workload.args.kind | default('pod')) == "vm" + when: resource_kind == "vm" - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 @@ -259,7 +260,7 @@ state: DB Created complete: true when: hammerdb_creator_pod | json_query('resources[].status.succeeded') and (workload_args.db_benchmark is not defined or not workload_args.db_benchmark) - when: (workload.args.kind | default('pod')) == "pod" + when: resource_kind == "pod" - block: - operator_sdk.util.k8s_status: @@ -284,31 +285,31 @@ state: DB Created complete: false when: db_creation_status.stdout == "true" - when: (workload.args.kind | default('pod')) == "vm" + when: resource_kind == "vm" when: resource_state.resources[0].status.state == "DB Creating" - block: - name: set complete to false command: "redis-cli set complete false" - when: (workload.args.kind | default('pod')) == "vm" + when: resource_kind == "vm" - name: start mssql db workload job k8s: state: present definition: "{{ lookup('template', 'db_mssql_workload.yml.j2') | from_yaml }}" - when: workload_args.db_type == "mssql" and (workload.args.kind | default('pod')) == "pod" + when: workload_args.db_type == "mssql" and resource_kind == "pod" - name: start mariadb db workload job k8s: state: present definition: "{{ lookup('template', 'db_mariadb_workload.yml.j2') | from_yaml }}" - when: workload_args.db_type == "mariadb" and (workload.args.kind | default('pod')) == "pod" + when: workload_args.db_type == "mariadb" and resource_kind == "pod" - name: start postgresql db workload job k8s: state: present definition: "{{ lookup ('template', 'db_postgres_workload.yml.j2') | from_yaml }}" - when: workload_args.db_type == "pg" and (workload.args.kind | default('pod')) == "pod" + when: workload_args.db_type == "pg" and resource_kind == "pod" - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 @@ -340,7 +341,7 @@ state: DB Workload Complete complete: true when: "hammerdb_workload_pod | json_query('resources[].status.succeeded')" - when: (workload.args.kind | default('pod')) == "pod" + when: resource_kind == "pod" - block: - name: get complete @@ -356,6 +357,6 @@ state: DB Workload Complete complete: true when: complete_status.stdout == "true" - when: (workload.args.kind | default('pod')) == "vm" + when: resource_kind == "vm" when: resource_state.resources[0].status.state == "DB workload running" diff --git a/roles/hammerdb/templates/db_creation.yml b/roles/hammerdb/templates/db_creation.yml index 750eab2bd..ed4cdea79 100644 --- a/roles/hammerdb/templates/db_creation.yml +++ b/roles/hammerdb/templates/db_creation.yml @@ -14,7 +14,7 @@ spec: spec: {% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + kubernetes.io/hostname: '{{ workload_args.pin_node }}' {% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/hammerdb/templates/db_creation_mariadb_vm.yml b/roles/hammerdb/templates/db_creation_mariadb_vm.yml index e6b1949b1..98dd0d7c8 100644 --- a/roles/hammerdb/templates/db_creation_mariadb_vm.yml +++ b/roles/hammerdb/templates/db_creation_mariadb_vm.yml @@ -38,8 +38,8 @@ spec: name: hammerdb-mariadb-workload-volume # set serial serial: CVLY623300HK240E -{% if workload_args.client_vm.storageclass is defined - or workload_args.client_vm.hostpath is defined %} +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} - disk: bus: virtio name: data-volume @@ -59,7 +59,7 @@ spec: terminationGracePeriodSeconds: 0 {% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + kubernetes.io/hostname: '{{ workload_args.pin_node }}' {% endif %} networks: - name: default @@ -82,8 +82,8 @@ spec: - "mkdir /tmp/hammerdb-mariadb-test" - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240E | cut -f1 -d' ') /tmp/hammerdb-mariadb-test" runcmd: -{% if workload_args.client_vm.storageclass is defined - or workload_args.client_vm.hostpath is defined %} +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} - "mkdir -p /var/lib/mysql || true" - mkfs.ext4 /dev/disk/by-id/virtio-data - "mount /dev/disk/by-id/virtio-data /var/lib/mysql" @@ -107,15 +107,15 @@ spec: - configMap: name: "{{ ansible_operator_meta.name }}-mariadb-workload-{{ trunc_uuid }}" name: hammerdb-mariadb-workload-volume -{% if workload_args.client_vm.storageclass is defined %} +{% if workload_args.client_vm.pvc is sameas true %} - name: data-volume persistentVolumeClaim: claimName: claim-{{ trunc_uuid }} -{% elif workload_args.client_vm.hostpath is defined %} +{% elif workload_args.client_vm.hostpath is sameas true %} - name: data-volume hostDisk: - path: {{ workload_args.hostpath }} - capacity: {{ workload_args.storagesize | default("5Gi") }} + path: {{ workload_args.client_vm.hostpath_path }} + capacity: {{ workload_args.client_vm.hostpath_storagesize | default("10Gi") }} type: DiskOrCreate {% endif %} status: {} diff --git a/roles/hammerdb/templates/db_creation_mssql_vm.yml b/roles/hammerdb/templates/db_creation_mssql_vm.yml index f62e7d504..6760b76e0 100644 --- a/roles/hammerdb/templates/db_creation_mssql_vm.yml +++ b/roles/hammerdb/templates/db_creation_mssql_vm.yml @@ -38,8 +38,8 @@ spec: name: hammerdb-mssql-workload-volume # set serial serial: CVLY623300HK240E -{% if workload_args.client_vm.storageclass is defined - or workload_args.client_vm.hostpath is defined %} +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} - disk: bus: virtio name: data-volume @@ -59,7 +59,7 @@ spec: terminationGracePeriodSeconds: 0 {% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + kubernetes.io/hostname: '{{ workload_args.pin_node }}' {% endif %} networks: - name: default @@ -82,8 +82,8 @@ spec: - "mkdir /tmp/hammerdb-mssql-test" - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240E | cut -f1 -d' ') /tmp/hammerdb-mssql-test" runcmd: -{% if workload_args.client_vm.storageclass is defined - or workload_args.client_vm.hostpath is defined %} +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} - "mkdir -p /var/opt/mssql || true" - mkfs.ext4 /dev/disk/by-id/virtio-data - "mount /dev/disk/by-id/virtio-data /var/opt/mssql" @@ -108,15 +108,15 @@ spec: - configMap: name: "{{ ansible_operator_meta.name }}-mssql-workload-{{ trunc_uuid }}" name: hammerdb-mssql-workload-volume -{% if workload_args.client_vm.storageclass is defined %} +{% if workload_args.client_vm.pvc is sameas true %} - name: data-volume persistentVolumeClaim: claimName: claim-{{ trunc_uuid }} -{% elif workload_args.client_vm.hostpath is defined %} +{% elif workload_args.client_vm.hostpath is sameas true %} - name: data-volume hostDisk: - path: {{ workload_args.hostpath }} - capacity: {{ workload_args.storagesize | default("5Gi") }} + path: {{ workload_args.client_vm.hostpath_path }} + capacity: {{ workload_args.client_vm.hostpath_storagesize | default("10Gi") }} type: DiskOrCreate {% endif %} status: {} diff --git a/roles/hammerdb/templates/db_creation_postgres_vm.yml b/roles/hammerdb/templates/db_creation_postgres_vm.yml index ae7faebe3..c609ced87 100644 --- a/roles/hammerdb/templates/db_creation_postgres_vm.yml +++ b/roles/hammerdb/templates/db_creation_postgres_vm.yml @@ -38,8 +38,8 @@ spec: name: hammerdb-postgres-workload-volume # set serial serial: CVLY623300HK240E -{% if workload_args.client_vm.storageclass is defined - or workload_args.client_vm.hostpath is defined %} +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} - disk: bus: virtio name: data-volume @@ -59,7 +59,7 @@ spec: terminationGracePeriodSeconds: 0 {% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + kubernetes.io/hostname: '{{ workload_args.pin_node }}' {% endif %} networks: - name: default @@ -81,8 +81,8 @@ spec: - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240D | cut -f1 -d' ') /workload" - "mkdir /tmp/hammerdb-postgres-test" - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240E | cut -f1 -d' ') /tmp/hammerdb-postgres-test" -{% if workload_args.client_vm.storageclass is defined - or workload_args.client_vm.hostpath is defined %} +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} - "mkdir -p /var/lib/pgsql || true" - mkfs.ext4 /dev/disk/by-id/virtio-data - "mount /dev/disk/by-id/virtio-data /var/lib/pgsql" @@ -107,15 +107,15 @@ spec: - configMap: name: "{{ ansible_operator_meta.name }}-postgres-workload-{{ trunc_uuid }}" name: hammerdb-postgres-workload-volume -{% if workload_args.client_vm.storageclass is defined %} +{% if workload_args.client_vm.pvc is sameas true %} - name: data-volume persistentVolumeClaim: claimName: claim-{{ trunc_uuid }} -{% elif workload_args.client_vm.hostpath is defined %} +{% elif workload_args.client_vm.hostpath is sameas true %} - name: data-volume hostDisk: - path: {{ workload_args.hostpath }} - capacity: {{ workload_args.storagesize | default("5Gi") }} + path: {{ workload_args.client_vm.hostpath_path }} + capacity: {{ workload_args.client_vm.hostpath_storagesize | default("10Gi") }} type: DiskOrCreate {% endif %} -status: {} +status: {} \ No newline at end of file diff --git a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 index f2c02a194..0e4fdc6c2 100644 --- a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 @@ -15,7 +15,7 @@ spec: spec: {% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + kubernetes.io/hostname: '{{ workload_args.pin_node }}' {% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -25,8 +25,8 @@ spec: {% if workload_args.resources is sameas true %} resources: requests: - cpu: {{ workload_args.requests_cpu }} - memory: {{ workload_args.requests_memory }} + cpu: {{ workload_args.requests_cpu | default("200m") }} + memory: {{ workload_args.requests_memory | default("100Mi") }} limits: cpu: {{ workload_args.limits_cpu }} memory: {{ workload_args.limits_memory }} diff --git a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 index 59fc4c362..91f7f459a 100644 --- a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 @@ -45,8 +45,8 @@ export es_os_version={{workload_args.es_os_version}}; export es_kind={{workload_args.es_kind}}; {% endif %} export HOME=/root; -{% if workload_args.client_vm.storageclass is defined - or workload_args.client_vm.hostpath is defined %} +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} sudo systemctl stop mariadb.service; echo "UPDATE mysql.user SET Password=PASSWORD('mysql') WHERE User='root';" > input; echo "GRANT ALL ON *.* to root@'%' IDENTIFIED BY 'mysql';" >> input; diff --git a/roles/hammerdb/templates/db_mssql_workload.yml.j2 b/roles/hammerdb/templates/db_mssql_workload.yml.j2 index cfc6ed8e6..64743a9ff 100644 --- a/roles/hammerdb/templates/db_mssql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mssql_workload.yml.j2 @@ -15,7 +15,7 @@ spec: spec: {% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + kubernetes.io/hostname: '{{ workload_args.pin_node }}' {% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -25,8 +25,8 @@ spec: {% if workload_args.resources is sameas true %} resources: requests: - cpu: {{ workload_args.requests_cpu }} - memory: {{ workload_args.requests_memory }} + cpu: {{ workload_args.requests_cpu | default("200m") }} + memory: {{ workload_args.requests_memory | default("100Mi") }} limits: cpu: {{ workload_args.limits_cpu }} memory: {{ workload_args.limits_memory }} diff --git a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 index 7ac6924ec..a18470733 100644 --- a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 @@ -51,13 +51,13 @@ export es_os_version={{workload_args.es_os_version}}; export es_kind={{workload_args.es_kind}}; {% endif %} export HOME=/root; -{% if workload_args.client_vm.storageclass is defined - or workload_args.client_vm.hostpath is defined %} -systemctl stop mssql-server -sudo ACCEPT_EULA='Y' MSSQL_PID='Enterprise' MSSQL_SA_PASSWORD='s3curePasswordString' /opt/mssql/bin/mssql-conf setup -sudo firewall-cmd --zone=public --add-port=1433/tcp --permanent -sudo firewall-cmd --reload -systemctl start mssql-server +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} +sudo systemctl stop mssql-server; +sudo ACCEPT_EULA='Y' MSSQL_PID='Enterprise' MSSQL_SA_PASSWORD='s3curePasswordString' /opt/mssql/bin/mssql-conf setup; +sudo firewall-cmd --zone=public --add-port=1433/tcp --permanent; +sudo firewall-cmd --reload; +sudo systemctl start mssql-server; {% endif %} cd /hammer; ./hammerdbcli auto /creator/createdb.tcl; diff --git a/roles/hammerdb/templates/db_postgres_workload.yml.j2 b/roles/hammerdb/templates/db_postgres_workload.yml.j2 index f4ab3e90a..dbff4ab60 100644 --- a/roles/hammerdb/templates/db_postgres_workload.yml.j2 +++ b/roles/hammerdb/templates/db_postgres_workload.yml.j2 @@ -15,7 +15,7 @@ spec: spec: {% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + kubernetes.io/hostname: '{{ workload_args.pin_node }}' {% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -25,8 +25,8 @@ spec: {% if workload_args.resources is sameas true %} resources: requests: - cpu: {{ workload_args.requests_cpu }} - memory: {{ workload_args.requests_memory }} + cpu: {{ workload_args.requests_cpu | default("200m") }} + memory: {{ workload_args.requests_memory | default("100Mi") }} limits: cpu: {{ workload_args.limits_cpu }} memory: {{ workload_args.limits_memory }} diff --git a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 index 5571f9c91..62580e6b4 100644 --- a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 @@ -49,6 +49,21 @@ export es_os_version={{workload_args.es_os_version}}; export es_kind={{workload_args.es_kind}}; {% endif %} export HOME=/root; +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} +sudo systemctl stop postgresql; +sudo postgresql-setup initdb; +/sbin/restorecon -v /var/lib/pgsql; +echo local all all trust > /var/lib/pgsql/data/pg_hba.conf; +echo host all all all trust >> /var/lib/pgsql/data/pg_hba.conf; +echo max_connections = 100 > /var/lib/pgsql/data/postgresql.conf +echo max_prepared_transactions = 0 >> /var/lib/pgsql/data/postgresql.conf +echo shared_buffers = 4096MB >> /var/lib/pgsql/data/postgresql.conf +echo effective_cache_size = 8192MB >> /var/lib/pgsql/data/postgresql.conf +sudo systemctl start postgresql; +echo "alter role postgres password 'postgres'" > input +psql -U postgres -d postgres < input +{% endif %} cd /hammer; ./hammerdbcli auto /creator/createdb.tcl; redis-cli set db-creation-{{trunc_uuid}} true; diff --git a/tests/test_crs/valid_hammerdb.yaml b/tests/test_crs/valid_hammerdb.yaml index 5fd145989..ed7ef873f 100644 --- a/tests/test_crs/valid_hammerdb.yaml +++ b/tests/test_crs/valid_hammerdb.yaml @@ -15,11 +15,10 @@ spec: index_name: ripsaw-hammerdb metadata: collection: true - cleanup: false workload: name: "hammerdb" args: - # image: quay.io/test/hammerdb:latest + # image: "quay.io/test/hammerdb:latest" db_type: "mssql" timed_test: true test_type: "tpc-c" @@ -28,7 +27,7 @@ spec: db_server: "mssql-deployment.sql-server" db_port: "1433" db_warehouses: 1 - db_num_workers: 1 + db_num_workers: 1 db_user: "SA" db_pass: "s3curePasswordString" db_name: "tpcc" @@ -68,4 +67,4 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" - debug: true \ No newline at end of file + debug: true From 18370a2ee5c7ea32b346d6c09fd21a50cec360fc Mon Sep 17 00:00:00 2001 From: Mohit Sheth Date: Thu, 17 Jun 2021 16:35:07 +0000 Subject: [PATCH 088/249] add alerts to kube-burner dash --- roles/kube-burner/files/metrics-aggregated.yaml | 3 +++ roles/kube-burner/files/metrics.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/roles/kube-burner/files/metrics-aggregated.yaml b/roles/kube-burner/files/metrics-aggregated.yaml index fb67eb574..71182443f 100644 --- a/roles/kube-burner/files/metrics-aggregated.yaml +++ b/roles/kube-burner/files/metrics-aggregated.yaml @@ -182,3 +182,6 @@ metrics: - query: go_gc_duration_seconds{job=~"apiserver|api|etcd",quantile="1"} metricName: goGCDurationSeconds + + - query: topk(10,ALERTS{severity!="none"}) + metricName: alerts diff --git a/roles/kube-burner/files/metrics.yaml b/roles/kube-burner/files/metrics.yaml index 306f0c0c0..88889552e 100644 --- a/roles/kube-burner/files/metrics.yaml +++ b/roles/kube-burner/files/metrics.yaml @@ -115,3 +115,6 @@ metrics: - query: cluster_version{type="completed"} metricName: clusterVersion instant: true + + - query: topk(10,ALERTS{severity!="none"}) + metricName: alerts From f7e59c5ea523c6e958d4ee6c03e63c5fbae106be Mon Sep 17 00:00:00 2001 From: Sarah Bennert Date: Fri, 9 Jul 2021 11:31:04 -0400 Subject: [PATCH 089/249] Add uperf resource labels --- roles/uperf/templates/server.yml.j2 | 3 +++ roles/uperf/templates/server_vm.yml.j2 | 3 +++ roles/uperf/templates/service.yml.j2 | 2 ++ roles/uperf/templates/workload.yml.j2 | 3 +++ roles/uperf/templates/workload_vm.yml.j2 | 3 +++ 5 files changed, 14 insertions(+) diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 0dc36a0fe..f314d622c 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -15,6 +15,9 @@ items: template: metadata: labels: + benchmark-operator-uuid: {{ uuid }} + benchmark-operator-workload: uperf + benchmark-operator-role: server app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'') }}-{{ item }}-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} annotations: diff --git a/roles/uperf/templates/server_vm.yml.j2 b/roles/uperf/templates/server_vm.yml.j2 index 71c130fbe..b9051a93c 100644 --- a/roles/uperf/templates/server_vm.yml.j2 +++ b/roles/uperf/templates/server_vm.yml.j2 @@ -5,6 +5,9 @@ metadata: name: 'uperf-server-{{ item }}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' labels: + benchmark-operator-uuid: {{ uuid }} + benchmark-operator-workload: uperf + benchmark-operator-role: server app : uperf-bench-server-{{ item }}-{{ trunc_uuid }} type : {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} spec: diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index d3e1305ac..a35b3dee6 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -10,6 +10,8 @@ items: name: uperf-service-{{ item }}-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' labels: + benchmark-operator-uuid: {{ uuid }} + benchmark-operator-workload: uperf app: uperf-bench-server-{{ worker_node_list[ node_idx_item ] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} annotations: diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index b6e9ae475..5b239e108 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -17,6 +17,9 @@ items: template: metadata: labels: + benchmark-operator-uuid: {{ uuid }} + benchmark-operator-workload: uperf + benchmark-operator-role: client app: uperf-bench-client-{{ trunc_uuid }} clientfor: {{ item.metadata.labels.app }} type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} diff --git a/roles/uperf/templates/workload_vm.yml.j2 b/roles/uperf/templates/workload_vm.yml.j2 index bc149e02e..a7bb4aac9 100644 --- a/roles/uperf/templates/workload_vm.yml.j2 +++ b/roles/uperf/templates/workload_vm.yml.j2 @@ -5,6 +5,9 @@ metadata: name: 'uperf-client-{{item.status.interfaces[0].ipAddress}}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' labels: + benchmark-operator-uuid: {{ uuid }} + benchmark-operator-workload: uperf + benchmark-operator-role: client app: uperf-bench-client-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} spec: From 7d1dcfd21818657230fd32cd79265b5f043a178a Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 15 Jul 2021 12:03:05 +0200 Subject: [PATCH 090/249] Update metric profiles to new schema Signed-off-by: Raul Sevilla --- max-namespaces.yml | 45 ++++ .../kube-burner/files/metrics-aggregated.yaml | 245 +++++++++--------- roles/kube-burner/files/metrics.yaml | 155 ++++++----- 3 files changed, 244 insertions(+), 201 deletions(-) create mode 100644 max-namespaces.yml diff --git a/max-namespaces.yml b/max-namespaces.yml new file mode 100644 index 000000000..927379ed2 --- /dev/null +++ b/max-namespaces.yml @@ -0,0 +1,45 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-benchmark + namespace: my-ripsaw +spec: + metadata: + collection: false + force: false + image: quay.io/cloud-bulldozer/backpack:latest + privileged: false + serviceaccount: default + ssl: false + stockpileSkipTags: [] + stockpileTags: + - common + - k8s + - openshift + targeted: true + workload: + args: + burst: 50 + cleanup: true + default_index: ripsaw-kube-burner + error_on_verify: true + image: quay.io/cloud-bulldozer/kube-burner:latest + job_iterations: 1 + job_timeout: 300 + log_level: info + metrics_profile: metrics.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: null + pin_server: + node-role.kubernetes.io/worker: "" + pod_wait: false + qps: 25 + step: 30s + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + verify_objects: true + wait_when_finished: true + workload: max-namespaces + name: kube-burner diff --git a/roles/kube-burner/files/metrics-aggregated.yaml b/roles/kube-burner/files/metrics-aggregated.yaml index 71182443f..4a3029712 100644 --- a/roles/kube-burner/files/metrics-aggregated.yaml +++ b/roles/kube-burner/files/metrics-aggregated.yaml @@ -1,187 +1,186 @@ -metrics: # API server - - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0 - metricName: API99thLatency +- query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0 + metricName: API99thLatency - - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0 - metricName: APIRequestRate +- query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0 + metricName: APIRequestRate - - query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0 - metricName: APIInflightRequests +- query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0 + metricName: APIInflightRequests # Container & pod metrics - - query: (sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 - metricName: containerMemory-Masters +- query: (sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 + metricName: containerMemory-Masters - - query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|sdn|ovn-kubernetes|.*apiserver|authentication|.*controller-manager|.*scheduler)"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 - metricName: containerCPU-Masters +- query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|sdn|ovn-kubernetes|.*apiserver|authentication|.*controller-manager|.*scheduler)"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 + metricName: containerCPU-Masters - - query: (sum(irate(container_cpu_usage_seconds_total{pod!="",container="prometheus",namespace="openshift-monitoring"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0 - metricName: containerCPU-Prometheus +- query: (sum(irate(container_cpu_usage_seconds_total{pod!="",container="prometheus",namespace="openshift-monitoring"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0 + metricName: containerCPU-Prometheus - - query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) by (namespace, container)) > 0 - metricName: containerCPU-AggregatedWorkers +- query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) by (namespace, container)) > 0 + metricName: containerCPU-AggregatedWorkers - - query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"}[2m]) * 100 and on (node) kube_node_role{role="infra"}) by (namespace, container)) > 0 - metricName: containerCPU-AggregatedInfra +- query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"}[2m]) * 100 and on (node) kube_node_role{role="infra"}) by (namespace, container)) > 0 + metricName: containerCPU-AggregatedInfra - - query: (sum(container_memory_rss{pod!="",namespace="openshift-monitoring",name!="",container="prometheus"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0 - metricName: containerMemory-Prometheus +- query: (sum(container_memory_rss{pod!="",namespace="openshift-monitoring",name!="",container="prometheus"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0 + metricName: containerMemory-Prometheus - - query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"} and on (node) kube_node_role{role="worker"}) by (container, namespace) - metricName: containerMemory-AggregatedWorkers +- query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"} and on (node) kube_node_role{role="worker"}) by (container, namespace) + metricName: containerMemory-AggregatedWorkers - - query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"} and on (node) kube_node_role{role="infra"}) by (container, namespace) - metricName: containerMemory-AggregatedInfra +- query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"} and on (node) kube_node_role{role="infra"}) by (container, namespace) + metricName: containerMemory-AggregatedInfra # Node metrics - - query: (sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)")) > 0 - metricName: nodeCPU-Masters +- query: (sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)")) > 0 + metricName: nodeCPU-Masters - - query: (avg((sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)"))) by (mode)) > 0 - metricName: nodeCPU-AggregatedWorkers +- query: (avg((sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)"))) by (mode)) > 0 + metricName: nodeCPU-AggregatedWorkers - - query: (avg((sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)"))) by (mode)) > 0 - metricName: nodeCPU-AggregatedInfra +- query: (avg((sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)"))) by (mode)) > 0 + metricName: nodeCPU-AggregatedInfra - - query: avg(node_memory_MemAvailable_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: nodeMemoryAvailable-Masters +- query: avg(node_memory_MemAvailable_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: nodeMemoryAvailable-Masters - - query: avg(node_memory_MemAvailable_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryAvailable-AggregatedWorkers +- query: avg(node_memory_MemAvailable_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryAvailable-AggregatedWorkers - - query: avg(node_memory_MemAvailable_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryAvailable-AggregatedInfra +- query: avg(node_memory_MemAvailable_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryAvailable-AggregatedInfra - - query: avg(node_memory_Active_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: nodeMemoryActive-Masters +- query: avg(node_memory_Active_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: nodeMemoryActive-Masters - - query: avg(node_memory_Active_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryActive-AggregatedWorkers +- query: avg(node_memory_Active_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryActive-AggregatedWorkers - - query: avg(avg(node_memory_Active_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryActive-AggregatedInfra +- query: avg(avg(node_memory_Active_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryActive-AggregatedInfra - - query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: nodeMemoryCached+nodeMemoryBuffers-Masters +- query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: nodeMemoryCached+nodeMemoryBuffers-Masters - - query: avg(node_memory_Cached_bytes + node_memory_Buffers_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryCached+nodeMemoryBuffers-AggregatedWorkers +- query: avg(node_memory_Cached_bytes + node_memory_Buffers_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryCached+nodeMemoryBuffers-AggregatedWorkers - - query: avg(node_memory_Cached_bytes + node_memory_Buffers_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryCached+nodeMemoryBuffers-AggregatedInfra +- query: avg(node_memory_Cached_bytes + node_memory_Buffers_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryCached+nodeMemoryBuffers-AggregatedInfra - - query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: rxNetworkBytes-Masters +- query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: rxNetworkBytes-Masters - - query: avg(irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: rxNetworkBytes-AggregatedWorkers +- query: avg(irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: rxNetworkBytes-AggregatedWorkers - - query: avg(irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: rxNetworkBytes-AggregatedInfra +- query: avg(irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: rxNetworkBytes-AggregatedInfra - - query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: txNetworkBytes-Masters +- query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: txNetworkBytes-Masters - - query: avg(irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: txNetworkBytes-AggregatedWorkers +- query: avg(irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: txNetworkBytes-AggregatedWorkers - - query: avg(irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: txNetworkBytes-AggregatedInfra +- query: avg(irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: txNetworkBytes-AggregatedInfra - - query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: nodeDiskWrittenBytes-Masters +- query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: nodeDiskWrittenBytes-Masters - - query: avg(rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: nodeDiskWrittenBytes-AggregatedWorkers +- query: avg(rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: nodeDiskWrittenBytes-AggregatedWorkers - - query: avg(rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: nodeDiskWrittenBytes-AggregatedInfra +- query: avg(rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: nodeDiskWrittenBytes-AggregatedInfra - - query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: nodeDiskReadBytes-Masters +- query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: nodeDiskReadBytes-Masters - - query: avg(rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: nodeDiskReadBytes-AggregatedWorkers +- query: avg(rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: nodeDiskReadBytes-AggregatedWorkers - - query: avg(rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: nodeDiskReadBytes-AggregatedInfra +- query: avg(rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: nodeDiskReadBytes-AggregatedInfra # Etcd metrics - - query: sum(rate(etcd_server_leader_changes_seen_total[2m])) - metricName: etcdLeaderChangesRate +- query: sum(rate(etcd_server_leader_changes_seen_total[2m])) + metricName: etcdLeaderChangesRate - - query: etcd_server_is_leader > 0 - metricName: etcdServerIsLeader +- query: etcd_server_is_leader > 0 + metricName: etcdServerIsLeader - - query: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m])) - metricName: 99thEtcdDiskBackendCommitDurationSeconds +- query: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m])) + metricName: 99thEtcdDiskBackendCommitDurationSeconds - - query: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m])) - metricName: 99thEtcdDiskWalFsyncDurationSeconds +- query: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m])) + metricName: 99thEtcdDiskWalFsyncDurationSeconds - - query: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) - metricName: 99thEtcdRoundTripTimeSeconds +- query: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) + metricName: 99thEtcdRoundTripTimeSeconds - - query: etcd_mvcc_db_total_size_in_bytes - metricName: etcdDBPhysicalSizeBytes +- query: etcd_mvcc_db_total_size_in_bytes + metricName: etcdDBPhysicalSizeBytes - - query: etcd_mvcc_db_total_size_in_use_in_bytes - metricName: etcdDBLogicalSizeBytes +- query: etcd_mvcc_db_total_size_in_use_in_bytes + metricName: etcdDBLogicalSizeBytes - - query: sum by (cluster_version)(etcd_cluster_version) - metricName: etcdVersion - instant: true +- query: sum by (cluster_version)(etcd_cluster_version) + metricName: etcdVersion + instant: true - - query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 - metricName: etcdObjectCount +- query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 + metricName: etcdObjectCount - - query: histogram_quantile(0.99,sum(rate(etcd_request_duration_seconds_bucket[2m])) by (le,operation,apiserver)) > 0 - metricName: P99APIEtcdRequestLatency +- query: histogram_quantile(0.99,sum(rate(etcd_request_duration_seconds_bucket[2m])) by (le,operation,apiserver)) > 0 + metricName: P99APIEtcdRequestLatency # Cluster metrics - - query: sum(kube_namespace_status_phase) by (phase) > 0 - metricName: namespaceCount +- query: sum(kube_namespace_status_phase) by (phase) > 0 + metricName: namespaceCount - - query: sum(kube_pod_status_phase{}) by (phase) - metricName: podStatusCount +- query: sum(kube_pod_status_phase{}) by (phase) + metricName: podStatusCount - - query: count(kube_secret_info{}) - metricName: secretCount +- query: count(kube_secret_info{}) + metricName: secretCount - - query: count(kube_deployment_labels{}) - metricName: deploymentCount +- query: count(kube_deployment_labels{}) + metricName: deploymentCount - - query: count(kube_configmap_info{}) - metricName: configmapCount +- query: count(kube_configmap_info{}) + metricName: configmapCount - - query: count(kube_service_info{}) - metricName: serviceCount +- query: count(kube_service_info{}) + metricName: serviceCount - - query: kube_node_role - metricName: nodeRoles - instant: true +- query: kube_node_role + metricName: nodeRoles + instant: true - - query: sum(kube_node_status_condition{status="true"}) by (condition) - metricName: nodeStatus +- query: sum(kube_node_status_condition{status="true"}) by (condition) + metricName: nodeStatus - - query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 - metricName: containerDiskUsage +- query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 + metricName: containerDiskUsage - - query: cluster_version{type="completed"} - metricName: clusterVersion - instant: true +- query: cluster_version{type="completed"} + metricName: clusterVersion + instant: true # Golang metrics - - query: go_memstats_heap_alloc_bytes{job=~"apiserver|api|etcd"} - metricName: goHeapAllocBytes +- query: go_memstats_heap_alloc_bytes{job=~"apiserver|api|etcd"} + metricName: goHeapAllocBytes - - query: go_memstats_heap_inuse_bytes{job=~"apiserver|api|etcd"} - metricName: goHeapInuseBytes - - - query: go_gc_duration_seconds{job=~"apiserver|api|etcd",quantile="1"} - metricName: goGCDurationSeconds +- query: go_memstats_heap_inuse_bytes{job=~"apiserver|api|etcd"} + metricName: goHeapInuseBytes - - query: topk(10,ALERTS{severity!="none"}) - metricName: alerts +- query: go_gc_duration_seconds{job=~"apiserver|api|etcd",quantile="1"} + metricName: goGCDurationSeconds + +- query: topk(10,ALERTS{severity!="none"}) + metricName: alerts diff --git a/roles/kube-burner/files/metrics.yaml b/roles/kube-burner/files/metrics.yaml index 88889552e..9e0df694a 100644 --- a/roles/kube-burner/files/metrics.yaml +++ b/roles/kube-burner/files/metrics.yaml @@ -1,120 +1,119 @@ -metrics: # API server - - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0 - metricName: API99thLatency +- query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0 + metricName: API99thLatency - - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0 - metricName: APIRequestRate +- query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0 + metricName: APIRequestRate - - query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0 - metricName: APIInflightRequests +- query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0 + metricName: APIInflightRequests # Containers & pod metrics - - query: sum(irate(container_cpu_usage_seconds_total{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}[2m]) * 100) by (pod, namespace, node) - metricName: podCPU +- query: sum(irate(container_cpu_usage_seconds_total{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}[2m]) * 100) by (pod, namespace, node) + metricName: podCPU - - query: sum(container_memory_rss{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}) by (pod, namespace, node) - metricName: podMemory +- query: sum(container_memory_rss{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}) by (pod, namespace, node) + metricName: podMemory - - query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 - metricName: containerDiskUsage +- query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 + metricName: containerDiskUsage # Kubelet & CRI-O metrics - - query: sum(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} - metricName: kubeletCPU +- query: sum(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} + metricName: kubeletCPU - - query: sum(process_resident_memory_bytes{service="kubelet",job="kubelet"}) by (node) and on (node) kube_node_role{role="worker"} - metricName: kubeletMemory +- query: sum(process_resident_memory_bytes{service="kubelet",job="kubelet"}) by (node) and on (node) kube_node_role{role="worker"} + metricName: kubeletMemory - - query: sum(irate(process_cpu_seconds_total{service="kubelet",job="crio"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} - metricName: crioCPU +- query: sum(irate(process_cpu_seconds_total{service="kubelet",job="crio"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} + metricName: crioCPU - - query: sum(process_resident_memory_bytes{service="kubelet",job="crio"}) by (node) and on (node) kube_node_role{role="worker"} - metricName: crioMemory +- query: sum(process_resident_memory_bytes{service="kubelet",job="crio"}) by (node) and on (node) kube_node_role{role="worker"} + metricName: crioMemory # Node metrics - - query: sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) > 0 - metricName: nodeCPU +- query: sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) > 0 + metricName: nodeCPU - - query: avg(node_memory_MemAvailable_bytes) by (instance) - metricName: nodeMemoryAvailable +- query: avg(node_memory_MemAvailable_bytes) by (instance) + metricName: nodeMemoryAvailable - - query: avg(node_memory_Active_bytes) by (instance) - metricName: nodeMemoryActive +- query: avg(node_memory_Active_bytes) by (instance) + metricName: nodeMemoryActive - - query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) - metricName: nodeMemoryCached+nodeMemoryBuffers +- query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) + metricName: nodeMemoryCached+nodeMemoryBuffers - - query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) - metricName: rxNetworkBytes +- query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: rxNetworkBytes - - query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) - metricName: txNetworkBytes +- query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: txNetworkBytes - - query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) - metricName: nodeDiskWrittenBytes +- query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskWrittenBytes - - query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) - metricName: nodeDiskReadBytes +- query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskReadBytes - - query: sum(rate(etcd_server_leader_changes_seen_total[2m])) - metricName: etcdLeaderChangesRate +- query: sum(rate(etcd_server_leader_changes_seen_total[2m])) + metricName: etcdLeaderChangesRate # Etcd metrics - - query: etcd_server_is_leader > 0 - metricName: etcdServerIsLeader +- query: etcd_server_is_leader > 0 + metricName: etcdServerIsLeader - - query: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m])) - metricName: 99thEtcdDiskBackendCommitDurationSeconds +- query: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m])) + metricName: 99thEtcdDiskBackendCommitDurationSeconds - - query: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m])) - metricName: 99thEtcdDiskWalFsyncDurationSeconds +- query: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m])) + metricName: 99thEtcdDiskWalFsyncDurationSeconds - - query: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) - metricName: 99thEtcdRoundTripTimeSeconds +- query: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) + metricName: 99thEtcdRoundTripTimeSeconds - - query: etcd_mvcc_db_total_size_in_bytes - metricName: etcdDBPhysicalSizeBytes +- query: etcd_mvcc_db_total_size_in_bytes + metricName: etcdDBPhysicalSizeBytes - - query: etcd_mvcc_db_total_size_in_use_in_bytes - metricName: etcdDBLogicalSizeBytes +- query: etcd_mvcc_db_total_size_in_use_in_bytes + metricName: etcdDBLogicalSizeBytes - - query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 - metricName: etcdObjectCount +- query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 + metricName: etcdObjectCount - - query: sum by (cluster_version)(etcd_cluster_version) - metricName: etcdVersion - instant: true +- query: sum by (cluster_version)(etcd_cluster_version) + metricName: etcdVersion + instant: true # Cluster metrics - - query: sum(kube_namespace_status_phase) by (phase) > 0 - metricName: namespaceCount +- query: sum(kube_namespace_status_phase) by (phase) > 0 + metricName: namespaceCount - - query: sum(kube_pod_status_phase{}) by (phase) - metricName: podStatusCount +- query: sum(kube_pod_status_phase{}) by (phase) + metricName: podStatusCount - - query: count(kube_secret_info{}) - metricName: secretCount +- query: count(kube_secret_info{}) + metricName: secretCount - - query: count(kube_deployment_labels{}) - metricName: deploymentCount +- query: count(kube_deployment_labels{}) + metricName: deploymentCount - - query: count(kube_configmap_info{}) - metricName: configmapCount +- query: count(kube_configmap_info{}) + metricName: configmapCount - - query: count(kube_service_info{}) - metricName: serviceCount +- query: count(kube_service_info{}) + metricName: serviceCount - - query: kube_node_role - metricName: nodeRoles - instant: true +- query: kube_node_role + metricName: nodeRoles + instant: true - - query: sum(kube_node_status_condition{status="true"}) by (condition) - metricName: nodeStatus +- query: sum(kube_node_status_condition{status="true"}) by (condition) + metricName: nodeStatus - - query: cluster_version{type="completed"} - metricName: clusterVersion - instant: true +- query: cluster_version{type="completed"} + metricName: clusterVersion + instant: true - - query: topk(10,ALERTS{severity!="none"}) - metricName: alerts +- query: topk(10,ALERTS{severity!="none"}) + metricName: alerts From 59028c2eeceef314d43b074e8716428ba04d1f51 Mon Sep 17 00:00:00 2001 From: Keith Whitley Date: Fri, 16 Jul 2021 13:18:59 -0400 Subject: [PATCH 091/249] operator sdk upgrade (#598) * operator sdk upgrade * changing ns * changing ns to benchmark-operator and changing directory structure to make sure templates and group vars are picked up * changing benchmark-operator wait command * changing benchmark-operator deploy pull policy * disabling auth proxy * adding redis to kustomize * adding redis to kustomize * renaming benchmark names and changing cleanup step logic * fixing cleanup logic * fixing cleanup logic pt2 * testing rescue block * delete ns before test * skipping cleanup * delete benchmarks before test * fixing delete * fixing byowl profile * kube-burner test * removing duplicate task in roles that already get called in the top level playbook and fixing sa * fixing bad indent * fixing bad file path * fixing uuid logic * refactoring test logic * fixing hammerdb * adding node selector * adding kernel cache dropper * adding kernel cache dropper * adding bug fixes * adding more bug fixes * adding more bug fixes and deleting molecule files * fixing uperf test logic * adding fail fast logic on timeout and fixing some test logic * increasing timeouts for log generator and uperf * fixing backpack requirements to add scc to right sa * removing old install docs and adding new to the README * removing delete_operator clal * changing default IMG * fixing crd symlink in chart * fix manager env * fixing typo * removing unused gitignore patterns and updating helm command in README --- .gitignore | 12 +- .travis.yml | 7 - CONTRIBUTING.md | 6 +- build/Dockerfile => Dockerfile | 11 +- Makefile | 179 ++++++++++++ PROJECT | 16 ++ README.md | 32 ++- build/test-framework/Dockerfile | 10 - build/test-framework/ansible-test.sh | 7 - charts/benchmark-operator/crds/crd.yaml | 1 + .../crds/ripsaw_v1alpha1_ripsaw_crd.yaml | 1 - .../ripsaw.cloudbulldozer.io_benchmarks.yaml | 3 + config/crd/kustomization.yaml | 6 + config/default/kustomization.yaml | 30 ++ config/default/manager_auth_proxy_patch.yaml | 27 ++ config/default/manager_config_patch.yaml | 20 ++ config/manager/controller_manager_config.yaml | 10 + config/manager/kustomization.yaml | 10 + config/manager/manager.yaml | 82 ++++++ config/manifests/kustomization.yaml | 7 + config/prometheus/kustomization.yaml | 2 + config/prometheus/monitor.yaml | 20 ++ .../rbac/auth_proxy_client_clusterrole.yaml | 9 + config/rbac/auth_proxy_role.yaml | 17 ++ .../rbac/auth_proxy_role_binding.yaml | 16 +- config/rbac/auth_proxy_service.yaml | 14 + config/rbac/benchmark_editor_role.yaml | 24 ++ config/rbac/benchmark_viewer_role.yaml | 20 ++ config/rbac/kustomization.yaml | 19 ++ config/rbac/leader_election_role.yaml | 37 +++ .../rbac/leader_election_role_binding.yaml | 17 +- .../rbac/pod_security_policy.yaml | 0 config/rbac/role.yaml | 257 +++++++++++++++++ config/rbac/role_binding.yaml | 25 ++ config/rbac/service_account.yaml | 5 + .../samples/byowl/cr.yaml | 2 +- .../samples/concurrent-builds/cr.yaml | 2 +- .../samples/cyclictest/cr.yaml | 2 +- .../samples/fio/cr.yaml | 2 +- .../samples/fio/vm-cr.yaml | 2 +- .../samples/flent/cr.yaml | 2 +- .../samples/fs-drift/cr.yaml | 2 +- .../samples/hammerdb/cr.yaml | 2 +- .../samples/hammerdb/mariadb/cr.yaml | 2 +- .../samples/hammerdb/mariadb/hostpath-cr.yaml | 0 .../samples/hammerdb/mariadb/local-cr.yaml | 0 .../samples/hammerdb/mariadb/pvc-cr.yaml | 0 .../samples/hammerdb/mariadb/vm-cr.yaml | 2 +- .../samples/hammerdb/mssql/cr.yaml | 2 +- .../samples/hammerdb/mssql/hostpath-cr.yaml | 0 .../samples/hammerdb/mssql/local-cr.yaml | 0 .../samples/hammerdb/mssql/vm-cr.yaml | 2 +- .../samples/hammerdb/postgres/cr.yaml | 2 +- .../hammerdb/postgres/hostpath-cr.yaml | 0 .../samples/hammerdb/postgres/local-cr.yaml | 0 .../samples/hammerdb/postgres/vm-cr.yaml | 2 +- .../samples/hammerdb/vm-cr.yaml | 2 +- .../samples/iperf3/cr.yaml | 2 +- .../samples/kube-burner/cr.yaml | 2 +- config/samples/kustomization.yaml | 33 +++ .../samples/log-generator/cr.yaml | 2 +- .../samples/oslat/cr.yaml | 2 +- .../samples/pgbench/cr.yaml | 2 +- config/samples/ripsaw_v1alpha1_benchmark.yaml | 7 + .../samples/servicemesh/cr.yaml | 2 +- .../samples/smallfile/cr.yaml | 2 +- .../samples/stressng/cr.yaml | 2 +- .../samples/stressng/vm-cr.yaml | 2 +- .../samples/sysbench/cr.yaml | 2 +- .../samples/testpmd/cr.yaml | 2 +- .../samples/uperf/cr.yaml | 2 +- .../samples/uperf/vm-cr.yaml | 2 +- .../samples/vegeta/cr.yaml | 2 +- .../samples/ycsb/cr.yaml | 2 +- config/scorecard/bases/config.yaml | 7 + config/scorecard/kustomization.yaml | 16 ++ config/scorecard/patches/basic.config.yaml | 10 + config/scorecard/patches/olm.config.yaml | 50 ++++ config/testing/debug_logs_patch.yaml | 14 + config/testing/kustomization.yaml | 23 ++ config/testing/manager_image.yaml | 12 + config/testing/pull_policy/Always.yaml | 12 + config/testing/pull_policy/IfNotPresent.yaml | 12 + config/testing/pull_policy/Never.yaml | 12 + deploy/10_service_account.yaml | 5 - deploy/20_role.yaml | 112 -------- deploy/25_role.yaml | 13 - deploy/40_cluster_role_kubevirt.yaml | 61 ----- docs/byowl.md | 2 +- docs/cerberus.md | 4 +- docs/cyclictest.md | 4 +- docs/elastic.md | 10 +- docs/fio_distributed.md | 2 +- docs/flent.md | 6 +- docs/fs-drift.md | 2 +- docs/hammerdb.md | 4 +- docs/image_pull.md | 2 +- docs/installation.md | 120 -------- docs/iperf.md | 2 +- docs/log_generator.md | 6 +- docs/metadata.md | 16 +- docs/oslat.md | 4 +- docs/pgbench.md | 2 +- docs/prometheus.md | 2 +- docs/scale_openshift.md | 8 +- docs/servicemesh.md | 6 +- docs/smallfile.md | 4 +- docs/stressng.md | 2 +- docs/sysbench.md | 2 +- docs/system-metrics.md | 2 +- docs/testpmd.md | 4 +- docs/uperf.md | 2 +- docs/vegeta.md | 2 +- docs/ycsb.md | 2 +- meta/main.yml | 3 - playbook.yml => playbooks/benchmark.yml | 49 ++-- {group_vars => playbooks/group_vars}/all.yml | 0 .../templates}/metadata.yml.j2 | 0 requirements.yml | 5 +- resources/backpack_role.yaml | 6 +- ...psaw_v1alpha1_hammerdb_mariadb_server.yaml | 86 ------ ...ripsaw_v1alpha1_hammerdb_mssql_server.yaml | 59 ---- ...aw_v1alpha1_hammerdb_mssql_server_pvc.yaml | 78 ------ ...saw_v1alpha1_hammerdb_postgres_server.yaml | 81 ------ ...v1alpha1_hammerdb_postgres_server_pvc.yaml | 99 ------- resources/kernel-cache-drop-clusterrole.yaml | 33 --- resources/kernel-cache-drop-daemonset.yaml | 5 +- resources/kube-burner-role.yml | 6 +- resources/namespace.yaml | 4 - resources/operator.yaml | 70 ----- resources/scale_role.yaml | 6 +- resources/self_provisioner_binding.yaml | 2 +- roles/benchmark_state/tasks/failure.yml | 15 + roles/flent/tasks/main.yml | 9 - roles/fs-drift/tasks/main.yml | 11 +- roles/fs-drift/templates/workload_job.yml.j2 | 2 +- roles/hammerdb/tasks/main.yaml | 4 +- roles/kube-burner/tasks/main.yml | 8 - roles/pgbench/tasks/main.yml | 9 - roles/pgbench/tasks/prep_workload.yml | 2 +- roles/smallfile/tasks/main.yml | 9 - roles/stressng/tasks/main.yaml | 8 +- roles/uperf/tasks/setup.yml | 9 - roles/vegeta/tasks/main.yml | 8 - test.sh | 33 +-- tests/check_es.py | 0 tests/common.sh | 258 +++++++++--------- tests/full_test_file_trigger | 4 - tests/mssql.yaml | 0 tests/test_backpack.sh | 14 +- tests/test_byowl.sh | 18 +- tests/test_crs/valid_backpack_daemonset.yaml | 2 +- tests/test_crs/valid_backpack_init.yaml | 2 +- tests/test_crs/valid_byowl.yaml | 7 +- tests/test_crs/valid_fiod.yaml | 4 +- tests/test_crs/valid_fiod_bsrange.yaml | 4 +- tests/test_crs/valid_fiod_hostpath.yaml | 4 +- tests/test_crs/valid_fiod_ocs_cache_drop.yaml | 4 +- tests/test_crs/valid_flent.yaml | 2 +- tests/test_crs/valid_flent_resources.yaml | 4 +- tests/test_crs/valid_fs_drift.yaml | 4 +- tests/test_crs/valid_fs_drift_hostpath.yaml | 4 +- tests/test_crs/valid_hammerdb.yaml | 2 +- tests/test_crs/valid_image_pull.yaml | 2 +- tests/test_crs/valid_iperf3.yaml | 4 +- tests/test_crs/valid_kube-burner.yaml | 2 +- tests/test_crs/valid_log_generator.yaml | 2 +- tests/test_crs/valid_pgbench.yaml | 2 +- tests/test_crs/valid_scale_down.yaml | 6 +- tests/test_crs/valid_scale_up.yaml | 6 +- tests/test_crs/valid_servicemesh.yaml | 4 +- tests/test_crs/valid_smallfile.yaml | 4 +- tests/test_crs/valid_smallfile_hostpath.yaml | 4 +- tests/test_crs/valid_stressng.yaml | 2 +- tests/test_crs/valid_sysbench.yaml | 4 +- tests/test_crs/valid_uperf.yaml | 2 +- tests/test_crs/valid_uperf_networkpolicy.yaml | 4 +- tests/test_crs/valid_uperf_resources.yaml | 4 +- tests/test_crs/valid_uperf_serviceip.yaml | 4 +- tests/test_crs/valid_vegeta.yaml | 2 +- tests/test_crs/valid_vegeta_hostnetwork.yaml | 2 +- tests/test_crs/valid_ycsb-mongo.yaml | 2 +- tests/test_fiod.sh | 21 +- tests/test_flent.sh | 16 +- tests/test_fs_drift.sh | 27 +- tests/test_hammerdb.sh | 18 +- tests/test_image_pull.sh | 12 +- tests/test_iperf3.sh | 20 +- tests/test_kubeburner.sh | 13 +- tests/test_list | 0 tests/test_log_generator.sh | 14 +- tests/test_pgbench.sh | 22 +- tests/test_scale_openshift.sh | 19 +- tests/test_servicemesh.sh | 18 +- tests/test_smallfile.sh | 28 +- tests/test_stressng.sh | 20 +- tests/test_sysbench.sh | 19 +- tests/test_uperf.sh | 16 +- tests/test_vegeta.sh | 14 +- tests/test_ycsb.sh | 18 +- watches.yaml | 9 +- 201 files changed, 1601 insertions(+), 1481 deletions(-) delete mode 100644 .travis.yml rename build/Dockerfile => Dockerfile (70%) create mode 100644 Makefile create mode 100644 PROJECT delete mode 100644 build/test-framework/Dockerfile delete mode 100644 build/test-framework/ansible-test.sh create mode 120000 charts/benchmark-operator/crds/crd.yaml delete mode 120000 charts/benchmark-operator/crds/ripsaw_v1alpha1_ripsaw_crd.yaml rename resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml => config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml (98%) create mode 100644 config/crd/kustomization.yaml create mode 100644 config/default/kustomization.yaml create mode 100644 config/default/manager_auth_proxy_patch.yaml create mode 100644 config/default/manager_config_patch.yaml create mode 100644 config/manager/controller_manager_config.yaml create mode 100644 config/manager/kustomization.yaml create mode 100644 config/manager/manager.yaml create mode 100644 config/manifests/kustomization.yaml create mode 100644 config/prometheus/kustomization.yaml create mode 100644 config/prometheus/monitor.yaml create mode 100644 config/rbac/auth_proxy_client_clusterrole.yaml create mode 100644 config/rbac/auth_proxy_role.yaml rename deploy/35_role_binding.yaml => config/rbac/auth_proxy_role_binding.yaml (59%) create mode 100644 config/rbac/auth_proxy_service.yaml create mode 100644 config/rbac/benchmark_editor_role.yaml create mode 100644 config/rbac/benchmark_viewer_role.yaml create mode 100644 config/rbac/kustomization.yaml create mode 100644 config/rbac/leader_election_role.yaml rename deploy/30_role_binding.yaml => config/rbac/leader_election_role_binding.yaml (56%) rename deploy/50_pod_security_policy.yml => config/rbac/pod_security_policy.yaml (100%) create mode 100644 config/rbac/role.yaml create mode 100644 config/rbac/role_binding.yaml create mode 100644 config/rbac/service_account.yaml rename resources/crds/ripsaw_v1alpha1_byowl_cr.yaml => config/samples/byowl/cr.yaml (89%) rename resources/crds/ripsaw_v1alpha1_concurrent_builds_cr.yaml => config/samples/concurrent-builds/cr.yaml (98%) rename resources/crds/ripsaw_v1alpha1_cyclictest_cr.yaml => config/samples/cyclictest/cr.yaml (94%) rename resources/crds/ripsaw_v1alpha1_fio_distributed_cr.yaml => config/samples/fio/cr.yaml (99%) rename resources/crds/ripsaw_v1alpha1_fio_distributed_vm_cr.yaml => config/samples/fio/vm-cr.yaml (99%) rename resources/crds/ripsaw_v1alpha1_flent_cr.yaml => config/samples/flent/cr.yaml (93%) rename resources/crds/ripsaw_v1alpha1_fs_drift_cr.yaml => config/samples/fs-drift/cr.yaml (97%) rename resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml => config/samples/hammerdb/cr.yaml (98%) rename resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml => config/samples/hammerdb/mariadb/cr.yaml (98%) rename resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_hostpath.yaml => config/samples/hammerdb/mariadb/hostpath-cr.yaml (100%) rename resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_local.yaml => config/samples/hammerdb/mariadb/local-cr.yaml (100%) rename resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml => config/samples/hammerdb/mariadb/pvc-cr.yaml (100%) rename resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml => config/samples/hammerdb/mariadb/vm-cr.yaml (99%) rename resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml => config/samples/hammerdb/mssql/cr.yaml (98%) rename resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_hostpath.yaml => config/samples/hammerdb/mssql/hostpath-cr.yaml (100%) rename resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_local.yaml => config/samples/hammerdb/mssql/local-cr.yaml (100%) rename resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml => config/samples/hammerdb/mssql/vm-cr.yaml (99%) rename resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml => config/samples/hammerdb/postgres/cr.yaml (98%) rename resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_hostpath.yaml => config/samples/hammerdb/postgres/hostpath-cr.yaml (100%) rename resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_local.yaml => config/samples/hammerdb/postgres/local-cr.yaml (100%) rename resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml => config/samples/hammerdb/postgres/vm-cr.yaml (99%) rename resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml => config/samples/hammerdb/vm-cr.yaml (99%) rename resources/crds/ripsaw_v1alpha1_iperf3_cr.yaml => config/samples/iperf3/cr.yaml (93%) rename resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml => config/samples/kube-burner/cr.yaml (98%) create mode 100644 config/samples/kustomization.yaml rename resources/crds/ripsaw_v1alpha1_log_generator_cr.yaml => config/samples/log-generator/cr.yaml (97%) rename resources/crds/ripsaw_v1alpha1_oslat_cr.yaml => config/samples/oslat/cr.yaml (94%) rename resources/crds/ripsaw_v1alpha1_pgbench_cr.yaml => config/samples/pgbench/cr.yaml (98%) create mode 100644 config/samples/ripsaw_v1alpha1_benchmark.yaml rename resources/crds/ripsaw_v1alpha1_servicemesh_cr.yaml => config/samples/servicemesh/cr.yaml (97%) rename resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml => config/samples/smallfile/cr.yaml (93%) rename resources/crds/ripsaw_v1alpha1_stressng_cr.yaml => config/samples/stressng/cr.yaml (95%) rename resources/crds/ripsaw_v1alpha1_stressng_vm.yaml => config/samples/stressng/vm-cr.yaml (97%) rename resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml => config/samples/sysbench/cr.yaml (92%) rename resources/crds/ripsaw_v1alpha1_testpmd_cr.yaml => config/samples/testpmd/cr.yaml (93%) rename resources/crds/ripsaw_v1alpha1_uperf_cr.yaml => config/samples/uperf/cr.yaml (98%) rename resources/crds/ripsaw_v1alpha1_uperf_vm.yaml => config/samples/uperf/vm-cr.yaml (98%) rename resources/crds/ripsaw_v1alpha1_vegeta_cr.yaml => config/samples/vegeta/cr.yaml (95%) rename resources/crds/ripsaw_v1alpha1_ycsb_cr.yaml => config/samples/ycsb/cr.yaml (95%) create mode 100644 config/scorecard/bases/config.yaml create mode 100644 config/scorecard/kustomization.yaml create mode 100644 config/scorecard/patches/basic.config.yaml create mode 100644 config/scorecard/patches/olm.config.yaml create mode 100644 config/testing/debug_logs_patch.yaml create mode 100644 config/testing/kustomization.yaml create mode 100644 config/testing/manager_image.yaml create mode 100644 config/testing/pull_policy/Always.yaml create mode 100644 config/testing/pull_policy/IfNotPresent.yaml create mode 100644 config/testing/pull_policy/Never.yaml delete mode 100644 deploy/10_service_account.yaml delete mode 100644 deploy/20_role.yaml delete mode 100644 deploy/25_role.yaml delete mode 100644 deploy/40_cluster_role_kubevirt.yaml delete mode 100644 docs/installation.md delete mode 100644 meta/main.yml rename playbook.yml => playbooks/benchmark.yml (79%) rename {group_vars => playbooks/group_vars}/all.yml (100%) rename {templates => playbooks/templates}/metadata.yml.j2 (100%) delete mode 100644 resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server.yaml delete mode 100644 resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml delete mode 100644 resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml delete mode 100644 resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml delete mode 100644 resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml delete mode 100644 resources/kernel-cache-drop-clusterrole.yaml delete mode 100644 resources/namespace.yaml delete mode 100644 resources/operator.yaml create mode 100644 roles/benchmark_state/tasks/failure.yml mode change 100644 => 100755 tests/check_es.py mode change 100644 => 100755 tests/common.sh mode change 100644 => 100755 tests/full_test_file_trigger mode change 100644 => 100755 tests/mssql.yaml mode change 100644 => 100755 tests/test_backpack.sh mode change 100644 => 100755 tests/test_byowl.sh mode change 100644 => 100755 tests/test_iperf3.sh mode change 100644 => 100755 tests/test_list mode change 100644 => 100755 tests/test_pgbench.sh mode change 100644 => 100755 tests/test_sysbench.sh mode change 100644 => 100755 tests/test_vegeta.sh mode change 100644 => 100755 tests/test_ycsb.sh diff --git a/.gitignore b/.gitignore index afd4228f9..98ff18d96 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,12 @@ -# Ignore + +# Binaries for programs and plugins +*.dll +*.so +*.dylib +bin + +# editor and IDE paraphernalia +.idea *.swp +*.swo +*~ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 594e194b3..000000000 --- a/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -sudo: required -services: docker -language: python -install: - - pip install docker molecule openshift -script: - - molecule test -s test-local diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eeec3769d..0e6317012 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,7 +83,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: your_workload_name @@ -95,14 +95,14 @@ spec: my_key_3: my_value_3 ``` -Note: The Benchmark has to be created in the namespace `my-ripsaw` +Note: The Benchmark has to be created in the namespace `benchmark-operator` ### Additional guidance for adding a workload * Please keep the [workload status](README.md#workloads-status) updated * To help users understand how the workload can be run, please add a guide similar to [uperf](docs/uperf.md) * Add the link for your workload guide to [installation guide](docs/installation.md#running-workloads) -* Ensure all resources created are within the `my-ripsaw` namespace, this can be done by setting namespace +* Ensure all resources created are within the `benchmark-operator` namespace, this can be done by setting namespace to use `operator_namespace` var. This is to ensure that the resources aren't defaulted to current active namespace which is what `ansible_operator_meta.namespace` would default to. * All resources created as part of your role should use `trunc_uuid` ansible var in their names and labels, so diff --git a/build/Dockerfile b/Dockerfile similarity index 70% rename from build/Dockerfile rename to Dockerfile index 998fe4f19..59c30f046 100644 --- a/build/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/operator-framework/ansible-operator:v1.5.0 +FROM quay.io/operator-framework/ansible-operator:v1.9.0 USER root COPY requirements.yml ${HOME}/requirements.yml @@ -9,10 +9,7 @@ COPY image_resources/centos8-appstream.repo /etc/yum.repos.d/centos8-appstream.r RUN dnf install -y --nodocs redis openssl --enablerepo=centos8-appstream && dnf clean all COPY resources/kernel-cache-drop-daemonset.yaml /opt/kernel_cache_dropper/ -COPY group_vars/ ${HOME}/group_vars/ -COPY roles/ ${HOME}/roles/ -COPY templates/ ${HOME}/templates/ -COPY meta/ ${HOME}/meta/ COPY watches.yaml ${HOME}/watches.yaml -COPY playbook.yml ${HOME}/playbook.yml -USER 1001 +COPY roles/ ${HOME}/roles/ +COPY playbooks/ ${HOME}/playbooks/ +USER 1001 \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..4bc5e712f --- /dev/null +++ b/Makefile @@ -0,0 +1,179 @@ +# VERSION defines the project version for the bundle. +# Update this value when you upgrade the version of your project. +# To re-generate a bundle for another specific version without changing the standard setup, you can: +# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) +# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) +VERSION ?= 0.0.1 + +# CHANNELS define the bundle channels used in the bundle. +# Add a new line here if you would like to change its default config. (E.g CHANNELS = "preview,fast,stable") +# To re-generate a bundle for other specific channels without changing the standard setup, you can: +# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=preview,fast,stable) +# - use environment variables to overwrite this value (e.g export CHANNELS="preview,fast,stable") +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=$(CHANNELS) +endif + +# DEFAULT_CHANNEL defines the default channel used in the bundle. +# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") +# To re-generate a bundle for any other default channel without changing the default setup, you can: +# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) +# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) +endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) + +# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. +# This variable is used to construct full image tags for bundle and catalog images. +# +# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both +# cloudbulldozer.io/ripsaw-bundle:$VERSION and cloudbulldozer.io/ripsaw-catalog:$VERSION. +IMAGE_TAG_BASE ?= cloudbulldozer.io/ripsaw + +# BUNDLE_IMG defines the image:tag used for the bundle. +# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) + +# Image URL to use all building/pushing image targets +IMG ?= quay.io/benchmark-operator/benchmark-operator:master + +all: docker-build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Build + +run: ansible-operator ## Run against the configured Kubernetes cluster in ~/.kube/config + $(ANSIBLE_OPERATOR) run + +docker-build: ## Build docker image with the manager. + docker build -t ${IMG} . + +docker-push: ## Push docker image with the manager. + docker push ${IMG} + +podman-build: ## Build docker image with the manager. + podman build -t ${IMG} . + +podman-push: ## Push docker image with the manager. + podman push ${IMG} + +##@ Deployment + +install: kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +uninstall: kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl delete -f - + +deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | kubectl apply -f - + +undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/default | kubectl delete -f - + +OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +ARCH := $(shell uname -m | sed 's/x86_64/amd64/') + +.PHONY: kustomize +KUSTOMIZE = $(shell pwd)/bin/kustomize +kustomize: ## Download kustomize locally if necessary. +ifeq (,$(wildcard $(KUSTOMIZE))) +ifeq (,$(shell which kustomize 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(KUSTOMIZE)) ;\ + curl -sSLo - https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v3.5.4/kustomize_v3.5.4_$(OS)_$(ARCH).tar.gz | \ + tar xzf - -C bin/ ;\ + } +else +KUSTOMIZE = $(shell which kustomize) +endif +endif + +.PHONY: ansible-operator +ANSIBLE_OPERATOR = $(shell pwd)/bin/ansible-operator +ansible-operator: ## Download ansible-operator locally if necessary, preferring the $(pwd)/bin path over global if both exist. +ifeq (,$(wildcard $(ANSIBLE_OPERATOR))) +ifeq (,$(shell which ansible-operator 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(ANSIBLE_OPERATOR)) ;\ + curl -sSLo $(ANSIBLE_OPERATOR) https://github.com/operator-framework/operator-sdk/releases/download/v1.7.2/ansible-operator_$(OS)_$(ARCH) ;\ + chmod +x $(ANSIBLE_OPERATOR) ;\ + } +else +ANSIBLE_OPERATOR = $(shell which ansible-operator) +endif +endif + +.PHONY: bundle +bundle: kustomize ## Generate bundle manifests and metadata, then validate generated files. + operator-sdk generate kustomize manifests -q + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) + operator-sdk bundle validate ./bundle + +.PHONY: bundle-build +bundle-build: ## Build the bundle image. + docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . + +.PHONY: bundle-push +bundle-push: ## Push the bundle image. + $(MAKE) docker-push IMG=$(BUNDLE_IMG) + +.PHONY: opm +OPM = ./bin/opm +opm: ## Download opm locally if necessary. +ifeq (,$(wildcard $(OPM))) +ifeq (,$(shell which opm 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPM)) ;\ + curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.15.1/$(OS)-$(ARCH)-opm ;\ + chmod +x $(OPM) ;\ + } +else +OPM = $(shell which opm) +endif +endif + +# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). +# These images MUST exist in a registry and be pull-able. +BUNDLE_IMGS ?= $(BUNDLE_IMG) + +# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) + +# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. +ifneq ($(origin CATALOG_BASE_IMG), undefined) +FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) +endif + +# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. +# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: +# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator +.PHONY: catalog-build +catalog-build: opm ## Build a catalog image. + $(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) + +# Push the catalog image. +.PHONY: catalog-push +catalog-push: ## Push a catalog image. + $(MAKE) docker-push IMG=$(CATALOG_IMG) diff --git a/PROJECT b/PROJECT new file mode 100644 index 000000000..56220cc3c --- /dev/null +++ b/PROJECT @@ -0,0 +1,16 @@ +domain: cloudbulldozer.io +layout: +- ansible.sdk.operatorframework.io/v1 +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} +projectName: ripsaw +resources: +- api: + crdVersion: v1 + namespaced: true + domain: cloudbulldozer.io + group: ripsaw + kind: Benchmark + version: v1alpha1 +version: "3" diff --git a/README.md b/README.md index 5b791ad8b..476b651af 100644 --- a/README.md +++ b/README.md @@ -3,25 +3,43 @@ The intent of this Operator is to deploy common workloads to establish a performance baseline of Kubernetes cluster on your provider. -## Installation + +## Installation (Default) +The easiest way to install the operator is through the operator-sdk methods provided in the `Makefile`. + +```bash +git clone https://github.com/cloud-bulldozer/benchmark-operator +make deploy +``` + +If you wish to build a version of the operator from your local copy of the repo, you can run + +```bash +git clone https://github.com/cloud-bulldozer/benchmark-operator +make podman-build podman-push deploy IMG=$YOUR_IMAGE +``` + +> Note: building the image requires podman + +## Installation (Helm) Installing the benchmark-operator is easiest by using the helm chart and can be done with the following commands. This requires your machine to have Helm installed. [Install Helm](https://helm.sh/docs/intro/install/) -> Note: If running on openshift you'll need to run this command before installing the chart. `oc adm policy -n my-ripsaw add-scc-to-user privileged -z benchmark-operator` +> Note: If running on openshift you'll need to run this command before installing the chart. `oc adm policy -n benchmark-operator add-scc-to-user privileged -z benchmark-operator` ```bash git clone https://github.com/cloud-bulldozer/benchmark-operator cd benchmark-operator/charts/benchmark-operator -helm install benchmark-operator . -n my-ripsaw --create-namespace +helm install benchmark-operator . -n benchmark-operator --create-namespace ``` To delete this release, you can do so with the following command: ```bash -helm delete benchmark-operator -n my-ripsaw --purge +helm uninstall benchmark-operator -n benchmark-operator ``` @@ -75,7 +93,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://my-es.foo.bar:80" @@ -101,7 +119,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://my-es.foo.bar:80" @@ -126,7 +144,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: uuid: 6060004a-7515-424e-93bb-c49844600dde elasticsearch: diff --git a/build/test-framework/Dockerfile b/build/test-framework/Dockerfile deleted file mode 100644 index e6c041367..000000000 --- a/build/test-framework/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG BASEIMAGE -FROM ${BASEIMAGE} -USER 0 -RUN yum install -y python-devel gcc libffi-devel && pip install molecule -ARG NAMESPACEDMAN -ADD $NAMESPACEDMAN /namespaced.yaml -ADD build/test-framework/ansible-test.sh /ansible-test.sh -RUN chmod +x /ansible-test.sh -USER 1001 -ADD . /opt/ansible/project diff --git a/build/test-framework/ansible-test.sh b/build/test-framework/ansible-test.sh deleted file mode 100644 index 9719f2609..000000000 --- a/build/test-framework/ansible-test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -export WATCH_NAMESPACE=${TEST_NAMESPACE} -(/usr/local/bin/entrypoint)& -trap "kill $!" SIGINT SIGTERM EXIT - -cd ${HOME}/project -exec molecule test -s test-cluster diff --git a/charts/benchmark-operator/crds/crd.yaml b/charts/benchmark-operator/crds/crd.yaml new file mode 120000 index 000000000..f28d729c9 --- /dev/null +++ b/charts/benchmark-operator/crds/crd.yaml @@ -0,0 +1 @@ +../../../config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml \ No newline at end of file diff --git a/charts/benchmark-operator/crds/ripsaw_v1alpha1_ripsaw_crd.yaml b/charts/benchmark-operator/crds/ripsaw_v1alpha1_ripsaw_crd.yaml deleted file mode 120000 index 54e584ebe..000000000 --- a/charts/benchmark-operator/crds/ripsaw_v1alpha1_ripsaw_crd.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml \ No newline at end of file diff --git a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml similarity index 98% rename from resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml rename to config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml index 94a0244dc..5f24460c1 100644 --- a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml +++ b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml @@ -167,6 +167,9 @@ spec: system_metrics: type: string default: Not collected + message: + type: string + default: None additionalPrinterColumns: - name: Type type: string diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 000000000..cdd73b4f8 --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,6 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/ripsaw.cloudbulldozer.io_benchmarks.yaml +#+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 000000000..0b6d73728 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,30 @@ +# Adds namespace to all resources. +namespace: benchmark-operator + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: benchmark- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../crd +- ../rbac +- ../manager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus + +patchesStrategicMerge: [] +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +# - manager_auth_proxy_patch.yaml + +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 000000000..151724322 --- /dev/null +++ b/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,27 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=10" + ports: + - containerPort: 8443 + name: https + - name: manager + args: + - "--health-probe-bind-address=:6789" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" + - "--leader-election-id=ripsaw" diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml new file mode 100644 index 000000000..6c400155c --- /dev/null +++ b/config/default/manager_config_patch.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--config=controller_manager_config.yaml" + volumeMounts: + - name: manager-config + mountPath: /controller_manager_config.yaml + subPath: controller_manager_config.yaml + volumes: + - name: manager-config + configMap: + name: manager-config diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml new file mode 100644 index 000000000..7d61df648 --- /dev/null +++ b/config/manager/controller_manager_config.yaml @@ -0,0 +1,10 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: ControllerManagerConfig +health: + healthProbeBindAddress: :6789 +metrics: + bindAddress: 127.0.0.1:8080 + +leaderElection: + leaderElect: true + resourceName: 811c9dc5.cloudbulldozer.io diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 000000000..2bcd3eeaa --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,10 @@ +resources: +- manager.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- name: manager-config + files: + - controller_manager_config.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 000000000..225839355 --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,82 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: operator +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + labels: + control-plane: controller-manager + spec: + securityContext: + runAsNonRoot: true + containers: + - name: manager + args: + - --leader-elect + - --leader-election-id=ripsaw + image: controller:latest + imagePullPolicy: Always + env: + - name: ANSIBLE_GATHERING + value: explicit + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: "benchmark-operator" + - name: MAX_CONCURRENT_RECONCILES_BENCHMARK_RIPSAW_CLOUDBULLDOZER_IO + value: "1" + - name: ANSIBLE_VERBOSITY_BENCHMARK_RIPSAW_CLOUDBULLDOZER_IO + value: "4" + securityContext: + allowPrivilegeEscalation: false + livenessProbe: + httpGet: + path: /healthz + port: 6789 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 6789 + initialDelaySeconds: 5 + periodSeconds: 10 + - name: redis-master + image: k8s.gcr.io/redis:v1 + env: + - name: MASTER + value: "true" + ports: + - containerPort: 6379 + resources: + limits: + cpu: "2.0" + volumeMounts: + - mountPath: /redis-master-data + name: data + serviceAccountName: operator + terminationGracePeriodSeconds: 10 + volumes: + - name: data + emptyDir: {} diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml new file mode 100644 index 000000000..2dc3e9a73 --- /dev/null +++ b/config/manifests/kustomization.yaml @@ -0,0 +1,7 @@ +# These resources constitute the fully configured set of manifests +# used to generate the 'manifests/' directory in a bundle. +resources: +- bases/ripsaw.clusterserviceversion.yaml +- ../default +- ../samples +- ../scorecard diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 000000000..ed137168a --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 000000000..d19136ae7 --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,20 @@ + +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml new file mode 100644 index 000000000..51a75db47 --- /dev/null +++ b/config/rbac/auth_proxy_client_clusterrole.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml new file mode 100644 index 000000000..80e1857c5 --- /dev/null +++ b/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/deploy/35_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml similarity index 59% rename from deploy/35_role_binding.yaml rename to config/rbac/auth_proxy_role_binding.yaml index 1deac0165..ec7acc0a1 100644 --- a/deploy/35_role_binding.yaml +++ b/config/rbac/auth_proxy_role_binding.yaml @@ -1,12 +1,12 @@ -kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - name: benchmark-operator-rolebinding -subjects: -- kind: ServiceAccount - name: benchmark-operator - namespace: my-ripsaw + name: proxy-rolebinding roleRef: - kind: ClusterRole - name: benchmark-operator-role apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml new file mode 100644 index 000000000..6cf656be1 --- /dev/null +++ b/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + targetPort: https + selector: + control-plane: controller-manager diff --git a/config/rbac/benchmark_editor_role.yaml b/config/rbac/benchmark_editor_role.yaml new file mode 100644 index 000000000..7d50193d5 --- /dev/null +++ b/config/rbac/benchmark_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit benchmarks. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: benchmark-editor-role +rules: +- apiGroups: + - ripsaw.cloudbulldozer.io + resources: + - benchmarks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ripsaw.cloudbulldozer.io + resources: + - benchmarks/status + verbs: + - get diff --git a/config/rbac/benchmark_viewer_role.yaml b/config/rbac/benchmark_viewer_role.yaml new file mode 100644 index 000000000..9601801a7 --- /dev/null +++ b/config/rbac/benchmark_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view benchmarks. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: benchmark-viewer-role +rules: +- apiGroups: + - ripsaw.cloudbulldozer.io + resources: + - benchmarks + verbs: + - get + - list + - watch +- apiGroups: + - ripsaw.cloudbulldozer.io + resources: + - benchmarks/status + verbs: + - get diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 000000000..caef677fc --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,19 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +- pod_security_policy.yaml +# Comment the following 4 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 000000000..4190ec805 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,37 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/deploy/30_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml similarity index 56% rename from deploy/30_role_binding.yaml rename to config/rbac/leader_election_role_binding.yaml index 9ff9929a1..1d1321ed4 100644 --- a/deploy/30_role_binding.yaml +++ b/config/rbac/leader_election_role_binding.yaml @@ -1,13 +1,12 @@ -kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding metadata: - name: benchmark-operator - namespace: my-ripsaw -subjects: -- kind: ServiceAccount - name: benchmark-operator - namespace: my-ripsaw + name: leader-election-rolebinding roleRef: - kind: Role - name: benchmark-operator apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/deploy/50_pod_security_policy.yml b/config/rbac/pod_security_policy.yaml similarity index 100% rename from deploy/50_pod_security_policy.yml rename to config/rbac/pod_security_policy.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 000000000..577062564 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,257 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manager-role +rules: + ## + ## Base operator rules + ## + - apiGroups: + - "" + resources: + - secrets + - pods + - pods/exec + - pods/log + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + ## + ## Rules Ported from old namespaced role + ## + - apiGroups: + - "" + resources: + - pods + - daemonsets + - services + - endpoints + - persistentvolumeclaims + - virtualmachineinstances + - events + - configmaps + - secrets + - pods/log + verbs: + - '*' + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + - deployments/finalizers + verbs: + - '*' + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create + - apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances + verbs: + - '*' + - apiGroups: + - ripsaw.cloudbulldozer.io + resources: + - '*' + verbs: + - '*' + - apiGroups: + - batch + - extensions + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - privileged + - apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use + - apiGroups: + - hyperfoil.io + resources: + - hyperfoils + verbs: + - '*' + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get + - list + - watch + + ## + ## Rules from old cluster role + ## + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + + ## + ## Rules from kube-virt + ## + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + - clusterroles + - clusterrolebindings + - daemonsets + verbs: + - '*' + - apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - list + - apiGroups: + - rbac.authorization.k8s.io + resources: + - nodes + verbs: + - get + - list + - apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/console + - virtualmachineinstances/vnc + verbs: + - get + - apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances + - virtualmachines + - virtualmachineinstancepresets + - virtualmachineinstancereplicasets + verbs: + - get + - delete + - create + - update + - patch + - list + - watch + - deletecollection + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubevirt-config + verbs: + - update + - get + - patch + + ## + ## Rules from kernel-cache-drop + ## + - apiGroups: + - '' + resources: + - nodes + - pods + verbs: + - get + - list + - apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - list + + ## + ## Rules for ripsaw.cloudbulldozer.io/v1alpha1, Kind: Benchmark + ## + - apiGroups: + - ripsaw.cloudbulldozer.io + resources: + - benchmarks + - benchmarks/status + - benchmarks/finalizers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +#+kubebuilder:scaffold:rules diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml new file mode 100644 index 000000000..2ffba9b95 --- /dev/null +++ b/config/rbac/role_binding.yaml @@ -0,0 +1,25 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: operator + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-self-provisioner +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: self-provisioner +subjects: +- kind: ServiceAccount + name: operator + namespace: system \ No newline at end of file diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml new file mode 100644 index 000000000..38423bc86 --- /dev/null +++ b/config/rbac/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator + namespace: system diff --git a/resources/crds/ripsaw_v1alpha1_byowl_cr.yaml b/config/samples/byowl/cr.yaml similarity index 89% rename from resources/crds/ripsaw_v1alpha1_byowl_cr.yaml rename to config/samples/byowl/cr.yaml index 467fa3113..bf5d7a506 100644 --- a/resources/crds/ripsaw_v1alpha1_byowl_cr.yaml +++ b/config/samples/byowl/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark-example - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: byowl diff --git a/resources/crds/ripsaw_v1alpha1_concurrent_builds_cr.yaml b/config/samples/concurrent-builds/cr.yaml similarity index 98% rename from resources/crds/ripsaw_v1alpha1_concurrent_builds_cr.yaml rename to config/samples/concurrent-builds/cr.yaml index d45129462..90ca8a5be 100644 --- a/resources/crds/ripsaw_v1alpha1_concurrent_builds_cr.yaml +++ b/config/samples/concurrent-builds/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: kube-burner-concurrent-builds-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # Metadata information elasticsearch: diff --git a/resources/crds/ripsaw_v1alpha1_cyclictest_cr.yaml b/config/samples/cyclictest/cr.yaml similarity index 94% rename from resources/crds/ripsaw_v1alpha1_cyclictest_cr.yaml rename to config/samples/cyclictest/cr.yaml index 60ad70e27..618ae1569 100644 --- a/resources/crds/ripsaw_v1alpha1_cyclictest_cr.yaml +++ b/config/samples/cyclictest/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: cyclictest-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # where elastic search is running elasticsearch: diff --git a/resources/crds/ripsaw_v1alpha1_fio_distributed_cr.yaml b/config/samples/fio/cr.yaml similarity index 99% rename from resources/crds/ripsaw_v1alpha1_fio_distributed_cr.yaml rename to config/samples/fio/cr.yaml index 679018358..ffd21cf29 100644 --- a/resources/crds/ripsaw_v1alpha1_fio_distributed_cr.yaml +++ b/config/samples/fio/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: fio-benchmark-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # where elastic search is running elasticsearch: diff --git a/resources/crds/ripsaw_v1alpha1_fio_distributed_vm_cr.yaml b/config/samples/fio/vm-cr.yaml similarity index 99% rename from resources/crds/ripsaw_v1alpha1_fio_distributed_vm_cr.yaml rename to config/samples/fio/vm-cr.yaml index 5855a19c6..175bc492a 100644 --- a/resources/crds/ripsaw_v1alpha1_fio_distributed_vm_cr.yaml +++ b/config/samples/fio/vm-cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: fio-vm-benchmark-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # where elastic search is running elasticsearch: diff --git a/resources/crds/ripsaw_v1alpha1_flent_cr.yaml b/config/samples/flent/cr.yaml similarity index 93% rename from resources/crds/ripsaw_v1alpha1_flent_cr.yaml rename to config/samples/flent/cr.yaml index 5dc9bb680..96cae8e2d 100644 --- a/resources/crds/ripsaw_v1alpha1_flent_cr.yaml +++ b/config/samples/flent/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: flent-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster elasticsearch: diff --git a/resources/crds/ripsaw_v1alpha1_fs_drift_cr.yaml b/config/samples/fs-drift/cr.yaml similarity index 97% rename from resources/crds/ripsaw_v1alpha1_fs_drift_cr.yaml rename to config/samples/fs-drift/cr.yaml index e1f5170d8..31f3be7b6 100644 --- a/resources/crds/ripsaw_v1alpha1_fs_drift_cr.yaml +++ b/config/samples/fs-drift/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: fs-drift-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # where elastic search is running elasticsearch: diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml b/config/samples/hammerdb/cr.yaml similarity index 98% rename from resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml rename to config/samples/hammerdb/cr.yaml index 34bdb856c..e72bd1c34 100644 --- a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml +++ b/config/samples/hammerdb/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: hammerdb-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://my.elasticsearch.server:80 diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml b/config/samples/hammerdb/mariadb/cr.yaml similarity index 98% rename from resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml rename to config/samples/hammerdb/mariadb/cr.yaml index 2a7f0547e..48405675d 100644 --- a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml +++ b/config/samples/hammerdb/mariadb/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: hammerdb-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://my.elasticsearch.server:80 diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_hostpath.yaml b/config/samples/hammerdb/mariadb/hostpath-cr.yaml similarity index 100% rename from resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_hostpath.yaml rename to config/samples/hammerdb/mariadb/hostpath-cr.yaml diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_local.yaml b/config/samples/hammerdb/mariadb/local-cr.yaml similarity index 100% rename from resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_local.yaml rename to config/samples/hammerdb/mariadb/local-cr.yaml diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml b/config/samples/hammerdb/mariadb/pvc-cr.yaml similarity index 100% rename from resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml rename to config/samples/hammerdb/mariadb/pvc-cr.yaml diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml b/config/samples/hammerdb/mariadb/vm-cr.yaml similarity index 99% rename from resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml rename to config/samples/hammerdb/mariadb/vm-cr.yaml index 5431ef132..03fd85c81 100644 --- a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml +++ b/config/samples/hammerdb/mariadb/vm-cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: hammerdb-vm-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://my.elasticsearch.server:80 diff --git a/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml b/config/samples/hammerdb/mssql/cr.yaml similarity index 98% rename from resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml rename to config/samples/hammerdb/mssql/cr.yaml index 34bdb856c..e72bd1c34 100644 --- a/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml +++ b/config/samples/hammerdb/mssql/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: hammerdb-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://my.elasticsearch.server:80 diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_hostpath.yaml b/config/samples/hammerdb/mssql/hostpath-cr.yaml similarity index 100% rename from resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_hostpath.yaml rename to config/samples/hammerdb/mssql/hostpath-cr.yaml diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_local.yaml b/config/samples/hammerdb/mssql/local-cr.yaml similarity index 100% rename from resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_local.yaml rename to config/samples/hammerdb/mssql/local-cr.yaml diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml b/config/samples/hammerdb/mssql/vm-cr.yaml similarity index 99% rename from resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml rename to config/samples/hammerdb/mssql/vm-cr.yaml index 47609fb0b..cb87abcae 100644 --- a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml +++ b/config/samples/hammerdb/mssql/vm-cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: hammerdb-vm-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://my.elasticsearch.server:80 diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml b/config/samples/hammerdb/postgres/cr.yaml similarity index 98% rename from resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml rename to config/samples/hammerdb/postgres/cr.yaml index 6a5f4bdbc..c64a29240 100644 --- a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml +++ b/config/samples/hammerdb/postgres/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: hammerdb-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://my.elasticsearch.server:80 diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_hostpath.yaml b/config/samples/hammerdb/postgres/hostpath-cr.yaml similarity index 100% rename from resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_hostpath.yaml rename to config/samples/hammerdb/postgres/hostpath-cr.yaml diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_local.yaml b/config/samples/hammerdb/postgres/local-cr.yaml similarity index 100% rename from resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_local.yaml rename to config/samples/hammerdb/postgres/local-cr.yaml diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml b/config/samples/hammerdb/postgres/vm-cr.yaml similarity index 99% rename from resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml rename to config/samples/hammerdb/postgres/vm-cr.yaml index f5366d850..2d8176d7e 100644 --- a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml +++ b/config/samples/hammerdb/postgres/vm-cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: hammerdb-vm-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://my.elasticsearch.server:80 diff --git a/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml b/config/samples/hammerdb/vm-cr.yaml similarity index 99% rename from resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml rename to config/samples/hammerdb/vm-cr.yaml index fbd607cc2..736f9451c 100644 --- a/resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_vm.yaml +++ b/config/samples/hammerdb/vm-cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: hammerdb-vm-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://my.elasticsearch.server:80 diff --git a/resources/crds/ripsaw_v1alpha1_iperf3_cr.yaml b/config/samples/iperf3/cr.yaml similarity index 93% rename from resources/crds/ripsaw_v1alpha1_iperf3_cr.yaml rename to config/samples/iperf3/cr.yaml index f47f7e6dc..a14c8a081 100644 --- a/resources/crds/ripsaw_v1alpha1_iperf3_cr.yaml +++ b/config/samples/iperf3/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: iperf3-benchmark-example - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: iperf3 diff --git a/resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml b/config/samples/kube-burner/cr.yaml similarity index 98% rename from resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml rename to config/samples/kube-burner/cr.yaml index 66983f8e9..86e68ae8a 100644 --- a/resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml +++ b/config/samples/kube-burner/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: kube-burner-cluster-density-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # Metadata information elasticsearch: diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml new file mode 100644 index 000000000..13d4215f6 --- /dev/null +++ b/config/samples/kustomization.yaml @@ -0,0 +1,33 @@ +## Append samples you want in your CSV to this file as resources ## +resources: +- ripsaw_v1alpha1_benchmark.yaml +- byowl/cr.yaml +- concurrent-builds/cr.yaml +- cyclictest/cr.yaml +- fio/cr.yaml +- fio/vm-cr.yaml +- fs-drift/cr.yaml +- hammerdb/cr.yaml +- hammerdb/vm-cr.yaml +- hammerdb/mariadb/cr.yaml +- hammerdb/mariadb/vm-cr.yaml +- hammerdb/mssql/cr.yaml +- hammerdb/mssql/vm-cr.yaml +- hammerdb/postgres/cr.yaml +- hammerdb/postgres/vm-cr.yaml +- iperf3/cr.yaml +- kube-burner/cr.yaml +- log-generator/cr.yaml +- oslat/cr.yaml +- pgbench/cr.yaml +- servicemesh/cr.yaml +- smallfile/cr.yaml +- stressng/cr.yaml +- stressng/vm-cr.yaml +- sysbench/cr.yaml +- testpmd/cr.yaml +- uperf/cr.yaml +- uperf/vm-cr.yaml +- vegeta/cr.yaml +- ycsb/cr.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/resources/crds/ripsaw_v1alpha1_log_generator_cr.yaml b/config/samples/log-generator/cr.yaml similarity index 97% rename from resources/crds/ripsaw_v1alpha1_log_generator_cr.yaml rename to config/samples/log-generator/cr.yaml index 5940f8ac3..af8e8045b 100644 --- a/resources/crds/ripsaw_v1alpha1_log_generator_cr.yaml +++ b/config/samples/log-generator/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: log-generator - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: # Elastic search instance with full URL format. https://elastic.apps.org:9200 diff --git a/resources/crds/ripsaw_v1alpha1_oslat_cr.yaml b/config/samples/oslat/cr.yaml similarity index 94% rename from resources/crds/ripsaw_v1alpha1_oslat_cr.yaml rename to config/samples/oslat/cr.yaml index 004f187d2..8d5d309e3 100644 --- a/resources/crds/ripsaw_v1alpha1_oslat_cr.yaml +++ b/config/samples/oslat/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: oslat-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # where elastic search is running elasticsearch: diff --git a/resources/crds/ripsaw_v1alpha1_pgbench_cr.yaml b/config/samples/pgbench/cr.yaml similarity index 98% rename from resources/crds/ripsaw_v1alpha1_pgbench_cr.yaml rename to config/samples/pgbench/cr.yaml index eb40ae377..84c22f6bf 100644 --- a/resources/crds/ripsaw_v1alpha1_pgbench_cr.yaml +++ b/config/samples/pgbench/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: pgbench-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster elasticsearch: diff --git a/config/samples/ripsaw_v1alpha1_benchmark.yaml b/config/samples/ripsaw_v1alpha1_benchmark.yaml new file mode 100644 index 000000000..45d204117 --- /dev/null +++ b/config/samples/ripsaw_v1alpha1_benchmark.yaml @@ -0,0 +1,7 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: benchmark-sample +spec: + # Add fields here + foo: bar diff --git a/resources/crds/ripsaw_v1alpha1_servicemesh_cr.yaml b/config/samples/servicemesh/cr.yaml similarity index 97% rename from resources/crds/ripsaw_v1alpha1_servicemesh_cr.yaml rename to config/samples/servicemesh/cr.yaml index 5f6d83362..158e499af 100644 --- a/resources/crds/ripsaw_v1alpha1_servicemesh_cr.yaml +++ b/config/samples/servicemesh/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: servicemesh-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: servicemesh diff --git a/resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml b/config/samples/smallfile/cr.yaml similarity index 93% rename from resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml rename to config/samples/smallfile/cr.yaml index 57e09ad41..427f4536b 100644 --- a/resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml +++ b/config/samples/smallfile/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: test_user: homer_simpson clustername: aws-2009-09-10 diff --git a/resources/crds/ripsaw_v1alpha1_stressng_cr.yaml b/config/samples/stressng/cr.yaml similarity index 95% rename from resources/crds/ripsaw_v1alpha1_stressng_cr.yaml rename to config/samples/stressng/cr.yaml index cd7a213dd..f5d11f0e5 100644 --- a/resources/crds/ripsaw_v1alpha1_stressng_cr.yaml +++ b/config/samples/stressng/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: stressng-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://es.server.com:80 diff --git a/resources/crds/ripsaw_v1alpha1_stressng_vm.yaml b/config/samples/stressng/vm-cr.yaml similarity index 97% rename from resources/crds/ripsaw_v1alpha1_stressng_vm.yaml rename to config/samples/stressng/vm-cr.yaml index d2442e8e5..805a66864 100644 --- a/resources/crds/ripsaw_v1alpha1_stressng_vm.yaml +++ b/config/samples/stressng/vm-cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: stressng-vm-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://es.server.com:80 diff --git a/resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml b/config/samples/sysbench/cr.yaml similarity index 92% rename from resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml rename to config/samples/sysbench/cr.yaml index b48d9a8a3..f414d504f 100644 --- a/resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml +++ b/config/samples/sysbench/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: sysbench-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: sysbench diff --git a/resources/crds/ripsaw_v1alpha1_testpmd_cr.yaml b/config/samples/testpmd/cr.yaml similarity index 93% rename from resources/crds/ripsaw_v1alpha1_testpmd_cr.yaml rename to config/samples/testpmd/cr.yaml index 2f90d68d3..5e50987de 100644 --- a/resources/crds/ripsaw_v1alpha1_testpmd_cr.yaml +++ b/config/samples/testpmd/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: testpmd-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster workload: diff --git a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml b/config/samples/uperf/cr.yaml similarity index 98% rename from resources/crds/ripsaw_v1alpha1_uperf_cr.yaml rename to config/samples/uperf/cr.yaml index 8d44d1a88..1570a9eda 100644 --- a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml +++ b/config/samples/uperf/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: uperf-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster elasticsearch: diff --git a/resources/crds/ripsaw_v1alpha1_uperf_vm.yaml b/config/samples/uperf/vm-cr.yaml similarity index 98% rename from resources/crds/ripsaw_v1alpha1_uperf_vm.yaml rename to config/samples/uperf/vm-cr.yaml index 0b3ad04f3..24ecf4c73 100644 --- a/resources/crds/ripsaw_v1alpha1_uperf_vm.yaml +++ b/config/samples/uperf/vm-cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: test-cluster test_user: user # user is a key that points to user triggering ripsaw, useful to search results in ES diff --git a/resources/crds/ripsaw_v1alpha1_vegeta_cr.yaml b/config/samples/vegeta/cr.yaml similarity index 95% rename from resources/crds/ripsaw_v1alpha1_vegeta_cr.yaml rename to config/samples/vegeta/cr.yaml index fa706ef30..dfa521393 100644 --- a/resources/crds/ripsaw_v1alpha1_vegeta_cr.yaml +++ b/config/samples/vegeta/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: vegeta-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: # Elastic search instance with full URL format. https://elastic.apps.org:9200 diff --git a/resources/crds/ripsaw_v1alpha1_ycsb_cr.yaml b/config/samples/ycsb/cr.yaml similarity index 95% rename from resources/crds/ripsaw_v1alpha1_ycsb_cr.yaml rename to config/samples/ycsb/cr.yaml index f7ede7797..15c33288c 100644 --- a/resources/crds/ripsaw_v1alpha1_ycsb_cr.yaml +++ b/config/samples/ycsb/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: ycsb-couchbase-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: server: http://my.elasticsearch.server:80 diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml new file mode 100644 index 000000000..c77047841 --- /dev/null +++ b/config/scorecard/bases/config.yaml @@ -0,0 +1,7 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml new file mode 100644 index 000000000..50cd2d084 --- /dev/null +++ b/config/scorecard/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- bases/config.yaml +patchesJson6902: +- path: patches/basic.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +- path: patches/olm.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +#+kubebuilder:scaffold:patchesJson6902 diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml new file mode 100644 index 000000000..2d4e0a9e7 --- /dev/null +++ b/config/scorecard/patches/basic.config.yaml @@ -0,0 +1,10 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: basic + test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml new file mode 100644 index 000000000..59c557683 --- /dev/null +++ b/config/scorecard/patches/olm.config.yaml @@ -0,0 +1,50 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: olm + test: olm-bundle-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: olm + test: olm-crds-have-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: olm + test: olm-crds-have-resources-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: olm + test: olm-spec-descriptors-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/config/testing/debug_logs_patch.yaml b/config/testing/debug_logs_patch.yaml new file mode 100644 index 000000000..3fb3d559f --- /dev/null +++ b/config/testing/debug_logs_patch.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + env: + - name: ANSIBLE_DEBUG_LOGS + value: "TRUE" diff --git a/config/testing/kustomization.yaml b/config/testing/kustomization.yaml new file mode 100644 index 000000000..410916239 --- /dev/null +++ b/config/testing/kustomization.yaml @@ -0,0 +1,23 @@ +# Adds namespace to all resources. +namespace: osdk-test + +namePrefix: osdk- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +patchesStrategicMerge: +- manager_image.yaml +- debug_logs_patch.yaml +- ../default/manager_auth_proxy_patch.yaml + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../crd +- ../rbac +- ../manager +images: +- name: testing + newName: testing-operator diff --git a/config/testing/manager_image.yaml b/config/testing/manager_image.yaml new file mode 100644 index 000000000..e44f542d9 --- /dev/null +++ b/config/testing/manager_image.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + image: testing diff --git a/config/testing/pull_policy/Always.yaml b/config/testing/pull_policy/Always.yaml new file mode 100644 index 000000000..6b0a8e2a8 --- /dev/null +++ b/config/testing/pull_policy/Always.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: Always diff --git a/config/testing/pull_policy/IfNotPresent.yaml b/config/testing/pull_policy/IfNotPresent.yaml new file mode 100644 index 000000000..2f52f496c --- /dev/null +++ b/config/testing/pull_policy/IfNotPresent.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: IfNotPresent diff --git a/config/testing/pull_policy/Never.yaml b/config/testing/pull_policy/Never.yaml new file mode 100644 index 000000000..86f13d816 --- /dev/null +++ b/config/testing/pull_policy/Never.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: Never diff --git a/deploy/10_service_account.yaml b/deploy/10_service_account.yaml deleted file mode 100644 index 92cc0365b..000000000 --- a/deploy/10_service_account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: benchmark-operator - namespace: my-ripsaw diff --git a/deploy/20_role.yaml b/deploy/20_role.yaml deleted file mode 100644 index 12b1be6b7..000000000 --- a/deploy/20_role.yaml +++ /dev/null @@ -1,112 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - creationTimestamp: null - name: benchmark-operator - namespace: my-ripsaw -rules: -- apiGroups: - - "" - resources: - - pods - - daemonsets - - services - - endpoints - - persistentvolumeclaims - - virtualmachineinstances - - events - - configmaps - - secrets - - pods/log - verbs: - - '*' -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - apps - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - - deployments/finalizers - verbs: - - '*' -- apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - create -- apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - verbs: - - '*' -- apiGroups: - - ripsaw.cloudbulldozer.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - batch - - extensions - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use - resourceNames: - - privileged -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use -- apiGroups: - - hyperfoil.io - resources: - - hyperfoils - verbs: - - '*' -- apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - k8s.cni.cncf.io - resources: - - network-attachment-definitions - verbs: - - get - - list - - watch diff --git a/deploy/25_role.yaml b/deploy/25_role.yaml deleted file mode 100644 index 4284753e1..000000000 --- a/deploy/25_role.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: benchmark-operator-role -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - patch diff --git a/deploy/40_cluster_role_kubevirt.yaml b/deploy/40_cluster_role_kubevirt.yaml deleted file mode 100644 index 31c2cd6db..000000000 --- a/deploy/40_cluster_role_kubevirt.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: benchmark-operator-kube -rules: -- apiGroups: - - rbac.authorization.k8s.io - resources: - - roles - - rolebindings - - clusterroles - - clusterrolebindings - - daemonsets - verbs: - - '*' -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - get - - list -- apiGroups: - - rbac.authorization.k8s.io - resources: - - nodes - verbs: - - get - - list -- apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachineinstances/console - - virtualmachineinstances/vnc - verbs: - - get -- apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - - virtualmachines - - virtualmachineinstancepresets - - virtualmachineinstancereplicasets - verbs: - - get - - delete - - create - - update - - patch - - list - - watch - - deletecollection -- apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubevirt-config - verbs: - - update - - get - - patch diff --git a/docs/byowl.md b/docs/byowl.md index 5bc775e6c..fbb614813 100644 --- a/docs/byowl.md +++ b/docs/byowl.md @@ -12,7 +12,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: byowl diff --git a/docs/cerberus.md b/docs/cerberus.md index 4bf99e8aa..777d345c1 100644 --- a/docs/cerberus.md +++ b/docs/cerberus.md @@ -38,7 +38,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://foo.bar.com:9200" @@ -55,7 +55,7 @@ Once Cerberus is enabled the connection status can be viewed by getting the benc enabled the connection status will simply be "not connected" ``` -$ kubectl -n my-ripsaw get benchmarks.ripsaw.cloudbulldozer.io +$ kubectl -n benchmark-operator get benchmarks.ripsaw.cloudbulldozer.io NAME TYPE STATE METADATA STATE CERBERUS UUID AGE byowl-benchmark byowl Error Complete Connected 11faec65-4009-58e8-ac36-0233f0fc822d 10m ``` diff --git a/docs/cyclictest.md b/docs/cyclictest.md index f665de0ce..59eebc51b 100644 --- a/docs/cyclictest.md +++ b/docs/cyclictest.md @@ -21,7 +21,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: cyclictest - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: server: @@ -51,7 +51,7 @@ You can look at the results from the cyclictest benchmark by doing ```bash NAME=cyclictest-workload-xxxxxxx -oc logs -n my-ripsaw $NAME +oc logs -n benchmark-operator $NAME ``` ## Cleanup diff --git a/docs/elastic.md b/docs/elastic.md index 6f44b5dcf..3e11ce0c1 100644 --- a/docs/elastic.md +++ b/docs/elastic.md @@ -72,7 +72,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: smallfile-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: test_user: homer_simpson clustername: test_ci @@ -118,7 +118,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: smallfile-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: test_user: homer_simpson clustername: test_ci @@ -158,9 +158,9 @@ cluster-monitoring-view ,or use the Prometheus user account's token. Command to setup a service account and retrieve token: ``` -oc create serviceaccount snafu -n my-ripsaw -oc create clusterrolebinding grafana-cluster-monitoring-view --clusterrole=cluster-monitoring-view --serviceaccount=my-ripsaw:snafu -oc sa get-token snafu -n my-ripsaw +oc create serviceaccount snafu -n benchmark-operator +oc create clusterrolebinding grafana-cluster-monitoring-view --clusterrole=cluster-monitoring-view --serviceaccount=benchmark-operator:snafu +oc sa get-token snafu -n benchmark-operator ``` *NOTE:* User tokens can expire, for long running test it is recommend to either set the necessary permissions diff --git a/docs/fio_distributed.md b/docs/fio_distributed.md index fddf09d10..13e1490a5 100644 --- a/docs/fio_distributed.md +++ b/docs/fio_distributed.md @@ -19,7 +19,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: fio-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://my.es.server:9200" diff --git a/docs/flent.md b/docs/flent.md index 23f21bad5..9675c8c37 100644 --- a/docs/flent.md +++ b/docs/flent.md @@ -12,7 +12,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: flent-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster #elasticsearch: @@ -244,7 +244,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster test_user: test_user # user is a key that points to user triggering ripsaw, useful to search results in ES @@ -279,7 +279,7 @@ By default we will utilize the `flent-results` index for Elasticsearch. Deploying the above(assuming pairs is set to 1) would result in ```bash -# oc get pods -n my-ripsaw +# oc get pods -n benchmark-operator NAME READY STATUS RESTARTS AGE benchmark-operator-f84bdbd8f-n6cnc 3/3 Running 0 34m flent-bench-client-10.116.0.54-533b6892-dc5cd 0/1 Completed 0 32m diff --git a/docs/fs-drift.md b/docs/fs-drift.md index 445544c1b..9a26cc99f 100644 --- a/docs/fs-drift.md +++ b/docs/fs-drift.md @@ -18,7 +18,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: fs-drift - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: fs-drift diff --git a/docs/hammerdb.md b/docs/hammerdb.md index 04322ada7..7d8151668 100644 --- a/docs/hammerdb.md +++ b/docs/hammerdb.md @@ -15,8 +15,8 @@ An example CR might look like this apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: hammerdb-benchmark - namespace: my-ripsaw + name: hammerdb + namespace: benchmark-operator spec: elasticsearch: url: http://my.elasticsearch.server:80 diff --git a/docs/image_pull.md b/docs/image_pull.md index 12b4a7908..88007a201 100644 --- a/docs/image_pull.md +++ b/docs/image_pull.md @@ -33,7 +33,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: image-pull - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://es-instance.com:9200" diff --git a/docs/installation.md b/docs/installation.md deleted file mode 100644 index 5c6e4887e..000000000 --- a/docs/installation.md +++ /dev/null @@ -1,120 +0,0 @@ -## Installation -This guide uses minikube version v1.5.2+ as the local Kubernetes cluster -and quay.io for the public registry. We also test on crc version v1.7.0+. -`kubectl` is used but can be substituted with `oc`. - -### Tested versions -* [OpenShift® Container Platform](https://www.openshift.com/products/container-platform/) - * Tested on: 4.3 and later -* [kubernetes](https://kubernetes.io/) - * Tested on: 1.16.2 and later - -Note: -* Experimental tag refers to some workloads that might be functioning -* To use versions of Openshift and kubernetes prior to 4.3 and 1.16.2 respectively, please use version 0.0.2 of ripsaw - -### Requirements - - -The main requirements are as follows: -* Running Kubernetes cluster - [supported versions](#Supported-Versions) -* [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -* [git](https://git-scm.com/downloads) - -Note: Please login as admin user - -The following requirements are needed to make/test changes to operator: -* [operator-sdk](https://github.com/operator-framework/operator-sdk) -* [docker](https://docs.docker.com/install/) - -Note: You also require a [quay](https://quay.io/) account to push images - -The following optional requirements are needed if using OKD/OCP < 4.0: -* [dep](https://golang.github.io/dep/docs/installation.html) -* [operator-sdk](https://github.com/operator-framework/operator-sdk) -* [go](https://golang.org/dl/) -* [OLM](https://github.com/operator-framework/operator-lifecycle-manager) - -The workloads could also have their own requirements which would be specified -in the installation guide. - -### Deploying operator -Note: The benchmark-operator's code-name is ripsaw, so the names have been -used interchangeably. - -Note: If you're on a vanilla k8s distribution, then you can also deploy Ripsaw through - operatorhub.io, please check [ripsaw in operatorhub](https://operatorhub.io/operator/ripsaw) for more details. - -First we'll need to clone the operator: - -```bash -# git clone https://github.com/cloud-bulldozer/benchmark-operator.git -# cd ripsaw -# export KUBECONFIG= # if not already done -``` - -We try to maintain all resources created/required by ripsaw in the namespace `my-ripsaw`, -as this would be the namespace, ripsaw would be installed into if deployed through operatorhub.io. - -Note: But in case you've a specific usecase where you want the resources to be in a different namespace, you'll just need to edit the namespace in deploy/ -as well as the operator definition. - -But for sake of the documentation, let's proceed with the namespace `my-ripsaw` - -so we'll create the namespace as follows - -```bash -# kubectl apply -f resources/namespace.yaml -``` - -or if you're used to `oc` it'd be `oc new-project my-ripsaw` and `oc project my-ripsaw` - -We'll now apply the permissions and operator definitions. - -```bash -# kubectl apply -f deploy -# kubectl apply -f resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -# kubectl apply -f resources/operator.yaml -``` - -### Running workload -Now that we've deployed our operator, follow workload specific instructions to -run workloads: -* [uperf](uperf.md) -* [fio](fio_distributed.md) -* [sysbench](sysbench.md) -* [YCSB](ycsb.md) -* [Bring your own workload](byowl.md) -* [pgbench](pgbench.md) -* [fs-drift](fs-drift.md) -* [servicemesh](servicemesh.md) -* [cyclictest](cyclictest.md) -* [oslat](oslat.md) - -If you want to add a new workload please follow these [instructions](../CONTRIBUTE.md#Add-workload) to submit a PR - -### Clean up -Now that we're running workloads we can cleanup by running following commands - -```bash -# kubectl delete -f -# kubectl delete -f resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -# kubectl delete -f resources/operator.yaml -# kubectl delete -f deploy -``` - -## running CI - -If you want to run CI on your laptop as part of developing a PR, you may want to use your own image location and account (i.e. your -own image repository account). To do this, set 2 environment variables: - -* RIPSAW_CI_IMAGE_LOCATION - host where image repository lives (default is quay.io) -* RIPSAW_CI_IMAGE_ACCOUNT - user account (default is rht_perf_ci). - -This allows you to have the CI run on your own private image built with your PR. This assumes that your benchmark's CI -script in tests/ utilizes the common code in tests/common.sh to launch ripsaw. - -You can modify your ripsaw image to use a test version of your benchmark image as well. For examples, see workload.yml.j2 files in the roles/ tree and look for the image: tag. diff --git a/docs/iperf.md b/docs/iperf.md index 3d50ff0f9..146215b88 100644 --- a/docs/iperf.md +++ b/docs/iperf.md @@ -13,7 +13,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: iperf3-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: name: iperf3 args: diff --git a/docs/log_generator.md b/docs/log_generator.md index be6332964..0f6c45bca 100644 --- a/docs/log_generator.md +++ b/docs/log_generator.md @@ -66,7 +66,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: log-generator - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://es-instance.com:9200" @@ -93,7 +93,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: log-generator - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://es-instance.com:9200" @@ -122,7 +122,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: log-generator - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://es-instance.com:9200" diff --git a/docs/metadata.md b/docs/metadata.md index 9e2d2282b..3388828fc 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -112,7 +112,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://my.elastic.server.foo:9200" @@ -165,7 +165,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: byowl @@ -181,7 +181,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: metadata: collection: true @@ -202,7 +202,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://my.elastic.server.foo:9200" @@ -293,13 +293,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: backpack-view - namespace: my-ripsaw + namespace: benchmark-operator --- apiVersion: v1 kind: Secret metadata: name: backpack-view - namespace: my-ripsaw + namespace: benchmark-operator annotations: kubernetes.io/service-account.name: backpack-view type: kubernetes.io/service-account-token @@ -308,7 +308,7 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: backpack-view - namespace: my-ripsaw + namespace: benchmark-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -316,7 +316,7 @@ roleRef: subjects: - kind: ServiceAccount name: backpack-view - namespace: my-ripsaw + namespace: benchmark-operator ``` Once the `backpack-view` service account is created you can modify the default backpack service account setting: diff --git a/docs/oslat.md b/docs/oslat.md index 60fc1694b..6e2449a8c 100644 --- a/docs/oslat.md +++ b/docs/oslat.md @@ -20,7 +20,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: oslat - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: server: @@ -50,7 +50,7 @@ You can look at the results from the oslat benchmark by doing ```bash NAME=oslat-workload-xxxxxxx -oc logs -n my-ripsaw $NAME +oc logs -n benchmark-operator $NAME ``` ## Cleanup diff --git a/docs/pgbench.md b/docs/pgbench.md index 9621f2c0f..689e19a7f 100644 --- a/docs/pgbench.md +++ b/docs/pgbench.md @@ -16,7 +16,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: pgbench-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: "pgbench" diff --git a/docs/prometheus.md b/docs/prometheus.md index ccb65d9f2..b4fa1b502 100644 --- a/docs/prometheus.md +++ b/docs/prometheus.md @@ -43,7 +43,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: smallfile-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: test_user: homer_simpson clustername: test_ci diff --git a/docs/scale_openshift.md b/docs/scale_openshift.md index dd365b7d8..a393b7e7d 100644 --- a/docs/scale_openshift.md +++ b/docs/scale_openshift.md @@ -35,7 +35,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: scale - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://es-instance.com:9200" @@ -86,13 +86,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: scaler - namespace: my-ripsaw + namespace: benchmark-operator --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: scaler - namespace: my-ripsaw + namespace: benchmark-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -100,5 +100,5 @@ roleRef: subjects: - kind: ServiceAccount name: scaler - namespace: my-ripsaw + namespace: benchmark-operator ``` diff --git a/docs/servicemesh.md b/docs/servicemesh.md index 1c4f6766b..1c3e405a9 100644 --- a/docs/servicemesh.md +++ b/docs/servicemesh.md @@ -29,7 +29,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: servicemesh @@ -71,7 +71,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: servicemesh-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: servicemesh @@ -117,7 +117,7 @@ While the output from benchmark is a JSON, you can easily display ```bash NAME=servicemesh-benchmark-xxxxxxx -oc logs -n my-ripsaw $NAME > /tmp/$NAME.json +oc logs -n benchmark-operator $NAME > /tmp/$NAME.json cat /tmp/$NAME.json | docker run -i --rm quay.io/hyperfoil/hyperfoil-report /opt/report.sh > /tmp/$NAME.html ``` diff --git a/docs/smallfile.md b/docs/smallfile.md index 2c3a21e89..9667e6f47 100644 --- a/docs/smallfile.md +++ b/docs/smallfile.md @@ -10,7 +10,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: smallfile @@ -31,7 +31,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: smallfile diff --git a/docs/stressng.md b/docs/stressng.md index 0f055a0ad..72b43b727 100644 --- a/docs/stressng.md +++ b/docs/stressng.md @@ -18,7 +18,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: stressng - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://es-instance.com:9200" diff --git a/docs/sysbench.md b/docs/sysbench.md index 0983ee7e7..8dfc13f2f 100644 --- a/docs/sysbench.md +++ b/docs/sysbench.md @@ -21,7 +21,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: sysbench-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: sysbench diff --git a/docs/system-metrics.md b/docs/system-metrics.md index 7eb818f20..22df8f9db 100644 --- a/docs/system-metrics.md +++ b/docs/system-metrics.md @@ -23,7 +23,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: system_metrics: enabled: true diff --git a/docs/testpmd.md b/docs/testpmd.md index cbafff86b..337daa22d 100644 --- a/docs/testpmd.md +++ b/docs/testpmd.md @@ -53,7 +53,7 @@ spec: spoofChk: "on" trust: "on" resourceName: intelnics - networkNamespace: my-ripsaw + networkNamespace: benchmark-operator ``` @@ -103,7 +103,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: testpmd-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster workload: diff --git a/docs/uperf.md b/docs/uperf.md index 66955208b..0de22ed8c 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -12,7 +12,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: uperf-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://es-instance.com:9200" diff --git a/docs/vegeta.md b/docs/vegeta.md index e14205e29..e387d75cf 100644 --- a/docs/vegeta.md +++ b/docs/vegeta.md @@ -16,7 +16,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: vegeta-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://esinstance.com:9200" diff --git a/docs/ycsb.md b/docs/ycsb.md index 346b4396c..d916158a1 100644 --- a/docs/ycsb.md +++ b/docs/ycsb.md @@ -15,7 +15,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: ycsb-mongo-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster workload: diff --git a/meta/main.yml b/meta/main.yml deleted file mode 100644 index 80a3c730f..000000000 --- a/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ -collections: -- community.kubernetes -- operator_sdk.util diff --git a/playbook.yml b/playbooks/benchmark.yml similarity index 79% rename from playbook.yml rename to playbooks/benchmark.yml index 3236e2b92..20551b759 100644 --- a/playbook.yml +++ b/playbooks/benchmark.yml @@ -12,13 +12,6 @@ name: "kernel_cache_drop" when: workload.args.drop_cache_kernel is defined - - name: Get update from Cerberus if connected - block: - - include_role: - name: "cerberus" - - when: cerberus_url is defined and cerberus_url != "" - - name: Get state k8s_facts: @@ -34,7 +27,7 @@ api_version: v1 namespace: "{{ operator_namespace }}" label_selectors: - - name = benchmark-operator + - control-plane = controller-manager register: bo - debug: @@ -42,21 +35,25 @@ - name: Set Workload UUID block: - - include_role: - name: "uuid" - - - name: Setting the uuid for the benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - uuid: "{{ uuid }}" - complete: false - suuid: "{{ trunc_uuid }}" - metadata: "not collected" - cerberus: "not connected" + - include_role: + name: "uuid" + + - name: Setting the uuid for the benchmark + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + uuid: "{{ uuid }}" + complete: false + suuid: "{{ trunc_uuid }}" + metadata: "not collected" + cerberus: "not connected" + rescue: + - include_role: + name: benchmark_state + tasks_from: failure when: workload is defined and (cr_state.resources[0].status is not defined or cr_state.resources[0].status.uuid is not defined) @@ -66,8 +63,11 @@ trunc_uuid: "{{ cr_state.resources[0].status.suuid }}" - name: Run Workload + rescue: + - include_role: + name: benchmark_state + tasks_from: failure block: - - block: - include_role: @@ -101,6 +101,7 @@ - include_role: name: system-metrics + vars: workload_args: "{{ workload.args }}" when: diff --git a/group_vars/all.yml b/playbooks/group_vars/all.yml similarity index 100% rename from group_vars/all.yml rename to playbooks/group_vars/all.yml diff --git a/templates/metadata.yml.j2 b/playbooks/templates/metadata.yml.j2 similarity index 100% rename from templates/metadata.yml.j2 rename to playbooks/templates/metadata.yml.j2 diff --git a/requirements.yml b/requirements.yml index 8a661f8b9..3f626ed2f 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,5 +1,6 @@ --- collections: - name: community.kubernetes - version: "<1.0.0" - - operator_sdk.util + version: "1.2.1" + - name: operator_sdk.util + version: "0.2.0" diff --git a/resources/backpack_role.yaml b/resources/backpack_role.yaml index 59e81076d..a0f8ea69f 100644 --- a/resources/backpack_role.yaml +++ b/resources/backpack_role.yaml @@ -32,13 +32,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: backpack-view - namespace: my-ripsaw + namespace: benchmark-operator --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: binding-backpack_role - namespace: my-ripsaw + namespace: benchmark-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -46,4 +46,4 @@ roleRef: subjects: - kind: ServiceAccount name: backpack-view - namespace: my-ripsaw + namespace: benchmark-operator diff --git a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server.yaml b/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server.yaml deleted file mode 100644 index ceed5fb1f..000000000 --- a/resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server.yaml +++ /dev/null @@ -1,86 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: mariadb-db ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: mariadb-custom-config - namespace: mariadb-db -data: - custom.conf: | - # This group is read both both by the client and the server - # use it for options that affect everything - # - [client-server] - # - # include all files from the config directory - # - !includedir /etc/my.cnf.d - port=3306 - ssl=0 - [mysqld] - innodb_file_per_table=1 - innodb_flush_method=O_DIRECT - innodb_adaptive_flushing=1 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mariadb-deployment - namespace: mariadb-db -spec: - selector: - matchLabels: - app: mariadb - template: - metadata: - labels: - app: mariadb - type: mariadb-database-server - spec: - nodeSelector: - kubernetes.io/hostname: "node1" - terminationGracePeriodSeconds: 10 - containers: - - name: mariadb - image: centos/mariadb-103-centos8:latest - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 3306 - resources: - requests: - cpu: 200m - memory: 100Mi - limits: - cpu: 4 - memory: 16Gi - env: - - name: MYSQL_USER - value: "test" - - name: MYSQL_PASSWORD - value: "test" - - name: MYSQL_ROOT_PASSWORD - value: "mysql" - volumeMounts: - - name: mariadb-custom-config - mountPath: /etc/my.cnf - subPath: custom.conf #should be the name used in the ConfigMap - volumes: - - name: mariadb-custom-config - configMap: - name: mariadb-custom-config ---- -apiVersion: v1 -kind: Service -metadata: - name: mariadb-deployment - namespace: mariadb-db -spec: - selector: - app: mariadb - ports: - - protocol: TCP - port: 3306 - targetPort: 3306 diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml deleted file mode 100644 index d293a476c..000000000 --- a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: mssql-db ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mssql-deployment - namespace: mssql-db -spec: - replicas: 1 - selector: - matchLabels: - app: mssql - template: - metadata: - labels: - app: mssql - type: mssql-database-server - spec: - nodeSelector: - kubernetes.io/hostname: "node1" - terminationGracePeriodSeconds: 10 - containers: - - name: mssql - image: mcr.microsoft.com/mssql/rhel/server:2019-latest - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 1433 - resources: - requests: - cpu: 200m - memory: 100Mi - limits: - cpu: 4 - memory: 16Gi - env: - - name: MSSQL_PID - value: "Enterprise" - - name: ACCEPT_EULA - value: "Y" - - name: MSSQL_SA_PASSWORD - value: "s3curePasswordString" ---- -apiVersion: v1 -kind: Service -metadata: - name: mssql-deployment - namespace: mssql-db -spec: - selector: - app: mssql - ports: - - protocol: TCP - port: 1433 - targetPort: 1433 - - diff --git a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml b/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml deleted file mode 100644 index 828e885cf..000000000 --- a/resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: mssql-db ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mssql-deployment - namespace: mssql-db -spec: - replicas: 1 - selector: - matchLabels: - app: mssql - template: - metadata: - labels: - app: mssql - type: mssql-database-server - spec: - nodeSelector: - kubernetes.io/hostname: "worker-0" - terminationGracePeriodSeconds: 10 - containers: - - name: mssql - image: mcr.microsoft.com/mssql/rhel/server:2019-latest - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 1433 - resources: - requests: - cpu: 200m - memory: 100Mi - limits: - cpu: 4 - memory: 16Gi - env: - - name: MSSQL_PID - value: "Enterprise" - - name: ACCEPT_EULA - value: "Y" - - name: MSSQL_SA_PASSWORD - value: "s3curePasswordString" - volumeMounts: - - name: mssql-persistent-storage - mountPath: /var/opt/mssql - readOnly: false - volumes: - - name: mssql-persistent-storage - persistentVolumeClaim: - claimName: mssql-persistent-storage ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: mssql-persistent-storage - namespace: mssql-db -spec: - storageClassName: ocs-storagecluster-ceph-rbd - accessModes: [ "ReadWriteOnce" ] - volumeMode: Filesystem - resources: - requests: - storage: 10Gi ---- -apiVersion: v1 -kind: Service -metadata: - name: mssql-deployment - namespace: mssql-db -spec: - selector: - app: mssql - ports: - - protocol: TCP - port: 1433 - targetPort: 1433 diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml deleted file mode 100644 index f996efec3..000000000 --- a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml +++ /dev/null @@ -1,81 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: postgres-db ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: postgres-custom-config - namespace: postgres-db -data: - custom.pg_hba.conf: | - # TYPE DATABASE USER ADDRESS METHOD - # "local" is for Unix domain socket connections only - local all all trust - # IPv4 local connections: - host all all all trust ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgres-deployment - namespace: postgres-db -spec: - selector: - matchLabels: - app: postgres - replicas: 1 - template: - metadata: - labels: - app: postgres - type: postgres-database-server - spec: - nodeSelector: - kubernetes.io/hostname: "node1" - terminationGracePeriodSeconds: 10 - containers: - - name: postgres - image: centos/postgresql-10-centos8:latest - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 5432 - resources: - requests: - cpu: 200m - memory: 100Mi - limits: - cpu: 4 - memory: 16Gi - env: - - name: POSTGRESQL_USER - value: "test" - - name: POSTGRESQL_PASSWORD - value: "test" - - name: POSTGRESQL_DATABASE - value: "test" - - name: POSTGRESQL_ADMIN_PASSWORD - value: "postgres" - volumeMounts: - - name: postgres-custom-config - mountPath: /var/lib/pgsql/data/pg_hba.conf - subPath: custom.pg_hba.conf #should be the name used in the ConfigMap - volumes: - - name: postgres-custom-config - configMap: - name: postgres-custom-config - ---- -apiVersion: v1 -kind: Service -metadata: - name: postgres-deployment - namespace: postgres-db -spec: - selector: - app: postgres - ports: - - protocol: TCP - port: 5432 - targetPort: 5432 diff --git a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml b/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml deleted file mode 100644 index 1a215f4c4..000000000 --- a/resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml +++ /dev/null @@ -1,99 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: postgres-db ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: postgres-custom-config - namespace: postgres-db -data: - custom.pg_hba.conf: | - # TYPE DATABASE USER ADDRESS METHOD - # "local" is for Unix domain socket connections only - local all all trust - # IPv4 local connections: - host all all all trust ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgres-deployment - namespace: postgres-db -spec: - selector: - matchLabels: - app: postgres - replicas: 1 - template: - metadata: - labels: - app: postgres - type: postgres-database-server - spec: - nodeSelector: - kubernetes.io/hostname: "worker-0" - terminationGracePeriodSeconds: 10 - containers: - - name: postgres - image: centos/postgresql-10-centos8:latest - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 5432 - resources: - requests: - cpu: 200m - memory: 100Mi - limits: - cpu: 4 - memory: 16Gi - env: - - name: POSTGRESQL_USER - value: "test" - - name: POSTGRESQL_PASSWORD - value: "test" - - name: POSTGRESQL_DATABASE - value: "test" - - name: POSTGRESQL_ADMIN_PASSWORD - value: "postgres" - volumeMounts: - - name: postgres-custom-config - mountPath: /var/lib/pgsql/data/pg_hba.conf - subPath: custom.pg_hba.conf #should be the name used in the ConfigMap - - name: postgres-persistent-storage - mountPath: /var/lib/pgsql/data - readOnly: false - volumes: - - name: postgres-custom-config - configMap: - name: postgres-custom-config - - name: postgres-persistent-storage - persistentVolumeClaim: - claimName: postgres-persistent-storage ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: postgres-persistent-storage - namespace: postgres-db -spec: - storageClassName: ocs-storagecluster-ceph-rbd - accessModes: [ "ReadWriteOnce" ] - volumeMode: Filesystem - resources: - requests: - storage: 10Gi ---- -apiVersion: v1 -kind: Service -metadata: - name: postgres-deployment - namespace: postgres-db -spec: - selector: - app: postgres - ports: - - protocol: TCP - port: 5432 - targetPort: 5432 \ No newline at end of file diff --git a/resources/kernel-cache-drop-clusterrole.yaml b/resources/kernel-cache-drop-clusterrole.yaml deleted file mode 100644 index 4fd0acd22..000000000 --- a/resources/kernel-cache-drop-clusterrole.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: benchmark-operator -rules: -- apiGroups: - - '' - resources: - - nodes - - pods - verbs: - - get - - list -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: benchmark-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: benchmark-operator -subjects: -- kind: ServiceAccount - name: benchmark-operator - namespace: my-ripsaw diff --git a/resources/kernel-cache-drop-daemonset.yaml b/resources/kernel-cache-drop-daemonset.yaml index fbf9c4b30..dd1a43ab6 100644 --- a/resources/kernel-cache-drop-daemonset.yaml +++ b/resources/kernel-cache-drop-daemonset.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: kernel-cache-dropper - namespace: my-ripsaw + namespace: benchmark-operator labels: app: kernel-cache-dropper spec: @@ -14,6 +14,8 @@ spec: labels: name: kernel-cache-dropper spec: + nodeSelector: + kernel-cache-dropper: "yes" serviceAccountName: benchmark-operator #tolerations: #- key: node-role.kubernetes.io/master @@ -29,7 +31,6 @@ spec: value: "{{ kernel_cache_drop_svc_port }}" command: ["/usr/bin/python3"] args: ["/opt/kernel_cache_drop/kernel-cache-drop-websvc.py"] - imagePullPolicy: Always securityContext: privileged: true # we don't need all the same volumes as the toolbox pod diff --git a/resources/kube-burner-role.yml b/resources/kube-burner-role.yml index 32bb9d57c..281f5ddf7 100644 --- a/resources/kube-burner-role.yml +++ b/resources/kube-burner-role.yml @@ -2,13 +2,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: kube-burner - namespace: my-ripsaw + namespace: benchmark-operator --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: binding-kube-burner - namespace: my-ripsaw + namespace: benchmark-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -16,4 +16,4 @@ roleRef: subjects: - kind: ServiceAccount name: kube-burner - namespace: my-ripsaw + namespace: benchmark-operator diff --git a/resources/namespace.yaml b/resources/namespace.yaml deleted file mode 100644 index 0908738fc..000000000 --- a/resources/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: my-ripsaw diff --git a/resources/operator.yaml b/resources/operator.yaml deleted file mode 100644 index d057ead85..000000000 --- a/resources/operator.yaml +++ /dev/null @@ -1,70 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: benchmark-operator - namespace: my-ripsaw -spec: - replicas: 1 - selector: - matchLabels: - name: benchmark-operator - template: - metadata: - labels: - name: benchmark-operator - spec: - tolerations: - - key: role - value: workload - effect: NoSchedule - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: node-role.kubernetes.io/workload - operator: In - values: - - "" - serviceAccountName: benchmark-operator - containers: - - name: benchmark-operator - image: quay.io/benchmark-operator/benchmark-operator:master - imagePullPolicy: Always - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPERATOR_NAME - value: "benchmark-operator" - - name: WORKER_BENCHMARK_RIPSAW_CLOUDBULLDOZER_IO - value: "1" - - name: ANSIBLE_VERBOSITY - value: "4" - volumeMounts: - - mountPath: /tmp/ansible-operator/runner - name: runner - - name: redis-master - image: k8s.gcr.io/redis:v1 - env: - - name: MASTER - value: "true" - ports: - - containerPort: 6379 - resources: - limits: - cpu: "2.0" - volumeMounts: - - mountPath: /redis-master-data - name: data - volumes: - - name: data - emptyDir: {} - - name: runner - emptyDir: {} diff --git a/resources/scale_role.yaml b/resources/scale_role.yaml index 2d4ca4c76..e0d7fe5b0 100644 --- a/resources/scale_role.yaml +++ b/resources/scale_role.yaml @@ -17,13 +17,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: scaler - namespace: my-ripsaw + namespace: benchmark-operator --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: scaler - namespace: my-ripsaw + namespace: benchmark-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -31,4 +31,4 @@ roleRef: subjects: - kind: ServiceAccount name: scaler - namespace: my-ripsaw + namespace: benchmark-operator diff --git a/resources/self_provisioner_binding.yaml b/resources/self_provisioner_binding.yaml index ae3a7484b..fbc43c503 100644 --- a/resources/self_provisioner_binding.yaml +++ b/resources/self_provisioner_binding.yaml @@ -9,4 +9,4 @@ roleRef: subjects: - kind: ServiceAccount name: benchmark-operator - namespace: my-ripsaw + namespace: benchmark-operator diff --git a/roles/benchmark_state/tasks/failure.yml b/roles/benchmark_state/tasks/failure.yml new file mode 100644 index 000000000..619205a39 --- /dev/null +++ b/roles/benchmark_state/tasks/failure.yml @@ -0,0 +1,15 @@ +--- +- name: Failure State + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + uuid: "{{ uuid }}" + complete: True + state: Failed + message: "Reconcile failed on {{ ansible_failed_task.name }} {{ ansible_failed_task.action }} \n ```{{ ansible_failed_result | to_nice_json }}```" + suuid: "{{ trunc_uuid }}" + metadata: "not collected" + cerberus: "not connected" \ No newline at end of file diff --git a/roles/flent/tasks/main.yml b/roles/flent/tasks/main.yml index e34c35852..f71db76bb 100644 --- a/roles/flent/tasks/main.yml +++ b/roles/flent/tasks/main.yml @@ -26,15 +26,6 @@ namespace: '{{ operator_namespace }}' register: resource_state -- name: Capture operator information - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - name = benchmark-operator - register: bo - - block: - name: Start Server(s) diff --git a/roles/fs-drift/tasks/main.yml b/roles/fs-drift/tasks/main.yml index 8c23bbe6f..d40dc8e40 100644 --- a/roles/fs-drift/tasks/main.yml +++ b/roles/fs-drift/tasks/main.yml @@ -8,7 +8,8 @@ namespace: '{{ operator_namespace }}' register: resource_state -- operator_sdk.util.k8s_status: +- name: Set state to starting + operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark name: "{{ ansible_operator_meta.name }}" @@ -68,14 +69,6 @@ subscriber: "{{ lookup('template', 'subscriber.py.j2') }}" publisher: "{{ lookup('template', 'publisher.py.j2') }}" - - name: Capture operator information - k8s_facts: - kind: Pod - api_version: v1 - namespace: "{{ operator_namespace }}" - label_selectors: - - name = benchmark-operator - register: bo - name: Run PUBLISHER Pod k8s: diff --git a/roles/fs-drift/templates/workload_job.yml.j2 b/roles/fs-drift/templates/workload_job.yml.j2 index 4f743d4ea..0878fb28f 100644 --- a/roles/fs-drift/templates/workload_job.yml.j2 +++ b/roles/fs-drift/templates/workload_job.yml.j2 @@ -2,7 +2,7 @@ kind: Job apiVersion: batch/v1 metadata: - name: "{{ ansible_operator_meta.name }}-fs-drift-client-benchmark-{{ trunc_uuid }}-pod-{{ item }}" + name: "{{ ansible_operator_meta.name }}-client-{{ trunc_uuid }}-{{ item }}" namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 diff --git a/roles/hammerdb/tasks/main.yaml b/roles/hammerdb/tasks/main.yaml index c3046ca6b..13f9892fb 100644 --- a/roles/hammerdb/tasks/main.yaml +++ b/roles/hammerdb/tasks/main.yaml @@ -338,7 +338,7 @@ name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: - state: DB Workload Complete + state: Complete complete: true when: "hammerdb_workload_pod | json_query('resources[].status.succeeded')" when: resource_kind == "pod" @@ -354,7 +354,7 @@ name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: - state: DB Workload Complete + state: Complete complete: true when: complete_status.stdout == "true" when: resource_kind == "vm" diff --git a/roles/kube-burner/tasks/main.yml b/roles/kube-burner/tasks/main.yml index 4066c6ab9..e00a41c10 100644 --- a/roles/kube-burner/tasks/main.yml +++ b/roles/kube-burner/tasks/main.yml @@ -26,14 +26,6 @@ namespace: '{{ operator_namespace }}' register: resource_state -- name: Capture operator information - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - name = benchmark-operator - register: bo - block: diff --git a/roles/pgbench/tasks/main.yml b/roles/pgbench/tasks/main.yml index 2bea5e9d3..ecf657537 100644 --- a/roles/pgbench/tasks/main.yml +++ b/roles/pgbench/tasks/main.yml @@ -25,15 +25,6 @@ namespace: '{{ operator_namespace }}' register: resource_state -- name: Capture benchmark operator information - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - name = benchmark-operator - register: bo - - name: Generate list of database counts include_tasks: generate_num_databases.yml diff --git a/roles/pgbench/tasks/prep_workload.yml b/roles/pgbench/tasks/prep_workload.yml index b886672a4..fa4f17ae8 100644 --- a/roles/pgbench/tasks/prep_workload.yml +++ b/roles/pgbench/tasks/prep_workload.yml @@ -7,7 +7,7 @@ - name: Setup pgbench test job(s) k8s: - definition: "{{ lookup('template', 'templates/workload.yml.j2') }}" + definition: "{{ lookup('template', 'workload.yml.j2') }}" with_indexed_items: "{{ workload_args.databases }}" when: item.0|int in range(dbnum|int) diff --git a/roles/smallfile/tasks/main.yml b/roles/smallfile/tasks/main.yml index d18148aa9..01ca474e2 100644 --- a/roles/smallfile/tasks/main.yml +++ b/roles/smallfile/tasks/main.yml @@ -68,15 +68,6 @@ subscriber: "{{ lookup('template', 'subscriber.py.j2') }}" publisher: "{{ lookup('template', 'publisher.py.j2') }}" - - name: Capture operator information - k8s_facts: - kind: Pod - api_version: v1 - namespace: "{{ operator_namespace }}" - label_selectors: - - name = benchmark-operator - register: bo - - name: Run PUBLISHER Pod k8s: definition: diff --git a/roles/stressng/tasks/main.yaml b/roles/stressng/tasks/main.yaml index c11e878fa..3dfeb0d5c 100644 --- a/roles/stressng/tasks/main.yaml +++ b/roles/stressng/tasks/main.yaml @@ -87,14 +87,14 @@ register: stressng_workload_pod - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha + api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: - state: Benchmark Complete + state: Complete complete: true - when: stressng_workload_pod | json_query('resources[].status.succeeded') + when: stressng_workload_pod | json_query('resources[].status.succeeded') when: resource_kind == "pod" - block: @@ -107,7 +107,7 @@ name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: - state: Benchmark Complete + state: Complete complete: true when: complete_status.stdout == "true" when: resource_kind == "vm" diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 1f99e4764..2270fe2b9 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -25,15 +25,6 @@ name: '{{ ansible_operator_meta.name }}' namespace: '{{ operator_namespace }}' register: resource_state - -- name: Capture operator information - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - name = benchmark-operator - register: bo - block: # diff --git a/roles/vegeta/tasks/main.yml b/roles/vegeta/tasks/main.yml index f9b01d614..ff648fce2 100644 --- a/roles/vegeta/tasks/main.yml +++ b/roles/vegeta/tasks/main.yml @@ -26,14 +26,6 @@ namespace: '{{ operator_namespace }}' register: resource_state -- name: Capture operator information - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - name = benchmark-operator - register: bo - block: - name: Create targets configmap diff --git a/test.sh b/test.sh index 4b37ddb78..d64d95fa3 100755 --- a/test.sh +++ b/test.sh @@ -16,6 +16,13 @@ do esac done +if ! command -v yq &> /dev/null +then + echo "yq not installed, installing" + wget https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_linux_amd64 -O /usr/local/bin/yq && chmod +x /usr/local/bin/yq +fi + + source tests/common.sh eStatus=0 @@ -69,21 +76,6 @@ UUID=${NEW_UUID%-*} tag_name="${NODE_NAME:-master}" sed -i "s#ES_SERVER#$ES_SERVER#g" tests/test_crs/* -sed -i "s/sql-server/sql-server-$UUID/g" tests/mssql.yaml tests/test_crs/valid_hammerdb.yaml tests/test_hammerdb.sh -sed -i "s/benchmarks.ripsaw.cloudbulldozer.io/benchmarks-$UUID.ripsaw.cloudbulldozer.io/g" resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -sed -i "s/kind: Benchmark/kind: Benchmark-$UUID/g" resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -sed -i "s/listKind: BenchmarkList/listKind: BenchmarkList-$UUID/g" resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -sed -i "s/plural: benchmarks/plural: benchmarks-$UUID/g" resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -sed -i "s/singular: benchmark/singular: benchmark-$UUID/g" resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -sed -i "s/benchmarks/benchmarks-$UUID/g" tests/common.sh -sed -i "s/kind: Benchmark/kind: Benchmark-$UUID/g" tests/test_crs/*.yaml -sed -i "s/kind: Benchmark/kind: Benchmark-$UUID/g" playbook.yml -sed -i "s/kind: Benchmark/kind: Benchmark-$UUID/g" watches.yaml -sed -i "s/backpack_role/backpack_role-$UUID/g" resources/backpack_role.yaml -sed -i "s/my-ripsaw/my-ripsaw-$UUID-test-fiod/g" resources/kernel-cache-drop-daemonset.yaml -grep -Rl "kind: Benchmark" roles/ | xargs sed -i "s/kind: Benchmark/kind: Benchmark-$UUID/g" -sed -i "s| image: quay.io/benchmark-operator/benchmark-operator:master*| image: $image_location/$image_account/benchmark-operator:$tag_name # |" resources/operator.yaml - cp -pr * gold/ # Create individual directories for each test @@ -91,15 +83,12 @@ for ci_dir in `cat tests/my_tests` do mkdir $ci_dir cp -pr gold/* $ci_dir/ - cd $ci_dir/ - # Edit the namespaces so we can run in parallel - sed -i "s/my-ripsaw/my-ripsaw-$UUID-$ci_dir/g" `grep -Rl my-ripsaw` - sed -i "s/benchmark-operator-role/benchmark-operator-role-$UUID-$ci_dir/g" `grep -Rl benchmark-operator-role` - cd .. done -# Update the operator image -update_operator_image +oc delete namespace benchmark-operator --ignore-not-found +kubectl delete benchmarks -n benchmark-operator --ignore-not-found --all +apply_operator + # Run scale test first if it is in the test list scale_test="false" diff --git a/tests/check_es.py b/tests/check_es.py old mode 100644 new mode 100755 diff --git a/tests/common.sh b/tests/common.sh old mode 100644 new mode 100755 index 57c4953db..2f07ed0f3 --- a/tests/common.sh +++ b/tests/common.sh @@ -9,75 +9,83 @@ echo "using container image location $image_location and account $image_account" function populate_test_list { rm -f tests/iterate_tests - for item in $@ - do - # Check for changes in roles - if [[ $(echo ${item} | grep 'roles/fs-drift') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/uperf') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/fio_distributed') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/iperf3') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/byowl') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/sysbench') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/pgbench') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/ycsb') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/backpack') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/hammerdb') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/smallfile') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/vegeta') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/stressng') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/scale_openshift') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/kube-burner') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/flent') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/log_generator') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/image_pull') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi - - - # Check for changes in cr files - if [[ $(echo ${item} | grep 'valid_backpack*') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_byowl*') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_fiod*') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_fs_drift*') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_hammerdb*') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_iperf3*') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_pgbench*') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_smallfile*') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_sysbench*') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_uperf*') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_ycsb*') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_vegeta*') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_stressng*') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_scale*') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_kube-burner*') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_flent*') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_log_generator*') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_image_pull*') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi - - - # Check for changes in test scripts - test_check=`echo $item | awk -F / '{print $2}'` - - if [[ $(echo ${test_check} | grep 'test_.*.sh') ]]; then echo ${test_check} >> tests/iterate_tests; fi - done + if [ -n $1 ]; then + echo $1 >> tests/iterate_tests + else + + for item in $@ + do + # Check for changes in roles + if [[ $(echo ${item} | grep 'roles/fs-drift') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/uperf') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/fio_distributed') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/iperf3') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/byowl') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/sysbench') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/pgbench') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/ycsb') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/backpack') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/hammerdb') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/smallfile') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/vegeta') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/stressng') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/scale_openshift') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/kube-burner') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/flent') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/log_generator') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/image_pull') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi + + + # Check for changes in cr files + if [[ $(echo ${item} | grep 'valid_backpack*') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_byowl*') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_fiod*') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_fs_drift*') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_hammerdb*') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_iperf3*') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_pgbench*') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_smallfile*') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_sysbench*') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_uperf*') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_ycsb*') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_vegeta*') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_stressng*') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_scale*') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_kube-burner*') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_flent*') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_log_generator*') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_image_pull*') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi + + + # Check for changes in test scripts + test_check=`echo $item | awk -F / '{print $2}'` + + if [[ $(echo ${test_check} | grep 'test_.*.sh') ]]; then echo ${test_check} >> tests/iterate_tests; fi + done + fi +} + +function delete_benchmark { + kubectl delete -f $1 --ignore-not-found=true } function wait_clean { if [[ `kubectl get benchmarks.ripsaw.cloudbulldozer.io --all-namespaces` ]] then - kubectl delete benchmarks -n my-ripsaw --all --ignore-not-found + echo "skipping cleanup" fi - kubectl delete -f deploy/25_role.yaml -f deploy/35_role_binding.yaml --ignore-not-found - kubectl delete namespace my-ripsaw --ignore-not-found } # The argument is 'timeout in seconds' function get_uuid () { - sleep_time=$1 + sleep_time=20 + benchmark_name=$1 sleep $sleep_time counter=0 counter_max=6 uuid="False" until [ $uuid != "False" ] ; do - uuid=$(kubectl -n my-ripsaw get benchmarks -o jsonpath='{.items[0].status.uuid}') + uuid=$(kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath='{.status.uuid}') if [ -z $uuid ]; then sleep $sleep_time uuid="False" @@ -100,7 +108,7 @@ function get_pod () { pod_name="False" until [ $pod_name != "False" ] ; do sleep $sleep_time - pod_name=$(kubectl get pods -l $1 --namespace ${3:-my-ripsaw} -o name | cut -d/ -f2) + pod_name=$(kubectl get pods -l $1 --namespace ${3:-benchmark-operator} -o name | cut -d/ -f2) if [ -z $pod_name ]; then pod_name="False" fi @@ -122,7 +130,7 @@ function pod_count () { export $1 until [ $pod_count == $2 ] ; do sleep $sleep_time - pod_count=$(kubectl get pods -n my-ripsaw -l $1 -o name | wc -l) + pod_count=$(kubectl get pods -n benchmark-operator -l $1 -o name | wc -l) if [ -z $pod_count ]; then pod_count=0 fi @@ -136,53 +144,15 @@ function pod_count () { } function apply_operator { - operator_requirements - BENCHMARK_OPERATOR_IMAGE=${BENCHMARK_OPERATOR_IMAGE:-"quay.io/benchmark-operator/benchmark-operator:master"} - cat resources/operator.yaml | \ - sed 's#quay.io/benchmark-operator/benchmark-operator:master#'$BENCHMARK_OPERATOR_IMAGE'#' | \ - kubectl apply -f - - kubectl wait --for=condition=available "deployment/benchmark-operator" -n my-ripsaw --timeout=300s + tag_name="${NODE_NAME:-master}" + make podman-build podman-push deploy IMG=$image_location/$image_account/benchmark-operator:$tag_name + kubectl wait --for=condition=available "deployment/benchmark-controller-manager" -n benchmark-operator --timeout=300s } function delete_operator { - kubectl delete -f resources/operator.yaml -} - -function marketplace_setup { - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/01_namespace.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/02_catalogsourceconfig.crd.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/03_operatorsource.crd.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/04_service_account.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/05_role.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/06_role_binding.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/07_upstream_operatorsource.cr.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/08_operator.yaml -} - -function marketplace_cleanup { - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/07_upstream_operatorsource.cr.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/08_operator.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/06_role_binding.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/05_role.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/04_service_account.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/03_operatorsource.crd.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/02_catalogsourceconfig.crd.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/01_namespace.yaml + make undeploy } -function operator_requirements { - kubectl apply -f resources/namespace.yaml - kubectl apply -f deploy - kubectl apply -f resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml - kubectl -n my-ripsaw get roles - kubectl -n my-ripsaw get rolebindings - kubectl -n my-ripsaw get podsecuritypolicies - kubectl -n my-ripsaw get serviceaccounts - kubectl -n my-ripsaw get serviceaccount benchmark-operator -o yaml - kubectl -n my-ripsaw get role benchmark-operator -o yaml - kubectl -n my-ripsaw get rolebinding benchmark-operator -o yaml - kubectl -n my-ripsaw get podsecuritypolicy privileged -o yaml -} function backpack_requirements { kubectl apply -f resources/backpack_role.yaml @@ -190,48 +160,21 @@ function backpack_requirements { then if [[ `oc get securitycontextconstraints.security.openshift.io` ]] then - oc adm policy -n my-ripsaw add-scc-to-user privileged -z benchmark-operator - oc adm policy -n my-ripsaw add-scc-to-user privileged -z backpack-view + oc adm policy -n benchmark-operator add-scc-to-user privileged -z benchmark-operator + oc adm policy -n benchmark-operator add-scc-to-user privileged -z backpack-view fi fi } -function create_operator { - operator_requirements - apply_operator -} - -function cleanup_resources { - echo "Exiting after cleanup of resources" - kubectl delete -f resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml - kubectl delete -f deploy -} function cleanup_operator_resources { delete_operator - cleanup_resources wait_clean } -function update_operator_image { - tag_name="${NODE_NAME:-master}" - if podman build -f build/Dockerfile -t $image_location/$image_account/benchmark-operator:$tag_name .; then - # In case we have issues uploading to quay we will retry a few times - for i in {1..3}; do - podman push $image_location/$image_account/benchmark-operator:$tag_name && break - echo "Could not upload image to quay. Exiting" - exit 1 - done - else - echo "Could not build image. Exiting" - exit 1 - fi - sed -i "s| image: quay.io/benchmark-operator/benchmark-operator:master*| image: $image_location/$image_account/benchmark-operator:$tag_name # |" resources/operator.yaml -} - function check_log(){ for i in {1..10}; do - if kubectl logs -f $1 --namespace my-ripsaw | grep -q $2 ; then + if kubectl logs -f $1 --namespace benchmark-operator | grep -q $2 ; then break; else sleep 10 @@ -239,8 +182,53 @@ function check_log(){ done } +function get_benchmark_name() { + benchmark_file=$1 + echo $(yq e '.metadata.name' $benchmark_file) +} + +function get_benchmark_state() { + benchmark_name=$1 + kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath={.status.state} +} + +function wait_for_benchmark() { + benchmark_name=$1 + desired_state=$2 + until [[ $(get_benchmark_state $benchmark_name) == "$desired_state" ]]; do + if [[ $(get_benchmark_state $benchmark_name) == "Failed" ]]; then + echo "Benchmark $benchmark_name failed" + return 1 + fi + sleep 5 + done +} + +function check_benchmark_for_desired_state(){ + benchmark_name=$1 + desired_state=$2 + timeout=${3:-500s} + export -f wait_for_benchmark + export -f get_benchmark_state + if ! timeout -k $timeout $timeout bash -c "wait_for_benchmark $benchmark_name $desired_state" + then + echo "Timeout exceeded for: "$benchmark_name + + counter=3 + until [ $counter -gt $# ] + do + echo "Logs from "${@:$counter} + kubectl -n benchmark-operator logs --tail=40 ${@:$counter} + counter=$(( counter+1 )) + done + return 1 + fi + echo $(kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath={.status.state}) + +} + # Takes 2 or more arguments: 'command to run', 'time to wait until true' -# Any additional arguments will be passed to kubectl -n my-ripsaw logs to provide logging if a timeout occurs +# Any additional arguments will be passed to kubectl -n benchmark-operator logs to provide logging if a timeout occurs function wait_for() { if ! timeout -k $2 $2 $1 then @@ -250,7 +238,7 @@ function wait_for() { until [ $counter -gt $# ] do echo "Logs from "${@:$counter} - kubectl -n my-ripsaw logs --tail=40 ${@:$counter} + kubectl -n benchmark-operator logs --tail=40 ${@:$counter} counter=$(( counter+1 )) done return 1 @@ -268,9 +256,11 @@ function error { echo "Error caught. Dumping logs before exiting" echo "Benchmark operator Logs" - kubectl -n my-ripsaw logs --tail=200 -l name=benchmark-operator -c benchmark-operator + kubectl -n benchmark-operator logs --tail=200 -l control-plane=controller-manager -c manager } + + function wait_for_backpack() { echo "Waiting for backpack to complete before starting benchmark test" @@ -279,10 +269,10 @@ function wait_for_backpack() { max_count=60 while [[ $count -lt $max_count ]] do - if [[ `kubectl -n my-ripsaw get daemonsets backpack-$uuid` ]] + if [[ `kubectl -n benchmark-operator get daemonsets backpack-$uuid` ]] then - desired=`kubectl -n my-ripsaw get daemonsets backpack-$uuid | grep -v NAME | awk '{print $2}'` - ready=`kubectl -n my-ripsaw get daemonsets backpack-$uuid | grep -v NAME | awk '{print $4}'` + desired=`kubectl -n benchmark-operator get daemonsets backpack-$uuid | grep -v NAME | awk '{print $2}'` + ready=`kubectl -n benchmark-operator get daemonsets backpack-$uuid | grep -v NAME | awk '{print $4}'` if [[ $desired -eq $ready ]] then echo "Backpack complete. Starting benchmark" diff --git a/tests/full_test_file_trigger b/tests/full_test_file_trigger old mode 100644 new mode 100755 index 6af13e9b3..7859a425c --- a/tests/full_test_file_trigger +++ b/tests/full_test_file_trigger @@ -1,8 +1,4 @@ build/Dockerfile -deploy/10_service_account.yaml -deploy/20_role.yaml -deploy/30_role_binding.yaml -deploy/40_cluster_role_kubevirt.yaml group_vars/all.yml resources/operator.yaml tests/common.sh diff --git a/tests/mssql.yaml b/tests/mssql.yaml old mode 100644 new mode 100755 diff --git a/tests/test_backpack.sh b/tests/test_backpack.sh old mode 100644 new mode 100755 index 0d3707133..28c1d6b8e --- a/tests/test_backpack.sh +++ b/tests/test_backpack.sh @@ -18,11 +18,12 @@ trap error ERR trap finish EXIT function functional_test_backpack { - wait_clean - apply_operator backpack_requirements - kubectl apply -f $1 - long_uuid=$(get_uuid 20) + cr=$1 + delete_benchmark $cr + kubectl apply -f $cr + benchmark_name=$(get_benchmark_name $cr) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} if [[ $1 == "tests/test_crs/valid_backpack_daemonset.yaml" ]] @@ -30,8 +31,8 @@ function functional_test_backpack { wait_for_backpack $uuid else byowl_pod=$(get_pod "app=byowl-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized pods/$byowl_pod --timeout=500s" "500s" $byowl_pod - wait_for "kubectl -n my-ripsaw wait --for=condition=complete -l app=byowl-$uuid jobs --timeout=500s" "500s" $byowl_pod + wait_for "kubectl -n benchmark-operator wait --for=condition=Initialized pods/$byowl_pod --timeout=500s" "500s" $byowl_pod + wait_for "kubectl -n benchmark-operator wait --for=condition=complete -l app=byowl-$uuid jobs --timeout=500s" "500s" $byowl_pod fi indexes="cpu_vulnerabilities-metadata cpuinfo-metadata dmidecode-metadata k8s_nodes-metadata lspci-metadata meminfo-metadata sysctl-metadata ocp_network_operator-metadata ocp_install_config-metadata ocp_kube_apiserver-metadata ocp_dns-metadata ocp_kube_controllermanager-metadata" @@ -42,6 +43,7 @@ function functional_test_backpack { echo "Failed to find data in ES" exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_byowl.sh b/tests/test_byowl.sh old mode 100644 new mode 100755 index 8edd2d15c..b210c9f44 --- a/tests/test_byowl.sh +++ b/tests/test_byowl.sh @@ -10,26 +10,22 @@ function finish { fi echo "Cleaning up byowl" - kubectl delete -f tests/test_crs/valid_byowl.yaml - delete_operator + wait_clean } trap error ERR trap finish EXIT function functional_test_byowl { - wait_clean - apply_operator token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" tests/test_crs/valid_byowl.yaml | kubectl apply -f - - long_uuid=$(get_uuid 20) - uuid=${long_uuid:0:8} + cr=tests/test_crs/valid_byowl.yaml - byowl_pod=$(get_pod "app=byowl-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized pods/$byowl_pod --timeout=500s" "500s" $byowl_pod - wait_for "kubectl -n my-ripsaw wait --for=condition=complete -l app=byowl-$uuid jobs --timeout=300s" "300s" $byowl_pod - kubectl -n my-ripsaw logs "$byowl_pod" | grep "Test" + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + check_benchmark_for_desired_state $benchmark_name Complete 500s echo "BYOWL test: Success" + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_crs/valid_backpack_daemonset.yaml b/tests/test_crs/valid_backpack_daemonset.yaml index c67788eeb..97fb74314 100644 --- a/tests/test_crs/valid_backpack_daemonset.yaml +++ b/tests/test_crs/valid_backpack_daemonset.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: backpack - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: ES_SERVER diff --git a/tests/test_crs/valid_backpack_init.yaml b/tests/test_crs/valid_backpack_init.yaml index 2842f769e..6886d3faa 100644 --- a/tests/test_crs/valid_backpack_init.yaml +++ b/tests/test_crs/valid_backpack_init.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: backpack - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: ES_SERVER diff --git a/tests/test_crs/valid_byowl.yaml b/tests/test_crs/valid_byowl.yaml index bcbc45f7a..621d08dc8 100644 --- a/tests/test_crs/valid_byowl.yaml +++ b/tests/test_crs/valid_byowl.yaml @@ -2,12 +2,11 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: - prometheus: - system_metrics_collection: true + system_metrics: + collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - system_metrics_index: system-metrics es_url: ES_SERVER prom_token: PROMETHEUS_TOKEN metrics_profile: node-metrics.yml diff --git a/tests/test_crs/valid_fiod.yaml b/tests/test_crs/valid_fiod.yaml index a11ef4386..9df7dfe1c 100644 --- a/tests/test_crs/valid_fiod.yaml +++ b/tests/test_crs/valid_fiod.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: fiod-benchmark + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_fiod_bsrange.yaml b/tests/test_crs/valid_fiod_bsrange.yaml index d2f798f7f..f88ee8a40 100644 --- a/tests/test_crs/valid_fiod_bsrange.yaml +++ b/tests/test_crs/valid_fiod_bsrange.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: fiod-bsrange-benchmark + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_fiod_hostpath.yaml b/tests/test_crs/valid_fiod_hostpath.yaml index 053198618..79092c6db 100644 --- a/tests/test_crs/valid_fiod_hostpath.yaml +++ b/tests/test_crs/valid_fiod_hostpath.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: fiod-hostpath-benchmark + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_fiod_ocs_cache_drop.yaml b/tests/test_crs/valid_fiod_ocs_cache_drop.yaml index 47f3d0fcc..2ebba6a0e 100644 --- a/tests/test_crs/valid_fiod_ocs_cache_drop.yaml +++ b/tests/test_crs/valid_fiod_ocs_cache_drop.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: fiod-ocs-benchmark + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_flent.yaml b/tests/test_crs/valid_flent.yaml index 966c549ef..4d17d4ce4 100644 --- a/tests/test_crs/valid_flent.yaml +++ b/tests/test_crs/valid_flent.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: flent-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_flent_resources.yaml b/tests/test_crs/valid_flent_resources.yaml index 81aabf6f2..bef6ba7ab 100644 --- a/tests/test_crs/valid_flent_resources.yaml +++ b/tests/test_crs/valid_flent_resources.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: flent-benchmark - namespace: my-ripsaw + name: flent-resources-benchmark + namespace: benchmark-operator spec: clustername: myk8scluster system_metrics: diff --git a/tests/test_crs/valid_fs_drift.yaml b/tests/test_crs/valid_fs_drift.yaml index ceb7cd6fc..9ccd180ce 100644 --- a/tests/test_crs/valid_fs_drift.yaml +++ b/tests/test_crs/valid_fs_drift.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: fs-drift-benchmark + namespace: benchmark-operator spec: test_user: homer_simpson # to separate this test run from everyone else's diff --git a/tests/test_crs/valid_fs_drift_hostpath.yaml b/tests/test_crs/valid_fs_drift_hostpath.yaml index 3983b219d..91c8f65bb 100644 --- a/tests/test_crs/valid_fs_drift_hostpath.yaml +++ b/tests/test_crs/valid_fs_drift_hostpath.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: fs-drift-hostpath-benchmark + namespace: benchmark-operator spec: test_user: homer_simpson # to separate this test run from everyone else's diff --git a/tests/test_crs/valid_hammerdb.yaml b/tests/test_crs/valid_hammerdb.yaml index ed7ef873f..6c3d1b7f3 100644 --- a/tests/test_crs/valid_hammerdb.yaml +++ b/tests/test_crs/valid_hammerdb.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: hammerdb - namespace: my-ripsaw + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_image_pull.yaml b/tests/test_crs/valid_image_pull.yaml index ffdc92ec7..c39cf5e63 100644 --- a/tests/test_crs/valid_image_pull.yaml +++ b/tests/test_crs/valid_image_pull.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: image-pull - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: ES_SERVER diff --git a/tests/test_crs/valid_iperf3.yaml b/tests/test_crs/valid_iperf3.yaml index 7660fd256..d0ebe5b91 100644 --- a/tests/test_crs/valid_iperf3.yaml +++ b/tests/test_crs/valid_iperf3.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: iperf3-benchmark + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_kube-burner.yaml b/tests/test_crs/valid_kube-burner.yaml index 839dac7cc..7efa673d8 100644 --- a/tests/test_crs/valid_kube-burner.yaml +++ b/tests/test_crs/valid_kube-burner.yaml @@ -3,7 +3,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: kube-burner-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: ES_SERVER diff --git a/tests/test_crs/valid_log_generator.yaml b/tests/test_crs/valid_log_generator.yaml index cf3e56e44..72b926db2 100644 --- a/tests/test_crs/valid_log_generator.yaml +++ b/tests/test_crs/valid_log_generator.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: log-generator - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: ES_SERVER diff --git a/tests/test_crs/valid_pgbench.yaml b/tests/test_crs/valid_pgbench.yaml index 485a5f12a..908a4e58b 100644 --- a/tests/test_crs/valid_pgbench.yaml +++ b/tests/test_crs/valid_pgbench.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: pgbench-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_scale_down.yaml b/tests/test_crs/valid_scale_down.yaml index e6d451f8d..4b1d03b04 100644 --- a/tests/test_crs/valid_scale_down.yaml +++ b/tests/test_crs/valid_scale_down.yaml @@ -1,9 +1,11 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: scale - namespace: my-ripsaw + name: scale-down + namespace: benchmark-operator spec: + metadata: + collection: true system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 diff --git a/tests/test_crs/valid_scale_up.yaml b/tests/test_crs/valid_scale_up.yaml index 0fd5ed785..e1f1aceff 100644 --- a/tests/test_crs/valid_scale_up.yaml +++ b/tests/test_crs/valid_scale_up.yaml @@ -1,9 +1,11 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: scale - namespace: my-ripsaw + name: scale-up + namespace: benchmark-operator spec: + metadata: + collection: true system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 diff --git a/tests/test_crs/valid_servicemesh.yaml b/tests/test_crs/valid_servicemesh.yaml index 7f7b93147..c41fab396 100644 --- a/tests/test_crs/valid_servicemesh.yaml +++ b/tests/test_crs/valid_servicemesh.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: servicemesh-benchmark + namespace: benchmark-operator spec: workload: name: servicemesh diff --git a/tests/test_crs/valid_smallfile.yaml b/tests/test_crs/valid_smallfile.yaml index 71efff93c..8d2dd480b 100644 --- a/tests/test_crs/valid_smallfile.yaml +++ b/tests/test_crs/valid_smallfile.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: smallfile-benchmark + namespace: benchmark-operator spec: test_user: homer_simpson # to separate this test run from everyone else's diff --git a/tests/test_crs/valid_smallfile_hostpath.yaml b/tests/test_crs/valid_smallfile_hostpath.yaml index 9d0e32477..daf1c273b 100644 --- a/tests/test_crs/valid_smallfile_hostpath.yaml +++ b/tests/test_crs/valid_smallfile_hostpath.yaml @@ -2,8 +2,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: smallfile-hostpath-benchmark + namespace: benchmark-operator spec: test_user: homer_simpson # to separate this test run from everyone else's diff --git a/tests/test_crs/valid_stressng.yaml b/tests/test_crs/valid_stressng.yaml index 19b9ecf8b..220004b18 100644 --- a/tests/test_crs/valid_stressng.yaml +++ b/tests/test_crs/valid_stressng.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: stressng - namespace: my-ripsaw + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_sysbench.yaml b/tests/test_crs/valid_sysbench.yaml index d291501ca..9bd1aca1d 100644 --- a/tests/test_crs/valid_sysbench.yaml +++ b/tests/test_crs/valid_sysbench.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: sysbench-benchmark + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_uperf.yaml b/tests/test_crs/valid_uperf.yaml index e023a3989..a0ff00c5e 100644 --- a/tests/test_crs/valid_uperf.yaml +++ b/tests/test_crs/valid_uperf.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: uperf - namespace: my-ripsaw + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_uperf_networkpolicy.yaml b/tests/test_crs/valid_uperf_networkpolicy.yaml index 286a8e73a..54cb87e1f 100644 --- a/tests/test_crs/valid_uperf_networkpolicy.yaml +++ b/tests/test_crs/valid_uperf_networkpolicy.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: uperf - namespace: my-ripsaw + name: uperf-networkpolicy + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_uperf_resources.yaml b/tests/test_crs/valid_uperf_resources.yaml index 148c1f889..da3f60392 100644 --- a/tests/test_crs/valid_uperf_resources.yaml +++ b/tests/test_crs/valid_uperf_resources.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: uperf - namespace: my-ripsaw + name: uperf-resources + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_uperf_serviceip.yaml b/tests/test_crs/valid_uperf_serviceip.yaml index 83fb356b6..9837bbe7c 100644 --- a/tests/test_crs/valid_uperf_serviceip.yaml +++ b/tests/test_crs/valid_uperf_serviceip.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: uperf - namespace: my-ripsaw + name: uperf-serviceip + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_vegeta.yaml b/tests/test_crs/valid_vegeta.yaml index fb6a93368..14a1d6fdc 100644 --- a/tests/test_crs/valid_vegeta.yaml +++ b/tests/test_crs/valid_vegeta.yaml @@ -3,7 +3,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: vegeta-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_vegeta_hostnetwork.yaml b/tests/test_crs/valid_vegeta_hostnetwork.yaml index 4ea03f4e7..6d3f1ecb8 100644 --- a/tests/test_crs/valid_vegeta_hostnetwork.yaml +++ b/tests/test_crs/valid_vegeta_hostnetwork.yaml @@ -3,7 +3,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: vegeta-benchmark-hostnetwork - namespace: my-ripsaw + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_crs/valid_ycsb-mongo.yaml b/tests/test_crs/valid_ycsb-mongo.yaml index 91ea4c469..3e8a56cda 100644 --- a/tests/test_crs/valid_ycsb-mongo.yaml +++ b/tests/test_crs/valid_ycsb-mongo.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: ycsb-mongo-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: system_metrics: collection: true diff --git a/tests/test_fiod.sh b/tests/test_fiod.sh index 042a764dc..78fb7d8af 100755 --- a/tests/test_fiod.sh +++ b/tests/test_fiod.sh @@ -10,7 +10,6 @@ function finish { fi echo "Cleaning up fio" - kubectl delete -f resources/kernel-cache-drop-clusterrole.yaml --ignore-not-found wait_clean } @@ -19,32 +18,34 @@ trap finish EXIT function functional_test_fio { wait_clean - apply_operator - kubectl apply -f resources/kernel-cache-drop-clusterrole.yaml test_name=$1 cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr echo "Performing: ${test_name}" token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - kubectl apply -f ${cr} - long_uuid=$(get_uuid 20) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} pod_count "app=fio-benchmark-$uuid" 2 300 - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=fio-benchmark-$uuid pods --timeout=300s" "300s" fio_pod=$(get_pod "app=fiod-client-$uuid" 300) - wait_for "kubectl wait --for=condition=Initialized pods/$fio_pod -n my-ripsaw --timeout=500s" "500s" $fio_pod - wait_for "kubectl wait --for=condition=complete -l app=fiod-client-$uuid jobs -n my-ripsaw --timeout=700s" "700s" $fio_pod - kubectl -n my-ripsaw logs $fio_pod > /tmp/$fio_pod.log + check_benchmark_for_desired_state $benchmark_name Complete 1200s + kubectl -n benchmark-operator logs $fio_pod > /tmp/$fio_pod.log + + + + indexes="ripsaw-fio-results ripsaw-fio-log ripsaw-fio-analyzed-result" if check_es "${long_uuid}" "${indexes}" then echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs "$fio_pod" -n my-ripsaw + kubectl logs "$fio_pod" -n benchmark-operator exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_flent.sh b/tests/test_flent.sh index 4602dd519..84d059cfe 100755 --- a/tests/test_flent.sh +++ b/tests/test_flent.sh @@ -17,22 +17,17 @@ trap error ERR trap finish EXIT function functional_test_flent { - wait_clean - apply_operator test_name=$1 cr=$2 + delete_benchmark $cr echo "Performing: ${test_name}" + benchmark_name=$(get_benchmark_name $cr) token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid 20) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} - pod_count "type=flent-bench-server-$uuid" 1 900 - flent_server_pod=$(get_pod "app=flent-bench-server-0-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=flent-bench-server-0-$uuid pods --timeout=300s" "300s" $flent_server_pod - flent_client_pod=$(get_pod "app=flent-bench-client-$uuid" 900) - wait_for "kubectl wait -n my-ripsaw --for=condition=Initialized pods/$flent_client_pod --timeout=500s" "500s" $flent_client_pod - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=flent-bench-client-$uuid jobs --timeout=500s" "500s" $flent_client_pod + check_benchmark_for_desired_state $benchmark_name Complete 800s index="ripsaw-flent-results" if check_es "${long_uuid}" "${index}" @@ -40,9 +35,10 @@ function functional_test_flent { echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs "$flent_client_pod" -n my-ripsaw + kubectl logs "$flent_client_pod" -n benchmark-operator exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_fs_drift.sh b/tests/test_fs_drift.sh index 67072a454..ac7d26d17 100755 --- a/tests/test_fs_drift.sh +++ b/tests/test_fs_drift.sh @@ -18,40 +18,27 @@ trap error ERR trap finish EXIT function functional_test_fs_drift { - wait_clean - apply_operator test_name=$1 cr=$2 + delete_benchmark $cr echo "Performing: ${test_name}" + benchmark_name=$(get_benchmark_name $cr) token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid 20) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} - - count=0 - while [[ $count -lt 24 ]]; do - if [[ `kubectl get pods -l app=fs-drift-benchmark-$uuid --namespace my-ripsaw -o name | cut -d/ -f2 | grep client` ]]; then - fsdrift_pod=$(kubectl get pods -l app=fs-drift-benchmark-$uuid --namespace my-ripsaw -o name | cut -d/ -f2 | grep client) - count=30 - fi - if [[ $count -ne 30 ]]; then - sleep 5 - count=$((count + 1)) - fi - done - echo fsdrift_pod $fs_drift_pod - wait_for "kubectl wait --for=condition=Initialized pods/$fsdrift_pod -n my-ripsaw --timeout=500s" "500s" $fsdrift_pod - wait_for "kubectl wait --for=condition=complete -l app=fs-drift-benchmark-$uuid jobs -n my-ripsaw --timeout=100s" "200s" $fsdrift_pod - + fsdrift_pod=$(get_pod "app=fs-drift-benchmark-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 600s indexes="ripsaw-fs-drift-results ripsaw-fs-drift-rsptimes ripsaw-fs-drift-rates-over-time" if check_es "${long_uuid}" "${indexes}" then echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs "$fsdrift_pod" -n my-ripsaw + kubectl logs "$fsdrift_pod" -n benchmark-operator exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_hammerdb.sh b/tests/test_hammerdb.sh index 6b6cbec38..78c7aa145 100755 --- a/tests/test_hammerdb.sh +++ b/tests/test_hammerdb.sh @@ -13,25 +13,24 @@ function initdb_pod { function finish { echo "Cleaning up hammerdb" kubectl delete -f tests/mssql.yaml - kubectl delete -f tests/test_crs/valid_hammerdb.yaml - delete_operator + wait_clean } trap finish EXIT function functional_test_hammerdb { - wait_clean - apply_operator initdb_pod token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" tests/test_crs/valid_hammerdb.yaml | kubectl apply -f - - long_uuid=$(get_uuid 20) + cr=tests/test_crs/valid_hammerdb.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} # Wait for the workload pod to run the actual workload hammerdb_workload_pod=$(get_pod "app=hammerdb_workload-$uuid" 300) - kubectl wait --for=condition=Initialized "pods/$hammerdb_workload_pod" --namespace my-ripsaw --timeout=400s - kubectl wait --for=condition=complete -l app=hammerdb_workload-$uuid --namespace my-ripsaw jobs --timeout=500s + check_benchmark_for_desired_state $benchmark_name Complete 900s index="ripsaw-hammerdb-results" if check_es "${long_uuid}" "${index}" @@ -39,9 +38,10 @@ function functional_test_hammerdb { echo "Hammerdb test: Success" else echo "Failed to find data for HammerDB test in ES" - kubectl logs "$hammerdb_workload_pod" --namespace my-ripsaw + kubectl logs "$hammerdb_workload_pod" --namespace benchmark-operator exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_image_pull.sh b/tests/test_image_pull.sh index 70f7259fd..07a47309b 100755 --- a/tests/test_image_pull.sh +++ b/tests/test_image_pull.sh @@ -17,18 +17,16 @@ trap error ERR trap finish EXIT function functional_test_image_pull { - wait_clean - apply_operator test_name=$1 cr=$2 - + delete_benchmark $cr echo "Performing: ${test_name}" kubectl apply -f ${cr} - long_uuid=$(get_uuid 20) + benchmark_name=$(get_benchmark_name $cr) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} - pod_count "app=image-pull-$uuid" 2 300 - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=image-pull-$uuid jobs --timeout=500s" "500s" + check_benchmark_for_desired_state $benchmark_name Complete 500s index="image-pull-results" if check_es "${long_uuid}" "${index}" @@ -38,7 +36,7 @@ function functional_test_image_pull { echo "Failed to find data for ${test_name} in ES" exit 1 fi - kubectl delete -f ${cr} + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_iperf3.sh b/tests/test_iperf3.sh old mode 100644 new mode 100755 index 25ce92a0e..18efecbe0 --- a/tests/test_iperf3.sh +++ b/tests/test_iperf3.sh @@ -10,31 +10,31 @@ function finish { fi echo "Cleaning up iperf3" - kubectl delete -f tests/test_crs/valid_iperf3.yaml - delete_operator + wait_clean } trap error ERR trap finish EXIT function functional_test_iperf { - wait_clean - apply_operator echo "Performing iperf3: ${1}" token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" -e "s/hostnetwork:.*/${1}/g" tests/test_crs/valid_iperf3.yaml | kubectl apply -f - - long_uuid=$(get_uuid 20) + cr=tests/test_crs/valid_iperf3.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" -e "s/hostnetwork:.*/${1}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} iperf_server_pod=$(get_pod "app=iperf3-bench-server-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=iperf3-bench-server-$uuid pods --timeout=300s" "300s" $iperf_server_pod + wait_for "kubectl -n benchmark-operator wait --for=condition=Initialized -l app=iperf3-bench-server-$uuid pods --timeout=300s" "300s" $iperf_server_pod iperf_client_pod=$(get_pod "app=iperf3-bench-client-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized pods/$iperf_client_pod --timeout=500s" "500s" $iperf_client_pod - wait_for "kubectl -n my-ripsaw wait --for=condition=complete -l app=iperf3-bench-client-$uuid jobs --timeout=100s" "100s" $iperf_client_pod + check_benchmark_for_desired_state $benchmark_name Complete 600s sleep 5 # ensuring that iperf actually ran and we can access metrics - kubectl logs "$iperf_client_pod" --namespace my-ripsaw | grep "iperf Done." + kubectl logs "$iperf_client_pod" --namespace benchmark-operator | grep "iperf Done." echo "iperf ${1}: Success" + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_kubeburner.sh b/tests/test_kubeburner.sh index 94f015591..88f57883f 100755 --- a/tests/test_kubeburner.sh +++ b/tests/test_kubeburner.sh @@ -9,7 +9,7 @@ function finish { error fi - [[ $check_logs == 1 ]] && kubectl logs -l app=kube-burner-benchmark-$uuid -n my-ripsaw + [[ $check_logs == 1 ]] && kubectl logs -l app=kube-burner-benchmark-$uuid -n benchmark-operator echo "Cleaning up kube-burner" kubectl delete -f resources/kube-burner-role.yml --ignore-not-found kubectl delete ns -l kube-burner-uuid=${long_uuid} @@ -25,18 +25,16 @@ function functional_test_kubeburner { metrics_profile=$2 token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) cr=tests/test_crs/valid_kube-burner.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) check_logs=0 - wait_clean - apply_operator kubectl apply -f resources/kube-burner-role.yml echo "Performing kube-burner: ${workload_name}" sed -e "s/WORKLOAD/${workload_name}/g" -e "s/PROMETHEUS_TOKEN/${token}/g" -e "s/METRICS_PROFILE/${metrics_profile}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid 20) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} - - pod_count "app=kube-burner-benchmark-$uuid" 1 900 check_logs=1 - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=kube-burner-benchmark-$uuid jobs --timeout=500s" "500s" + check_benchmark_for_desired_state $benchmark_name Complete 1800s index="ripsaw-kube-burner" if check_es "${long_uuid}" "${index}" @@ -47,6 +45,7 @@ function functional_test_kubeburner { exit 1 fi kubectl delete ns -l kube-burner-uuid=${long_uuid} + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_list b/tests/test_list old mode 100644 new mode 100755 diff --git a/tests/test_log_generator.sh b/tests/test_log_generator.sh index bd49169b6..3d394f981 100755 --- a/tests/test_log_generator.sh +++ b/tests/test_log_generator.sh @@ -17,19 +17,17 @@ trap error ERR trap finish EXIT function functional_test_log_generator { - wait_clean - apply_operator test_name=$1 cr=$2 - + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) echo "Performing: ${test_name}" kubectl apply -f ${cr} - long_uuid=$(get_uuid 20) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} log_gen_pod=$(get_pod "app=log-generator-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=log-generator-$uuid pods --timeout=300s" "300s" $log_gen_pod - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=log-generator-$uuid jobs --timeout=300s" "300s" $log_gen_pod + check_benchmark_for_desired_state $benchmark_name Complete 800s index="log-generator-results" if check_es "${long_uuid}" "${index}" @@ -37,10 +35,10 @@ function functional_test_log_generator { echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs "$log_gen_pod" -n my-ripsaw + kubectl logs "$log_gen_pod" -n benchmark-operator exit 1 fi - kubectl delete -f ${cr} + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_pgbench.sh b/tests/test_pgbench.sh old mode 100644 new mode 100755 index 7c589ca63..d8d33a249 --- a/tests/test_pgbench.sh +++ b/tests/test_pgbench.sh @@ -18,15 +18,13 @@ trap finish EXIT # Note we don't test persistent storage here function functional_test_pgbench { - wait_clean - apply_operator # stand up postgres deployment cat << EOF | kubectl apply -f - apiVersion: v1 kind: ConfigMap metadata: name: postgres-config - namespace: my-ripsaw + namespace: benchmark-operator labels: app: postgres data: @@ -40,7 +38,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: postgres - namespace: my-ripsaw + namespace: benchmark-operator spec: selector: matchLabels: @@ -70,18 +68,19 @@ EOF counter=0 until [[ $postgres_ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ || $counter -eq 10 ]]; do let counter+=1 - postgres_ip=$(kubectl get pod -n my-ripsaw $postgres_pod --template={{.status.podIP}}) + postgres_ip=$(kubectl get pod -n benchmark-operator $postgres_pod --template={{.status.podIP}}) sleep 2 done # deploy the test CR with the postgres pod IP - sed s/host:/host:\ ${postgres_ip}/ tests/test_crs/valid_pgbench.yaml | kubectl apply -f - - long_uuid=$(get_uuid 20) + cr=tests/test_crs/valid_pgbench.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed s/host:/host:\ ${postgres_ip}/ $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} pgbench_pod=$(get_pod "app=pgbench-client-$uuid" 300) - wait_for "kubectl wait --for=condition=Initialized pods/$pgbench_pod -n my-ripsaw --timeout=360s" "360s" $pgbench_pod - wait_for "kubectl wait --for=condition=Ready pods/$pgbench_pod -n my-ripsaw --timeout=60s" "60s" $pgbench_pod - wait_for "kubectl wait --for=condition=Complete jobs -l app=pgbench-client-$uuid -n my-ripsaw --timeout=300s" "300s" $pgbench_pod + check_benchmark_for_desired_state $benchmark_name Complete 500s index="ripsaw-pgbench-summary ripsaw-pgbench-raw" if check_es "${long_uuid}" "${index}" @@ -89,9 +88,10 @@ EOF echo "pgbench test: Success" else echo "Failed to find data for PGBench in ES" - kubectl logs -n my-ripsaw $pgbench_pod + kubectl logs -n benchmark-operator $pgbench_pod exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_scale_openshift.sh b/tests/test_scale_openshift.sh index e0b26b0d8..42b38ab79 100755 --- a/tests/test_scale_openshift.sh +++ b/tests/test_scale_openshift.sh @@ -18,33 +18,30 @@ trap error ERR trap finish EXIT function functional_test_scale_openshift { - wait_clean - apply_operator test_name=$1 cr=$2 - + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr # Apply scale role and service account kubectl apply -f resources/scale_role.yaml - + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) echo "Performing: ${test_name}" - kubectl apply -f ${cr} - long_uuid=$(get_uuid 20) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} scale_pod=$(get_pod "app=scale-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=scale-$uuid pods --timeout=300s" "300s" $scale_pod - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=scale-$uuid jobs --timeout=500s" "500s" $scale_pod - + check_benchmark_for_desired_state $benchmark_name Complete 500s index="openshift-cluster-timings" if check_es "${long_uuid}" "${index}" then echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs "$scale_pod" -n my-ripsaw + kubectl logs "$scale_pod" -n benchmark-operator exit 1 fi - kubectl delete -f ${cr} + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_servicemesh.sh b/tests/test_servicemesh.sh index 8b878fe7d..6f5bf18d0 100755 --- a/tests/test_servicemesh.sh +++ b/tests/test_servicemesh.sh @@ -14,17 +14,18 @@ function finish { uuid=${long_uuid:0:8} kubectl delete --ignore-not-found=true namespace example-benchmark-controlplane-$uuid kubectl delete --ignore-not-found=true namespace example-benchmark-workload-$uuid - kubectl delete -f tests/test_crs/valid_servicemesh.yaml - delete_operator + wait_clean } trap error ERR trap finish EXIT function functional_test_servicemesh { - apply_operator - kubectl apply -f tests/test_crs/valid_servicemesh.yaml - long_uuid=$(get_uuid 20) + cr=tests/test_crs/valid_servicemesh.yaml + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr + kubectl apply -f $cr + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} # wait until the job appears @@ -32,24 +33,25 @@ function functional_test_servicemesh { max_count=60 while [[ $count -lt $max_count ]] do - if kubectl get --namespace my-ripsaw job example-benchmark-$uuid; then + if kubectl get --namespace benchmark-operator job example-benchmark-$uuid; then break fi sleep 15 count=$((count + 1)) done - wait_for "kubectl -n my-ripsaw wait --for=condition=complete jobs --timeout=300s example-benchmark-$uuid" "300s" + wait_for "kubectl -n benchmark-operator wait --for=condition=complete jobs --timeout=300s example-benchmark-$uuid" "300s" job_pod=$(get_pod app=example-benchmark-$uuid 30) # ensuring that uperf actually ran and we can access metrics - INFO=$(kubectl logs $job_pod --namespace my-ripsaw | jq .info) + INFO=$(kubectl logs $job_pod --namespace benchmark-operator | jq .info) if [ -n "$INFO" ]; then echo "Successful: $INFO" else echo "Failed to verify benchmark results" exit 1 fi + delete_benchmark $cr } functional_test_servicemesh diff --git a/tests/test_smallfile.sh b/tests/test_smallfile.sh index 58e5c102b..b921adfb2 100755 --- a/tests/test_smallfile.sh +++ b/tests/test_smallfile.sh @@ -10,7 +10,6 @@ function finish { fi echo "Cleaning up smallfile" - kubectl delete -f resources/kernel-cache-drop-clusterrole.yaml --ignore-not-found wait_clean } @@ -19,31 +18,16 @@ trap error ERR trap finish EXIT function functional_test_smallfile { - wait_clean - apply_operator - kubectl apply -f resources/kernel-cache-drop-clusterrole.yaml test_name=$1 cr=$2 + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) echo "Performing: ${test_name}" token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid 20) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} - - count=0 - while [[ $count -lt 24 ]]; do - if [[ `kubectl get pods -l app=smallfile-benchmark-$uuid --namespace my-ripsaw -o name | cut -d/ -f2 | grep client` ]]; then - smallfile_pod=$(kubectl get pods -l app=smallfile-benchmark-$uuid --namespace my-ripsaw -o name | cut -d/ -f2 | grep client) - count=30 - fi - if [[ $count -ne 30 ]]; then - sleep 5 - count=$((count + 1)) - fi - done - echo "smallfile_pod ${smallfile_pod}" - wait_for "kubectl wait --for=condition=Initialized -l app=smallfile-benchmark-$uuid pods --namespace my-ripsaw --timeout=500s" "500s" - wait_for "kubectl wait --for=condition=complete -l app=smallfile-benchmark-$uuid jobs --namespace my-ripsaw --timeout=100s" "100s" + check_benchmark_for_desired_state $benchmark_name Complete 500s indexes="ripsaw-smallfile-results ripsaw-smallfile-rsptimes" if check_es "${long_uuid}" "${indexes}" @@ -51,11 +35,9 @@ function functional_test_smallfile { echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - for pod in ${smallfile_pod}; do - kubectl logs ${pod} --namespace my-ripsaw | grep "RUN STATUS" - done exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_stressng.sh b/tests/test_stressng.sh index cb15567bb..d9c98c05c 100755 --- a/tests/test_stressng.sh +++ b/tests/test_stressng.sh @@ -10,24 +10,24 @@ function finish { error fi echo "Cleaning up stressng" - kubectl delete -f tests/test_crs/valid_stressng.yaml - delete_operator + wait_clean } trap finish EXIT function functional_test_stressng { - wait_clean - apply_operator token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" tests/test_crs/valid_stressng.yaml | kubectl apply -f - - long_uuid=$(get_uuid 20) + cr=tests/test_crs/valid_stressng.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} # Wait for the workload pod to run the actual workload stressng_workload_pod=$(get_pod "app=stressng_workload-$uuid" 300) - kubectl wait --for=condition=Initialized "pods/$stressng_workload_pod" --namespace my-ripsaw --timeout=400s - kubectl wait --for=condition=complete -l app=stressng_workload-$uuid --namespace my-ripsaw jobs --timeout=500s + check_benchmark_for_desired_state $benchmark_name Complete 500s + index="ripsaw-stressng-results" if check_es "${long_uuid}" "${index}" @@ -35,9 +35,11 @@ function functional_test_stressng { echo "StressNG test: Success" else echo "Failed to find data for StressNG test in ES" - kubectl logs "$stressng_workload_pod" --namespace my-ripsaw + kubectl logs "$stressng_workload_pod" --namespace benchmark-operator exit 1 fi + + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_sysbench.sh b/tests/test_sysbench.sh old mode 100644 new mode 100755 index a0cac0470..78571fe2c --- a/tests/test_sysbench.sh +++ b/tests/test_sysbench.sh @@ -10,28 +10,27 @@ function finish { fi echo "Cleaning up sysbench" - kubectl delete -f tests/test_crs/valid_sysbench.yaml - delete_operator + wait_clean } trap error ERR trap finish EXIT function functional_test_sysbench { - wait_clean - apply_operator - kubectl apply -f tests/test_crs/valid_sysbench.yaml - long_uuid=$(get_uuid 20) + cr=tests/test_crs/valid_sysbench.yaml + delete_benchmark $cr + kubectl apply -f $cr + benchmark_name=$(get_benchmark_name $cr) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} sysbench_pod=$(get_pod "app=sysbench-$uuid" 300) - wait_for "kubectl wait --for=condition=Initialized pods/$sysbench_pod --namespace my-ripsaw --timeout=500s" "500s" $sysbench_pod - # Higher timeout as it takes longer - wait_for "kubectl wait --for=condition=complete -l app=sysbench-$uuid --namespace my-ripsaw jobs" "300s" $sysbench_pod + check_benchmark_for_desired_state $benchmark_name Complete 800s # sleep isn't needed as the sysbench is kind: job so once it's complete we can access logs # ensuring the run has actually happened - kubectl logs "$sysbench_pod" --namespace my-ripsaw | grep "execution time" + kubectl logs "$sysbench_pod" --namespace benchmark-operator | grep "execution time" echo "Sysbench test: Success" + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_uperf.sh b/tests/test_uperf.sh index 543ce966d..3621268fe 100755 --- a/tests/test_uperf.sh +++ b/tests/test_uperf.sh @@ -17,22 +17,18 @@ trap error ERR trap finish EXIT function functional_test_uperf { - wait_clean - apply_operator token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) test_name=$1 cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr echo "Performing: ${test_name}" sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid 20) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} + check_benchmark_for_desired_state $benchmark_name Complete 1800s + - pod_count "type=uperf-bench-server-$uuid" 1 900 - uperf_server_pod=$(get_pod "type=uperf-bench-server-${uuid}" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l type=uperf-bench-server-${uuid} pods --timeout=300s" "300s" $uperf_server_pod - uperf_client_pod=$(get_pod "app=uperf-bench-client-$uuid" 900) - wait_for "kubectl wait -n my-ripsaw --for=condition=Initialized pods/$uperf_client_pod --timeout=500s" "500s" $uperf_client_pod - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=uperf-bench-client-$uuid jobs --timeout=500s" "500s" $uperf_client_pod index="ripsaw-uperf-results" if check_es "${long_uuid}" "${index}" @@ -40,9 +36,9 @@ function functional_test_uperf { echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs "$uperf_client_pod" -n my-ripsaw exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_vegeta.sh b/tests/test_vegeta.sh old mode 100644 new mode 100755 index 687d48fa0..ca3d19a19 --- a/tests/test_vegeta.sh +++ b/tests/test_vegeta.sh @@ -9,7 +9,7 @@ function finish { error fi - [[ $check_logs == 1 ]] && kubectl logs -l app=vegeta-benchmark-$uuid -n my-ripsaw + [[ $check_logs == 1 ]] && kubectl logs -l app=vegeta-benchmark-$uuid -n benchmark-operator echo "Cleaning up vegeta" wait_clean } @@ -20,18 +20,16 @@ trap finish EXIT function functional_test_vegeta { check_logs=0 - wait_clean - apply_operator test_name=$1 cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr echo "Performing: ${test_name}" token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid 20) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} - - pod_count "app=vegeta-benchmark-$uuid" 2 900 - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=vegeta-benchmark-$uuid jobs --timeout=500s" "500s" + check_benchmark_for_desired_state $benchmark_name Complete 500s check_logs=1 index="ripsaw-vegeta-results" @@ -42,6 +40,8 @@ function functional_test_vegeta { echo "Failed to find data for ${test_name} in ES" exit 1 fi + delete_benchmark $cr + } figlet $(basename $0) diff --git a/tests/test_ycsb.sh b/tests/test_ycsb.sh old mode 100644 new mode 100755 index 143ca0e61..dc113fca4 --- a/tests/test_ycsb.sh +++ b/tests/test_ycsb.sh @@ -23,7 +23,7 @@ apiVersion: v1 kind: Service metadata: name: mongo - namespace: my-ripsaw + namespace: benchmark-operator labels: name: mongo spec: @@ -39,7 +39,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: mongo - namespace: my-ripsaw + namespace: benchmark-operator spec: selector: matchLabels: @@ -65,13 +65,15 @@ spec: - containerPort: 27017 EOF token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" tests/test_crs/valid_ycsb-mongo.yaml | kubectl apply -f - - long_uuid=$(get_uuid 20) + cr=tests/test_crs/valid_ycsb-mongo.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} ycsb_load_pod=$(get_pod "name=ycsb-load-$uuid" 300) - wait_for "kubectl wait --for=condition=Initialized pods/$ycsb_load_pod -n my-ripsaw --timeout=500s" "500s" $ycsb_load_pod - wait_for "kubectl wait --for=condition=Complete jobs -l name=ycsb-load-$uuid -n my-ripsaw --timeout=300s" "300s" $ycsb_load_pod + check_benchmark_for_desired_state $benchmark_name Complete 500s indexes="ripsaw-ycsb-summary ripsaw-ycsb-results" if check_es "${long_uuid}" "${indexes}" @@ -79,11 +81,11 @@ EOF echo "ycsb test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs -n my-ripsaw $ycsb_load_pod + kubectl logs -n benchmark-operator $ycsb_load_pod exit 1 fi + delete_benchmark $cr } figlet $(basename $0) -apply_operator functional_test_ycsb diff --git a/watches.yaml b/watches.yaml index 7b2aee108..c42519d09 100644 --- a/watches.yaml +++ b/watches.yaml @@ -1,8 +1,11 @@ --- +# Use the 'create api' subcommand to add watches to this file. - version: v1alpha1 group: ripsaw.cloudbulldozer.io kind: Benchmark - playbook: /opt/ansible/playbook.yml - reconcilePeriod: 3s - manageStatus: false + playbook: ./playbooks/benchmark.yml + reconcilePeriod: 1m + manageStatus: true snakeCaseParameters: false + # FIXME: Specify the role or playbook for this resource. +#+kubebuilder:scaffold:watch From 2ec10aec0d984e1cc3d4b565c8bc20cb42d1f7f5 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 22 Jul 2021 17:11:28 +0200 Subject: [PATCH 092/249] Add cri-o network setup latency metrics Signed-off-by: Raul Sevilla --- roles/kube-burner/files/metrics-aggregated.yaml | 6 ++++++ roles/kube-burner/files/metrics.yaml | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/roles/kube-burner/files/metrics-aggregated.yaml b/roles/kube-burner/files/metrics-aggregated.yaml index 4a3029712..0bb086c65 100644 --- a/roles/kube-burner/files/metrics-aggregated.yaml +++ b/roles/kube-burner/files/metrics-aggregated.yaml @@ -106,6 +106,12 @@ - query: avg(rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) metricName: nodeDiskReadBytes-AggregatedInfra +- query: irate(container_runtime_crio_operations_latency_microseconds{operation_type="network_setup_pod"}[2m]) > 0 + metricName: containerNetworkSetupLatency + +- query: irate(container_runtime_crio_operations_latency_microseconds{operation_type="network_setup_overall"}[2m]) > 0 + metricName: containerNetworkSetupOverallLatency + # Etcd metrics - query: sum(rate(etcd_server_leader_changes_seen_total[2m])) metricName: etcdLeaderChangesRate diff --git a/roles/kube-burner/files/metrics.yaml b/roles/kube-burner/files/metrics.yaml index 9e0df694a..fb53f1bc0 100644 --- a/roles/kube-burner/files/metrics.yaml +++ b/roles/kube-burner/files/metrics.yaml @@ -31,6 +31,12 @@ - query: sum(process_resident_memory_bytes{service="kubelet",job="crio"}) by (node) and on (node) kube_node_role{role="worker"} metricName: crioMemory +- query: irate(container_runtime_crio_operations_latency_microseconds{operation_type="network_setup_pod"}[2m]) > 0 + metricName: containerNetworkSetupLatency + +- query: irate(container_runtime_crio_operations_latency_microseconds{operation_type="network_setup_overall"}[2m]) > 0 + metricName: containerNetworkSetupOverallLatency + # Node metrics - query: sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) > 0 metricName: nodeCPU @@ -56,10 +62,10 @@ - query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) metricName: nodeDiskReadBytes +# Etcd metrics - query: sum(rate(etcd_server_leader_changes_seen_total[2m])) metricName: etcdLeaderChangesRate -# Etcd metrics - query: etcd_server_is_leader > 0 metricName: etcdServerIsLeader From a3b34e9346405e1025cd11903aa9bfda985df4a6 Mon Sep 17 00:00:00 2001 From: ebattat <73884315+ebattat@users.noreply.github.com> Date: Mon, 26 Jul 2021 13:21:51 +0300 Subject: [PATCH 093/249] Stressng (#606) * update changes to support operator-sdk version 1.5.0 * add stressng vm elasticsearch/nodeselector support * add stressng labels --- config/samples/stressng/cr.yaml | 2 +- config/samples/stressng/vm-cr.yaml | 2 +- roles/stressng/templates/stressng_workload.yml.j2 | 5 ++++- roles/stressng/templates/stressng_workload_vm.yml.j2 | 7 ++++++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/config/samples/stressng/cr.yaml b/config/samples/stressng/cr.yaml index f5d11f0e5..f96327eb9 100644 --- a/config/samples/stressng/cr.yaml +++ b/config/samples/stressng/cr.yaml @@ -16,7 +16,7 @@ spec: timeout: "30" instances: 1 pin: false # true for nodeSelector - pin_client: "node1" + pin_node: "node1" resources: false # true for resources requests/limits requests_cpu: 200m requests_memory: 100Mi diff --git a/config/samples/stressng/vm-cr.yaml b/config/samples/stressng/vm-cr.yaml index 805a66864..28b8d28f1 100644 --- a/config/samples/stressng/vm-cr.yaml +++ b/config/samples/stressng/vm-cr.yaml @@ -16,7 +16,7 @@ spec: timeout: "30" instances: 1 pin: false # true for nodeSelector - pin_client: "node1" + pin_node: "node1" # cpu stressor options cpu_stressors: "1" cpu_percentage: "100" diff --git a/roles/stressng/templates/stressng_workload.yml.j2 b/roles/stressng/templates/stressng_workload.yml.j2 index 53e8e95de..dec80fa1f 100644 --- a/roles/stressng/templates/stressng_workload.yml.j2 +++ b/roles/stressng/templates/stressng_workload.yml.j2 @@ -11,10 +11,13 @@ spec: metadata: labels: app: stressng_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-operator-uuid: {{ uuid }} + benchmark-operator-workload: stressng spec: {% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + kubernetes.io/hostname: '{{ workload_args.pin_node }}' {% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/stressng/templates/stressng_workload_vm.yml.j2 b/roles/stressng/templates/stressng_workload_vm.yml.j2 index 5618c70eb..0d8435cf3 100644 --- a/roles/stressng/templates/stressng_workload_vm.yml.j2 +++ b/roles/stressng/templates/stressng_workload_vm.yml.j2 @@ -4,6 +4,11 @@ kind: VirtualMachineInstance metadata: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" + labels: + app: stressng_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-operator-uuid: {{ uuid }} + benchmark-operator-workload: stressng spec: domain: cpu: @@ -40,7 +45,7 @@ spec: terminationGracePeriodSeconds: 0 {% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + kubernetes.io/hostname: '{{ workload_args.pin_node }}' {% endif %} networks: - name: default From 30444b94377b916a9ee25a73f2f46e2d7cf92a84 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 28 Jul 2021 21:50:20 +0200 Subject: [PATCH 094/249] Add container granurality Signed-off-by: Raul Sevilla --- roles/kube-burner/files/metrics.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/kube-burner/files/metrics.yaml b/roles/kube-burner/files/metrics.yaml index fb53f1bc0..e7e562a09 100644 --- a/roles/kube-burner/files/metrics.yaml +++ b/roles/kube-burner/files/metrics.yaml @@ -9,11 +9,11 @@ metricName: APIInflightRequests # Containers & pod metrics -- query: sum(irate(container_cpu_usage_seconds_total{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}[2m]) * 100) by (pod, namespace, node) - metricName: podCPU +- query: sum(irate(container_cpu_usage_seconds_total{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}[2m]) * 100) by (container, pod, namespace, node) + metricName: containerCPU -- query: sum(container_memory_rss{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}) by (pod, namespace, node) - metricName: podMemory +- query: sum(container_memory_rss{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}) by (container, pod, namespace, node) + metricName: containerMemory - query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 metricName: containerDiskUsage From c72fa1debb6b86efa68d27de4c64fc0eaeac6799 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 29 Jul 2021 10:47:32 +0200 Subject: [PATCH 095/249] Add P&F metrics Signed-off-by: Raul Sevilla --- .../kube-burner/files/metrics-aggregated.yaml | 43 +++++++++++++------ roles/kube-burner/files/metrics.yaml | 27 +++++++++--- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/roles/kube-burner/files/metrics-aggregated.yaml b/roles/kube-burner/files/metrics-aggregated.yaml index 0bb086c65..10093b637 100644 --- a/roles/kube-burner/files/metrics-aggregated.yaml +++ b/roles/kube-burner/files/metrics-aggregated.yaml @@ -5,14 +5,28 @@ - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0 metricName: APIRequestRate -- query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0 - metricName: APIInflightRequests +# P&F +- query: rate(apiserver_flowcontrol_dispatched_requests_total[2m]) > 0 + metricName: APIFlowControlDispatchedRequests -# Container & pod metrics -- query: (sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 +- query: sum(apiserver_flowcontrol_current_executing_requests) by (instance, priority_level,flow_schema) > 0 + metricName: APIFlowControlCurrentExecutingRequests + +- query: rate(apiserver_flowcontrol_rejected_requests_total[2m]) > 0 + metricName: APIFlowControlRejectedRequests + +- query: sum(apiserver_flowcontrol_current_inqueue_requests) by (instance, flow_schema, priority_level) > 0 + metricName: APIFlowControlInqueueRequests + +- query: avg(apiserver_flowcontrol_request_concurrency_limit{}) by (priority_level) + metricName: APIFlowControlRequestConcurrencyLimit + instant: true + +# Containers & pod metrics +- query: (sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|image-registry|operator-lifecycle-manager)"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 metricName: containerMemory-Masters -- query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|sdn|ovn-kubernetes|.*apiserver|authentication|.*controller-manager|.*scheduler)"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 +- query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|sdn|ovn-kubernetes|.*apiserver|authentication|.*controller-manager|.*scheduler|image-registry|operator-lifecycle-manager)"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 metricName: containerCPU-Masters - query: (sum(irate(container_cpu_usage_seconds_total{pod!="",container="prometheus",namespace="openshift-monitoring"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0 @@ -21,7 +35,7 @@ - query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) by (namespace, container)) > 0 metricName: containerCPU-AggregatedWorkers -- query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"}[2m]) * 100 and on (node) kube_node_role{role="infra"}) by (namespace, container)) > 0 +- query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|operator-lifecycle-manager)"}[2m]) * 100 and on (node) kube_node_role{role="infra"}) by (namespace, container)) > 0 metricName: containerCPU-AggregatedInfra - query: (sum(container_memory_rss{pod!="",namespace="openshift-monitoring",name!="",container="prometheus"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0 @@ -30,7 +44,7 @@ - query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"} and on (node) kube_node_role{role="worker"}) by (container, namespace) metricName: containerMemory-AggregatedWorkers -- query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|logging)"} and on (node) kube_node_role{role="infra"}) by (container, namespace) +- query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|operator-lifecycle-manager)"} and on (node) kube_node_role{role="infra"}) by (container, namespace) metricName: containerMemory-AggregatedInfra # Node metrics @@ -52,14 +66,17 @@ - query: avg(node_memory_MemAvailable_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) metricName: nodeMemoryAvailable-AggregatedInfra -- query: avg(node_memory_Active_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: nodeMemoryActive-Masters +- query: avg(node_memory_MemTotal_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: nodeMemoryTotal-Masters + instant: true -- query: avg(node_memory_Active_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryActive-AggregatedWorkers +- query: avg(node_memory_MemTotal_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryTotal-AggregatedWorkers + instant: true -- query: avg(avg(node_memory_Active_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryActive-AggregatedInfra +- query: avg(node_memory_MemTotal_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryTotal-AggregatedInfra + instant: true - query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") metricName: nodeMemoryCached+nodeMemoryBuffers-Masters diff --git a/roles/kube-burner/files/metrics.yaml b/roles/kube-burner/files/metrics.yaml index e7e562a09..0db189309 100644 --- a/roles/kube-burner/files/metrics.yaml +++ b/roles/kube-burner/files/metrics.yaml @@ -5,14 +5,28 @@ - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0 metricName: APIRequestRate -- query: sum(apiserver_current_inflight_requests{}) by (request_kind) > 0 - metricName: APIInflightRequests +# P&F +- query: rate(apiserver_flowcontrol_dispatched_requests_total[2m]) > 0 + metricName: APIFlowControlDispatchedRequests + +- query: sum(apiserver_flowcontrol_current_executing_requests) by (instance, priority_level,flow_schema) > 0 + metricName: APIFlowControlCurrentExecutingRequests + +- query: rate(apiserver_flowcontrol_rejected_requests_total[2m]) > 0 + metricName: APIFlowControlRejectedRequests + +- query: sum(apiserver_flowcontrol_current_inqueue_requests) by (instance, flow_schema, priority_level) > 0 + metricName: APIFlowControlInqueueRequests + +- query: avg(apiserver_flowcontrol_request_concurrency_limit{}) by (priority_level) + metricName: APIFlowControlRequestConcurrencyLimit + instant: true # Containers & pod metrics -- query: sum(irate(container_cpu_usage_seconds_total{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}[2m]) * 100) by (container, pod, namespace, node) +- query: sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|image-registry|operator-lifecycle-manager)"}[2m]) * 100) by (container, pod, namespace, node) metricName: containerCPU -- query: sum(container_memory_rss{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|logging|image-registry)"}) by (container, pod, namespace, node) +- query: sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|image-registry|operator-lifecycle-manager)"}) by (container, pod, namespace, node) metricName: containerMemory - query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 @@ -44,8 +58,9 @@ - query: avg(node_memory_MemAvailable_bytes) by (instance) metricName: nodeMemoryAvailable -- query: avg(node_memory_Active_bytes) by (instance) - metricName: nodeMemoryActive +- query: avg(node_memory_MemTotal_bytes) by (instance) + metricName: nodeMemoryTotal + instant: true - query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) metricName: nodeMemoryCached+nodeMemoryBuffers From 8efce1e4486efaf94aae2fb973a65ab30ea7fa19 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 29 Jul 2021 21:52:59 +0200 Subject: [PATCH 096/249] Arbirary env vars Signed-off-by: Raul Sevilla --- docs/kube-burner.md | 18 ++++++++++++++---- roles/kube-burner/tasks/main.yml | 2 -- roles/kube-burner/templates/kube-burner.yml.j2 | 10 +++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/kube-burner.md b/docs/kube-burner.md index a2a0ad604..be859fce5 100644 --- a/docs/kube-burner.md +++ b/docs/kube-burner.md @@ -76,7 +76,7 @@ The workload type is specified by the parameter `workload` from the `args` objec All kube-burner's workloads support the following parameters: -- **`workload`**: Type of kube-burner workload. As mentioned before, allowed values are cluster-density, node-density and node-density-heavy +- **``workload``**: Type of kube-burner workload. As mentioned before, allowed values are cluster-density, node-density and node-density-heavy - **``default_index``**: ElasticSearch index name. Defaults to __ripsaw-kube-burner__ - **``job_iterations``**: How many iterations to execute of the specified kube-burner workload - **``qps``**: Limit object creation queries per second. Defaults to __5__ @@ -103,6 +103,7 @@ Where key defaults to __node-role.kubernetes.io/worker__ and value defaults to e - **``step``**: Prometheus step size, useful for long benchmarks. Defaults to 30s - **``metrics_profile``**: kube-burner metric profile that indicates what prometheus metrics kube-burner will collect. Defaults to `metrics.yaml` in node-density workloads and `metrics-aggregated.yaml` in the remaining. Detailed in the [Metrics section](#Metrics) of this document - **``runtime_class``** : If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. +- **``extra_env_vars``** : List of dictionaries that will be injected to the kube-burner pod as environment variables. e.g. `extra_env_vars: [{"foo": "bar}, {"foo2": "bar2"}]` kube-burner is able to collect complex prometheus metrics and index them in a ElasticSearch. This feature can be configured by the prometheus object of kube-burner's CR. @@ -186,14 +187,23 @@ Keep in mind that the object templated declared in this remote configuration fil replicas: 1 ``` -> `kube-burner` is able to use go template based configuration files, in addition to the default behaviour, this template can reference environment variables using the syntax `{{ .MY_ENV_VAR }}`. The kube-burner job created by `benchmark-operator` always injects the environment variable `prom_es` with the value of `prometheus.es_url`. This can be useful to overwrite the ElasticSearch URL in remote configuration files as shown in the code snippet below. +> `kube-burner` is able to use go template based configuration files, in addition to the default behaviour, this template can reference environment variables using the syntax `{{ .MY_ENV_VAR }}`. The kube-burner job created by `benchmark-operator` always injects a list of environment variables which can be defined with the parameter `extra_env_vars`. This can be useful to parametrize remote configuration files as shown in the code snippet below. + +Supossing a CR with `extra_env_vars` configured as: +```yaml +workload: + args: + extra_env_vars: + - INDEXING: true + - ES_SERVER: https://example-es.instance.com:9200 +``` ```yaml global: writeToFile: false indexerConfig: - enabled: true - esServers: ["{{.prom_es}}"] + enabled: {{.INDEXING}} + esServers: ["{{.ES_SERVER}}"] insecureSkipVerify: true defaultIndex: ripsaw-kube-burner type: elastic diff --git a/roles/kube-burner/tasks/main.yml b/roles/kube-burner/tasks/main.yml index e00a41c10..656d52265 100644 --- a/roles/kube-burner/tasks/main.yml +++ b/roles/kube-burner/tasks/main.yml @@ -42,8 +42,6 @@ when: - workload_args.remote_config is defined - workload_args.remote_config - - workload_args.remote_metrics_profile is not defined - - workload_args.remote_metrics_profile - name: Create cluster-density configmaps k8s: diff --git a/roles/kube-burner/templates/kube-burner.yml.j2 b/roles/kube-burner/templates/kube-burner.yml.j2 index 69fb1c48e..f8fff104d 100644 --- a/roles/kube-burner/templates/kube-burner.yml.j2 +++ b/roles/kube-burner/templates/kube-burner.yml.j2 @@ -30,10 +30,14 @@ spec: - name: kube-burner image: {{ workload_args.image | default('quay.io/cloud-bulldozer/kube-burner:latest') }} imagePullPolicy: Always -{% if prometheus.es_url is defined %} +{% if workload_args.extra_env_vars is defined and workload_args.extra_env_vars is sequence %} env: - - name: prom_es - value: "{{ prometheus.es_url }}" +{% for envvar in workload_args.extra_env_vars %} +{% for name, value in envvar.items() %} + - name: {{ name }} + value: "{{ value }}" +{% endfor %} +{% endfor %} {% endif %} workingDir: /tmp/kube-burner command: ["/bin/sh", "-c"] From 0edc388e04c3c5b8a637792095821b760c8aba7c Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Fri, 30 Jul 2021 11:48:24 +0200 Subject: [PATCH 097/249] Extra env vars as dictionary rather than a list of dictionaries Signed-off-by: Raul Sevilla --- docs/kube-burner.md | 8 ++++---- roles/kube-burner/templates/kube-burner.yml.j2 | 6 ++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/kube-burner.md b/docs/kube-burner.md index be859fce5..792890218 100644 --- a/docs/kube-burner.md +++ b/docs/kube-burner.md @@ -103,7 +103,7 @@ Where key defaults to __node-role.kubernetes.io/worker__ and value defaults to e - **``step``**: Prometheus step size, useful for long benchmarks. Defaults to 30s - **``metrics_profile``**: kube-burner metric profile that indicates what prometheus metrics kube-burner will collect. Defaults to `metrics.yaml` in node-density workloads and `metrics-aggregated.yaml` in the remaining. Detailed in the [Metrics section](#Metrics) of this document - **``runtime_class``** : If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. -- **``extra_env_vars``** : List of dictionaries that will be injected to the kube-burner pod as environment variables. e.g. `extra_env_vars: [{"foo": "bar}, {"foo2": "bar2"}]` +- **``extra_env_vars``** : This dictionary defines a set of fields that will be injected to the kube-burner pod as environment variables. e.g. `extra_env_vars: {"foo": "bar", "foo2": "bar2"}` kube-burner is able to collect complex prometheus metrics and index them in a ElasticSearch. This feature can be configured by the prometheus object of kube-burner's CR. @@ -187,15 +187,15 @@ Keep in mind that the object templated declared in this remote configuration fil replicas: 1 ``` -> `kube-burner` is able to use go template based configuration files, in addition to the default behaviour, this template can reference environment variables using the syntax `{{ .MY_ENV_VAR }}`. The kube-burner job created by `benchmark-operator` always injects a list of environment variables which can be defined with the parameter `extra_env_vars`. This can be useful to parametrize remote configuration files as shown in the code snippet below. +> `kube-burner` is able to use go template based configuration files, in addition to the default behaviour, this template can reference environment variables using the syntax `{{ .MY_ENV_VAR }}`. The kube-burner job created by `benchmark-operator` always injects a list of environment variables which can be defined with the parameter `extra_env_vars` mentioned previously. This can be useful to parametrize remote configuration files as shown in the code snippet below. Supossing a CR with `extra_env_vars` configured as: ```yaml workload: args: extra_env_vars: - - INDEXING: true - - ES_SERVER: https://example-es.instance.com:9200 + INDEXING: true + ES_SERVER: https://example-es.instance.com:9200 ``` ```yaml diff --git a/roles/kube-burner/templates/kube-burner.yml.j2 b/roles/kube-burner/templates/kube-burner.yml.j2 index f8fff104d..9fa7deacf 100644 --- a/roles/kube-burner/templates/kube-burner.yml.j2 +++ b/roles/kube-burner/templates/kube-burner.yml.j2 @@ -30,14 +30,12 @@ spec: - name: kube-burner image: {{ workload_args.image | default('quay.io/cloud-bulldozer/kube-burner:latest') }} imagePullPolicy: Always -{% if workload_args.extra_env_vars is defined and workload_args.extra_env_vars is sequence %} +{% if workload_args.extra_env_vars is defined and workload_args.extra_env_vars is mapping %} env: -{% for envvar in workload_args.extra_env_vars %} -{% for name, value in envvar.items() %} +{% for name, value in extra_env_vars.items() %} - name: {{ name }} value: "{{ value }}" {% endfor %} -{% endfor %} {% endif %} workingDir: /tmp/kube-burner command: ["/bin/sh", "-c"] From 21ab5c729f56a89543fa3706018bef5d66ddf0c2 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Fri, 30 Jul 2021 12:28:13 +0200 Subject: [PATCH 098/249] Add missing workload_args to extra_env_vars Signed-off-by: Raul Sevilla --- roles/kube-burner/templates/kube-burner.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kube-burner/templates/kube-burner.yml.j2 b/roles/kube-burner/templates/kube-burner.yml.j2 index 9fa7deacf..472360f7e 100644 --- a/roles/kube-burner/templates/kube-burner.yml.j2 +++ b/roles/kube-burner/templates/kube-burner.yml.j2 @@ -32,7 +32,7 @@ spec: imagePullPolicy: Always {% if workload_args.extra_env_vars is defined and workload_args.extra_env_vars is mapping %} env: -{% for name, value in extra_env_vars.items() %} +{% for name, value in workload_args.extra_env_vars.items() %} - name: {{ name }} value: "{{ value }}" {% endfor %} From a8481a10d3947fba6b986c1c9e8ff99d76651c62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Fri, 30 Jul 2021 15:05:51 +0200 Subject: [PATCH 099/249] Fix uuid error (#621) Signed-off-by: Raul Sevilla --- playbooks/benchmark.yml | 6 ++++-- roles/uuid/tasks/main.yml | 3 --- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/playbooks/benchmark.yml b/playbooks/benchmark.yml index 20551b759..3256b9a81 100644 --- a/playbooks/benchmark.yml +++ b/playbooks/benchmark.yml @@ -30,8 +30,6 @@ - control-plane = controller-manager register: bo - - debug: - msg: "{{ cr_state }}" - name: Set Workload UUID block: @@ -61,6 +59,10 @@ - set_fact: uuid: "{{ cr_state.resources[0].status.uuid }}" trunc_uuid: "{{ cr_state.resources[0].status.suuid }}" + when: + - cr_state.resources[0].status is defined + - cr_state.resources[0].status.uuid is defined + - cr_state.resources[0].status.uuid != "" - name: Run Workload rescue: diff --git a/roles/uuid/tasks/main.yml b/roles/uuid/tasks/main.yml index 4ff8d2ec7..2ba7e2c58 100644 --- a/roles/uuid/tasks/main.yml +++ b/roles/uuid/tasks/main.yml @@ -5,6 +5,3 @@ - name: set the truncated uuid set_fact: trunc_uuid={{ uuid.split("-")[0] }} - -- debug: - msg: " The results of current execution of the workload {{ workload.name }} will be associated with uuid: {{ uuid }}" From 665b98e37e7c03aacb3bb0a10925e7f0eca44ccd Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Fri, 30 Jul 2021 13:32:48 +0200 Subject: [PATCH 100/249] Lower reconcilePeriod Signed-off-by: Raul Sevilla --- watches.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/watches.yaml b/watches.yaml index c42519d09..fa0f84f97 100644 --- a/watches.yaml +++ b/watches.yaml @@ -4,7 +4,7 @@ group: ripsaw.cloudbulldozer.io kind: Benchmark playbook: ./playbooks/benchmark.yml - reconcilePeriod: 1m + reconcilePeriod: 3s manageStatus: true snakeCaseParameters: false # FIXME: Specify the role or playbook for this resource. From f440267480fb36b8262c2b84be664ff8ca08672d Mon Sep 17 00:00:00 2001 From: Mohit Sheth Date: Fri, 11 Jun 2021 18:00:03 +0000 Subject: [PATCH 101/249] add networkpolicy test cases --- .../files/case2-networkpolicy.yaml | 16 +++++ .../files/case3-networkpolicy.yaml | 17 +++++ .../files/cross-ns-networkpolicy.yaml | 18 ++++++ .../files/networkpolicy-deny-all.yaml | 8 +++ .../files/networkpolicy-ingress.yaml | 13 ++++ .../files/networkpolicy-monitoring.yaml | 13 ++++ .../files/networkpolicy-same-ns.yaml | 9 +++ roles/kube-burner/files/nginx-cross-ns.yaml | 12 ++++ roles/kube-burner/files/nginx.yaml | 14 +++++ roles/kube-burner/tasks/main.yml | 62 +++++++++++++++++++ .../templates/networkpolicy-case2.yml.j2 | 47 ++++++++++++++ .../templates/networkpolicy-case3.yml.j2 | 47 ++++++++++++++ .../templates/networkpolicy-cross-ns.yml.j2 | 34 ++++++++++ .../networkpolicy-multitenant.yml.j2 | 35 +++++++++++ 14 files changed, 345 insertions(+) create mode 100644 roles/kube-burner/files/case2-networkpolicy.yaml create mode 100644 roles/kube-burner/files/case3-networkpolicy.yaml create mode 100644 roles/kube-burner/files/cross-ns-networkpolicy.yaml create mode 100644 roles/kube-burner/files/networkpolicy-deny-all.yaml create mode 100644 roles/kube-burner/files/networkpolicy-ingress.yaml create mode 100644 roles/kube-burner/files/networkpolicy-monitoring.yaml create mode 100644 roles/kube-burner/files/networkpolicy-same-ns.yaml create mode 100644 roles/kube-burner/files/nginx-cross-ns.yaml create mode 100644 roles/kube-burner/files/nginx.yaml create mode 100644 roles/kube-burner/templates/networkpolicy-case2.yml.j2 create mode 100644 roles/kube-burner/templates/networkpolicy-case3.yml.j2 create mode 100644 roles/kube-burner/templates/networkpolicy-cross-ns.yml.j2 create mode 100644 roles/kube-burner/templates/networkpolicy-multitenant.yml.j2 diff --git a/roles/kube-burner/files/case2-networkpolicy.yaml b/roles/kube-burner/files/case2-networkpolicy.yaml new file mode 100644 index 000000000..1aefbfbaa --- /dev/null +++ b/roles/kube-burner/files/case2-networkpolicy.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: case2-{{.podselector_label}}{{.Replica}}-{{.ingress_label}} +spec: + podSelector: + matchLabels: + {{.podselector_label_num}}: {{.podselector_label}}-{{.Replica}} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + {{.ingress_label_num}}: {{.ingress_label}}-{{randInteger 1 5}} + diff --git a/roles/kube-burner/files/case3-networkpolicy.yaml b/roles/kube-burner/files/case3-networkpolicy.yaml new file mode 100644 index 000000000..9e7cee4f2 --- /dev/null +++ b/roles/kube-burner/files/case3-networkpolicy.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: case3-{{.podselector_label}}{{.Replica}}-{{.ingress_label}} +spec: + podSelector: + matchLabels: + {{.podselector_label_num}}: {{.podselector_label}}-{{.Replica}} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchExpressions: + - key: {{.ingress_label_num}} + operator: NotIn + values: [{{.ingress_label}}-{{randInteger 1 5}}] diff --git a/roles/kube-burner/files/cross-ns-networkpolicy.yaml b/roles/kube-burner/files/cross-ns-networkpolicy.yaml new file mode 100644 index 000000000..818e11999 --- /dev/null +++ b/roles/kube-burner/files/cross-ns-networkpolicy.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: cross-ns-{{randInteger 1 1000000}} +spec: + podSelector: + matchLabels: + label1: foo-{{randInteger 1 5}} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + label1: foo-{{randInteger 1 5}} + namespaceSelector: + matchLabels: + ns_label: bar-{{randInteger 1 4}} diff --git a/roles/kube-burner/files/networkpolicy-deny-all.yaml b/roles/kube-burner/files/networkpolicy-deny-all.yaml new file mode 100644 index 000000000..64548414e --- /dev/null +++ b/roles/kube-burner/files/networkpolicy-deny-all.yaml @@ -0,0 +1,8 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all +spec: + podSelector: {} + policyTypes: + - Ingress diff --git a/roles/kube-burner/files/networkpolicy-ingress.yaml b/roles/kube-burner/files/networkpolicy-ingress.yaml new file mode 100644 index 000000000..c180e2736 --- /dev/null +++ b/roles/kube-burner/files/networkpolicy-ingress.yaml @@ -0,0 +1,13 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-from-openshift-ingress +spec: + ingress: + - from: + - namespaceSelector: + matchLabels: + network.openshift.io/policy-group: ingress + podSelector: {} + policyTypes: + - Ingress diff --git a/roles/kube-burner/files/networkpolicy-monitoring.yaml b/roles/kube-burner/files/networkpolicy-monitoring.yaml new file mode 100644 index 000000000..941162c3b --- /dev/null +++ b/roles/kube-burner/files/networkpolicy-monitoring.yaml @@ -0,0 +1,13 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-from-openshift-monitoring +spec: + ingress: + - from: + - namespaceSelector: + matchLabels: + network.openshift.io/policy-group: monitoring + podSelector: {} + policyTypes: + - Ingress diff --git a/roles/kube-burner/files/networkpolicy-same-ns.yaml b/roles/kube-burner/files/networkpolicy-same-ns.yaml new file mode 100644 index 000000000..0517b1767 --- /dev/null +++ b/roles/kube-burner/files/networkpolicy-same-ns.yaml @@ -0,0 +1,9 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-same-namespace +spec: + podSelector: + ingress: + - from: + - podSelector: {} diff --git a/roles/kube-burner/files/nginx-cross-ns.yaml b/roles/kube-burner/files/nginx-cross-ns.yaml new file mode 100644 index 000000000..2e1a1afa7 --- /dev/null +++ b/roles/kube-burner/files/nginx-cross-ns.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-{{.Replica}}-{{.Iteration}}-{{randInteger 0 1000000}} + labels: + label1: {{.label1}}-{{randInteger 1 5}} +spec: + containers: + - name: nginx + image: quay.io/openshift-scale/nginx:latest + ports: + - containerPort: 8080 diff --git a/roles/kube-burner/files/nginx.yaml b/roles/kube-burner/files/nginx.yaml new file mode 100644 index 000000000..35c14fb3a --- /dev/null +++ b/roles/kube-burner/files/nginx.yaml @@ -0,0 +1,14 @@ + +apiVersion: v1 +kind: Pod +metadata: + name: nginx-{{.Replica}}-{{.Iteration}}-{{randInteger 0 1000000}} + labels: + label1: {{.label1}}-{{.Replica}} + label2: {{.label2}}-{{.Replica}} +spec: + containers: + - name: nginx + image: quay.io/openshift-scale/nginx:latest + ports: + - containerPort: 8080 diff --git a/roles/kube-burner/tasks/main.yml b/roles/kube-burner/tasks/main.yml index 656d52265..e09238f93 100644 --- a/roles/kube-burner/tasks/main.yml +++ b/roles/kube-burner/tasks/main.yml @@ -111,6 +111,68 @@ secret.yml: "{{ lookup('file', 'secret.yml')}}" when: workload_args.workload == "max-namespaces" + - name: Create networkpolicy-multitenant configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-multitenant.yml.j2')}}" + nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" + networkpolicy-ingress.yaml: "{{ lookup('file', 'networkpolicy-ingress.yaml')}}" + networkpolicy-monitoring.yaml: "{{ lookup('file', 'networkpolicy-monitoring.yaml')}}" + networkpolicy-same-ns.yaml: "{{ lookup('file', 'networkpolicy-same-ns.yaml')}}" + when: workload_args.workload == "networkpolicy-multitenant" + + - name: Create networkpolicy-case2 configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-case2.yml.j2')}}" + nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" + case2-networkpolicy.yaml: "{{ lookup('file', 'case2-networkpolicy.yaml')}}" + networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" + when: workload_args.workload == "networkpolicy-case2" + + - name: Create networkpolicy-case3 configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-case3.yml.j2')}}" + nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" + case3-networkpolicy.yaml: "{{ lookup('file', 'case3-networkpolicy.yaml')}}" + networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" + when: workload_args.workload == "networkpolicy-case3" + + + - name: Create networkpolicy-cross-ns configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-cross-ns.yml.j2')}}" + nginx-cross-ns.yaml: "{{ lookup('file', 'nginx-cross-ns.yaml')}}" + cross-ns-networkpolicy.yaml: "{{ lookup('file', 'cross-ns-networkpolicy.yaml')}}" + networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" + when: workload_args.workload == "networkpolicy-cross-ns" + - name: Create max-services configmaps k8s: definition: diff --git a/roles/kube-burner/templates/networkpolicy-case2.yml.j2 b/roles/kube-burner/templates/networkpolicy-case2.yml.j2 new file mode 100644 index 000000000..8802f3cf9 --- /dev/null +++ b/roles/kube-burner/templates/networkpolicy-case2.yml.j2 @@ -0,0 +1,47 @@ +--- +global: + writeToFile: false +jobs: + - name: networkpolicy-case2 + jobIterations: {{ workload_args.job_iterations|default(400) }} + qps: {{ workload_args.qps|default(20) }} + burst: {{ workload_args.burst|default(20) }} + namespacedIterations: true + namespace: np-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + cleanup: {{ workload_args.cleanup|default(false) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + objects: + +{% for num in range(5) %} + - objectTemplate: nginx.yaml + replicas: {{ workload_args.replicas|default(20) }} + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + label1: "foo" + label2: "bar" +{% endfor %} + + + - objectTemplate: case2-networkpolicy.yaml + replicas: {{ workload_args.replicas|default(20) }} + inputVars: + podselector_label_num: "label1" + podselector_label: "foo" + ingress_label_num: "label1" + ingress_label: "foo" + + - objectTemplate: case2-networkpolicy.yaml + replicas: {{ workload_args.replicas|default(20) }} + inputVars: + podselector_label_num: "label2" + podselector_label: "bar" + ingress_label_num: "label2" + ingress_label: "bar" + + - objectTemplate: networkpolicy-deny-all.yaml + replicas: 1 + diff --git a/roles/kube-burner/templates/networkpolicy-case3.yml.j2 b/roles/kube-burner/templates/networkpolicy-case3.yml.j2 new file mode 100644 index 000000000..132d93f2c --- /dev/null +++ b/roles/kube-burner/templates/networkpolicy-case3.yml.j2 @@ -0,0 +1,47 @@ +--- +global: + writeToFile: false +jobs: + - name: networkpolicy-case3 + jobIterations: {{ workload_args.job_iterations|default(400) }} + qps: {{ workload_args.qps|default(20) }} + burst: {{ workload_args.burst|default(20) }} + namespacedIterations: true + namespace: np-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + cleanup: {{ workload_args.cleanup|default(false) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + objects: + +{% for num in range(5) %} + - objectTemplate: nginx.yaml + replicas: {{ workload_args.replicas|default(5) }} + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + label1: "foo" + label2: "bar" +{% endfor %} + + + - objectTemplate: case3-networkpolicy.yaml + replicas: {{ workload_args.replicas|default(5) }} + inputVars: + podselector_label_num: "label1" + podselector_label: "foo" + ingress_label_num: "label1" + ingress_label: "foo" + + - objectTemplate: case3-networkpolicy.yaml + replicas: {{ workload_args.replicas|default(5) }} + inputVars: + podselector_label_num: "label2" + podselector_label: "bar" + ingress_label_num: "label2" + ingress_label: "bar" + + - objectTemplate: networkpolicy-deny-all.yaml + replicas: 1 + diff --git a/roles/kube-burner/templates/networkpolicy-cross-ns.yml.j2 b/roles/kube-burner/templates/networkpolicy-cross-ns.yml.j2 new file mode 100644 index 000000000..88f23b395 --- /dev/null +++ b/roles/kube-burner/templates/networkpolicy-cross-ns.yml.j2 @@ -0,0 +1,34 @@ +--- +global: + writeToFile: false +jobs: + - name: networkpolicy-cross-ns + jobIterations: {{ workload_args.job_iterations|default(400) }} + qps: {{ workload_args.qps|default(20) }} + burst: {{ workload_args.burst|default(20) }} + namespacedIterations: true + namespace: np-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + cleanup: {{ workload_args.cleanup|default(false) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + objects: + +{% for num in range(5) %} + - objectTemplate: nginx-cross-ns.yaml + replicas: {{ workload_args.replicas|default(1) }} + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + label1: "foo" + label2: "bar" +{% endfor %} + + + - objectTemplate: cross-ns-networkpolicy.yaml + replicas: {{ workload_args.replicas|default(5) }} + + - objectTemplate: networkpolicy-deny-all.yaml + replicas: 1 + diff --git a/roles/kube-burner/templates/networkpolicy-multitenant.yml.j2 b/roles/kube-burner/templates/networkpolicy-multitenant.yml.j2 new file mode 100644 index 000000000..83a8ac67c --- /dev/null +++ b/roles/kube-burner/templates/networkpolicy-multitenant.yml.j2 @@ -0,0 +1,35 @@ +--- +global: + writeToFile: false +jobs: + - name: networkpolicy-multitenant + jobIterations: {{ workload_args.job_iterations|default(400) }} + qps: {{ workload_args.qps|default(20) }} + burst: {{ workload_args.burst|default(20) }} + namespacedIterations: true + namespace: np-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + cleanup: {{ workload_args.cleanup|default(false) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + objects: + +{% for num in range(5) %} + - objectTemplate: nginx.yaml + replicas: {{ workload_args.replicas|default(1) }} + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + label1: "foo" + label2: "bar" +{% endfor %} + + + - objectTemplate: networkpolicy-same-ns.yaml + replicas: 1 + - objectTemplate: networkpolicy-monitoring.yaml + replicas: 1 + - objectTemplate: networkpolicy-ingress.yaml + replicas: 1 + From 61c1257588396fdbdc96f591ebe5231893d87ed1 Mon Sep 17 00:00:00 2001 From: Harshith u <44830597+Harshith-umesh@users.noreply.github.com> Date: Mon, 2 Aug 2021 20:21:18 +0530 Subject: [PATCH 102/249] Fix broken links in the docs (#616) * fix link * fix broken links --- docs/cyclictest.md | 2 +- docs/fio_distributed.md | 4 ++-- docs/flent.md | 2 +- docs/fs-drift.md | 2 +- docs/hammerdb.md | 34 +++++++++++++++++----------------- docs/iperf.md | 2 +- docs/kube-burner.md | 4 ++-- docs/oslat.md | 2 +- docs/pgbench.md | 2 +- docs/servicemesh.md | 4 ++-- docs/smallfile.md | 4 ++-- docs/stressng.md | 2 +- docs/sysbench.md | 4 ++-- docs/uperf.md | 2 +- docs/vegeta.md | 2 +- docs/ycsb.md | 2 +- 16 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docs/cyclictest.md b/docs/cyclictest.md index 59eebc51b..0e2d6a23e 100644 --- a/docs/cyclictest.md +++ b/docs/cyclictest.md @@ -10,7 +10,7 @@ SCHED_FIFO real-time tasks which are higher priority. ## Running cyclictest -Given that you followed instructions to deploy operator, you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_cyclictest.yaml) to your needs. +Given that you followed instructions to deploy operator, you can modify [cr.yaml](../config/samples/cyclictest/cr.yaml) to your needs. It is recommended to define pod requests and limits when running cyclict test, to give guaranteed CPUs to the pods. It is also expected to have the realtime kernel installed with required isolation for pods using the [Performance Add-On Operator](https://github.com/openshift-kni/performance-addon-operators). diff --git a/docs/fio_distributed.md b/docs/fio_distributed.md index 13e1490a5..73a967e4b 100644 --- a/docs/fio_distributed.md +++ b/docs/fio_distributed.md @@ -250,13 +250,13 @@ pairs under `params` here are used to append additional fio job options based on Dropping the OSD caches before workloads is a normal and advised part of tests that involve storage I/O. Doing this with Rook-Ceph requires a privileged pod running the same namespace as the Ceph pods and with the Ceph command tools available. To facilitate this, we provide the -[resources/rook_ceph_drop_cache_pod.yaml](../resources/rook_ceph_drop_cache_pod.yaml) file, which will +[config/samples/fio/cr.yaml](../config/samples/fio/cr.yaml) file, which will deploy a pod with the correct permissions and tools, as well as running a simple HTTP listener to trigger the cache drop by URL. **You must deploy this privileged pod in order for the drop caches requests in the workload to function.** ```bash -kubectl apply -f resources/rook_ceph_drop_cache_pod.yaml +kubectl apply -f config/samples/fio/cr.yaml ``` *Note: If Ceph is in a namespace other than `rook-ceph` you will need to modify the provided YAML accordingly.* diff --git a/docs/flent.md b/docs/flent.md index 9675c8c37..1ff19cc53 100644 --- a/docs/flent.md +++ b/docs/flent.md @@ -5,7 +5,7 @@ ## Running Flent Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_flent_cr.yaml) +you can modify [cr.yaml](../config/samples/flent/cr.yaml) ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 diff --git a/docs/fs-drift.md b/docs/fs-drift.md index 9a26cc99f..678c2690f 100644 --- a/docs/fs-drift.md +++ b/docs/fs-drift.md @@ -11,7 +11,7 @@ tree - see the benchmark documentation for details. fs-drift requires that a st Once the operator has been installed following the instructions, one needs to modify the [cr.yaml](../resources/crds/ripsaw_v1alpha1_fs-drift_cr.yaml) to customize workload parameters - the defaults are selected to demonstrate its operation and are not intended to specify a long-duration test. -The parameters in [cr.yaml](../resources/crds/ripsaw_v1alpha1_fs-drift_cr.yaml) would look similar to this example: +The parameters in [cr.yaml](../config/samples/fs-drift/cr.yaml) would look similar to this example: ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 diff --git a/docs/hammerdb.md b/docs/hammerdb.md index 7d8151668..3dd8bfc5f 100644 --- a/docs/hammerdb.md +++ b/docs/hammerdb.md @@ -5,9 +5,9 @@ ## Running hammerdb Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/hammerdb_crds/ripsaw_v1alpha1_hammerdb_cr.yaml) to your needs. +you can modify [cr.yaml](../config/samples/hammerdb/cr.yaml) to your needs. -The hammerdb workload needs to be pointed at an existing [MSSQL](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_cr.yaml), [MARIADB](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_cr.yaml), [POSTGRES](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_cr.yaml) database via the CR file. +The hammerdb workload needs to be pointed at an existing [MSSQL](../config/samples/hammerdb/mssql/cr.yaml), [MARIADB](../config/samples/hammerdb/mariadb/cr.yaml), [POSTGRES](../config/samples/hammerdb/postgres/cr.yaml) database via the CR file. An example CR might look like this @@ -126,24 +126,24 @@ PVC: The data will be stored on Container Storage, it's required a Pre-installed MSSQL examples: -[MSSQL ephemeral](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server.yaml), -[MSSQL HostPath](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_hostpath.yaml), -[MSSQL Local](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_local.yaml), -[MSSQL PVC](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_server_pvc.yaml) +[MSSQL ephemeral](../config/samples/hammerdb/mssql/cr.yaml), +[MSSQL HostPath](../config/samples/hammerdb/mssql/hostpath-cr.yaml), +[MSSQL Local](../config/samples/hammerdb/mssql/local-cr.yaml), +[MSSQL PVC](../config/samples/hammerdb/mssql/vm-cr.yaml) Postgres examples: -[Postgres ephemeral](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server.yaml), -[Postgres HostPath](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_hostpath.yaml), -[Postgres Local](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_local.yaml), -[Postgres PVC](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_server_pvc.yaml) +[Postgres ephemeral](../config/samples/hammerdb/postgres/cr.yaml), +[Postgres HostPath](../config/samples/hammerdb/postgres/hostpath-cr.yaml), +[Postgres Local](../config/samples/hammerdb/postgres/local-cr.yaml), +[Postgres PVC](../config/samples/hammerdb/postgres/vm-cr.yaml) Mariadb examples: -[Mariadb ephemeral](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server.yaml), -[Mariadb HostPath](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_hostpath.yaml), -[Mariadb Local](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_local.yaml), -[Mariadb PVC](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_server_pvc.yaml) +[Mariadb ephemeral](../config/samples/hammerdb/mariadb/cr.yaml), +[Mariadb HostPath](../config/samples/hammerdb/mariadb/hostpath-cr.yaml), +[Mariadb Local](../config/samples/hammerdb/mariadb/local-cr.yaml), +[Mariadb PVC](../config/samples/hammerdb/mariadb/vm-cr.yaml) The option **runtime_class** can be set to specify an optional @@ -219,9 +219,9 @@ The `hostpath` feature is `true` for enabling HostPath on VM, there several para `hostpath_storagesize` the HostPath storage size examples: -[MSSQL](../resources/crds/hammerdb_crds/mssql/ripsaw_v1alpha1_hammerdb_mssql_vm.yaml), -[Postgres](../resources/crds/hammerdb_crds/postgres/ripsaw_v1alpha1_hammerdb_postgres_vm.yaml), -[Mariadb](../resources/crds/hammerdb_crds/mariadb/ripsaw_v1alpha1_hammerdb_mariadb_vm.yaml), +[MSSQL](../config/samples/hammerdb/mssql/vm-cr.yaml), +[Postgres](../config/samples/hammerdb/postgres/vm-cr.yaml), +[Mariadb](../config/samples/hammerdb/mariadb/vm-cr.yaml), Currently, we only support images that can be used as [containerDisk](https://docs.openshift.com/container-platform/4.6/virt/virtual_machines/virtual_disks/virt-using-container-disks-with-vms.html#virt-preparing-container-disk-for-vms_virt-using-container-disks-with-vms). diff --git a/docs/iperf.md b/docs/iperf.md index 146215b88..fb4a205db 100644 --- a/docs/iperf.md +++ b/docs/iperf.md @@ -5,7 +5,7 @@ ## Running iperf Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_iperf3_cr.yaml) +you can modify [cr.yaml](../config/samples/iperf3/cr.yaml) ```yaml diff --git a/docs/kube-burner.md b/docs/kube-burner.md index 792890218..329d5fbcc 100644 --- a/docs/kube-burner.md +++ b/docs/kube-burner.md @@ -19,7 +19,7 @@ The benchmark-operator integration here is meant to run only some workloads usef ## Running kube-burner Given that you followed instructions to deploy benchmark-operator. Kube-burner needs an additional serviceaccount and clusterrole to run. Available at [kube-burner-role.yml](../resources/kube-burner-role.yml) -You can modify kube-burner's [cr.yaml](../resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml) to fit your requirements. +You can modify kube-burner's [cr.yaml](../config/samples/kube-burner/cr.yaml) to fit your requirements. ---- @@ -64,7 +64,7 @@ Each iteration of this workload creates the following object: - 1 pod. (sleep) - **concurrent-builds**: Creates a buildconfig, imagestream and corresponding build for a set application. **This will create as many namespaces with these objects as the configured job_iterations**. -See https://github.com/cloud-bulldozer/e2e-benchmarking/tree/master/workloads/kube-burner/builds for example parameters for each application +See https://github.com/cloud-bulldozer/e2e-benchmarking/tree/master/workloads/kube-burner for example parameters for each application Each iteration of this workload creates the following object: - 1 imagestream (dependent on application type set) - 1 buildconfig (also dependent on application type set) diff --git a/docs/oslat.md b/docs/oslat.md index 6e2449a8c..9cd3db504 100644 --- a/docs/oslat.md +++ b/docs/oslat.md @@ -5,7 +5,7 @@ The goal of the oslat workload in the benchmark-operator is to run oslat inside ## Running oslat -Given that you followed instructions to deploy operator, you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_oslat.yaml) to your needs. +Given that you followed instructions to deploy operator, you can modify [cr.yaml](../config/samples/oslat/cr.yaml) to your needs. It is recommended to define pod requests and limits when running oslat test, to give guaranteed CPUs to the pods. It is also expected to have the realtime kernel installed with required isolation for pods using the [Performance Add-On Operator](https://github.com/openshift-kni/performance-addon-operators). diff --git a/docs/pgbench.md b/docs/pgbench.md index 689e19a7f..740073f38 100644 --- a/docs/pgbench.md +++ b/docs/pgbench.md @@ -5,7 +5,7 @@ ## Running pgbench Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_pgbench_cr.yaml) to your needs. +you can modify [cr.yaml](../config/samples/pgbench/cr.yaml) to your needs. The pgbench workload needs to be pointed at one or more existing PostgreSQL databases via the CR file. diff --git a/docs/servicemesh.md b/docs/servicemesh.md index 1c3e405a9..7e4ec1543 100644 --- a/docs/servicemesh.md +++ b/docs/servicemesh.md @@ -22,7 +22,7 @@ intended for Kata containers. ## Running the benchmark -Here is an example of the [benchmark CR](../resources/crds/ripsaw_v1alpha1_servicemesh_cr.yaml): +Here is an example of the [benchmark CR](../config/samples/servicemesh/cr.yaml): ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 @@ -108,7 +108,7 @@ spec: You can run it by: ```bash -oc apply -f resources/crds/ripsaw_v1alpha1_servicemesh_cr.yaml # if edited the original one +oc apply -f config/samples/servicemesh/cr.yaml # if edited the original one ``` ## Visualize the report diff --git a/docs/smallfile.md b/docs/smallfile.md index 9667e6f47..7b3955803 100644 --- a/docs/smallfile.md +++ b/docs/smallfile.md @@ -3,7 +3,7 @@ [Smallfile](https://github.com/distributed-system-analysis/smallfile) is a python-based distributed POSIX workload generator which can be used to quickly measure performance for a variety of metadata-intensive workloads across an entire cluster. ## Running Smallfile Benchmark using Ripsaw -Once the operator has been installed following the instructions, one needs to modify the clients parameter(which is currently set to 0), to value greater than 0 in [ripsaw_v1alpha1_smallfile_cr.yaml](../resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml) to run default "create" the test. Also, in addtion to that, smallfile operator is completely dependent on storageclass and storagesize. Please make sure to double check the parameters in CRD file. +Once the operator has been installed following the instructions, one needs to modify the clients parameter(which is currently set to 0), to value greater than 0 in [ripsaw_v1alpha1_smallfile_cr.yaml](../config/samples/smallfile/cr.yaml) to run default "create" the test. Also, in addtion to that, smallfile operator is completely dependent on storageclass and storagesize. Please make sure to double check the parameters in CRD file. ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 @@ -24,7 +24,7 @@ spec: storagesize: 5Gi # Provide if PV is needed ``` -Smallfile operator also gives the leverage to run multiple test operations in a user-defined sequence. Like in the [Custom Resource Definition file](../resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml), the series of operation can be mentioned as: +Smallfile operator also gives the leverage to run multiple test operations in a user-defined sequence. Like in the [Custom Resource Definition file](../config/samples/smallfile/cr.yaml), the series of operation can be mentioned as: ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 diff --git a/docs/stressng.md b/docs/stressng.md index 72b43b727..61ec360ce 100644 --- a/docs/stressng.md +++ b/docs/stressng.md @@ -5,7 +5,7 @@ ## Running stressng Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_stressng.yaml) to your needs. +you can modify [cr.yaml](../config/samples/stressng/cr.yaml) to your needs. The optional argument **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is diff --git a/docs/sysbench.md b/docs/sysbench.md index 8dfc13f2f..46e5ccc56 100644 --- a/docs/sysbench.md +++ b/docs/sysbench.md @@ -5,14 +5,14 @@ ## Running Sysbench Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml) +you can modify [cr.yaml](../config/samples/sysbench/cr.yaml) The optional argument **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. Note: please ensure you set 0 for other workloads if editing the -[cr.yaml](../resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml) file otherwise +[cr.yaml](../config/samples/sysbench/cr.yaml) file otherwise your resource file may look like this: diff --git a/docs/uperf.md b/docs/uperf.md index 0de22ed8c..799fa4f00 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -5,7 +5,7 @@ ## Running UPerf Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_uperf_cr.yaml) +you can modify [cr.yaml](../config/samples/uperf/cr.yaml) ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 diff --git a/docs/vegeta.md b/docs/vegeta.md index e387d75cf..a0623a42e 100644 --- a/docs/vegeta.md +++ b/docs/vegeta.md @@ -5,7 +5,7 @@ ## Running Vegeta Given that you followed instructions to deploy operator, -you can modify Vegeta's [cr.yaml](../resources/crds/ripsaw_v1alpha1_vegeta_cr.yaml) to make it fit with your requirements. +you can modify Vegeta's [cr.yaml](../config/samples/vegeta/cr.yaml) to make it fit with your requirements. The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily diff --git a/docs/ycsb.md b/docs/ycsb.md index d916158a1..0a6162a66 100644 --- a/docs/ycsb.md +++ b/docs/ycsb.md @@ -5,7 +5,7 @@ ## Running YCSB Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_ycsb_cr.yaml) to your needs. +you can modify [cr.yaml](../config/samples/ycsb/cr.yaml) to your needs. YCSB is a workload that requires a database/key-value store to run workloads against and benchmark. From 8b6de5e5ce059ba7dc16b6719c4cbf3f30f6ae48 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 2 Aug 2021 13:40:07 +0200 Subject: [PATCH 103/249] Fix system-metrics collection Signed-off-by: Raul Sevilla --- roles/system-metrics/tasks/main.yml | 2 +- .../templates/kube-burner.yml.j2 | 2 +- .../system-metrics/templates/node-metrics.yml | 45 +++++++++---------- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/roles/system-metrics/tasks/main.yml b/roles/system-metrics/tasks/main.yml index 361499297..09af09a6c 100644 --- a/roles/system-metrics/tasks/main.yml +++ b/roles/system-metrics/tasks/main.yml @@ -12,7 +12,7 @@ namespace: "{{ operator_namespace }}" data: index.yml: "{{ lookup('template', 'index.yml')}}" - metrics.yml: "{{ lookup('template', 'node-metrics.yml')}}" + node-metrics.yml: "{{ lookup('template', 'node-metrics.yml')}}" - name: Launching kube-burner job to index system-metrics k8s: diff --git a/roles/system-metrics/templates/kube-burner.yml.j2 b/roles/system-metrics/templates/kube-burner.yml.j2 index 9117725a7..03f8cb8e7 100644 --- a/roles/system-metrics/templates/kube-burner.yml.j2 +++ b/roles/system-metrics/templates/kube-burner.yml.j2 @@ -30,7 +30,7 @@ spec: -u {{ system_metrics.prom_url }} -t {{ system_metrics.prom_token }} --start={{ (cr_state.resources[0].metadata.creationTimestamp|to_datetime('%Y-%m-%dT%H:%M:%SZ')).strftime('%s') }} - -m {{ system_metrics.metrics_profile|default("metrics.yml") }} + -m {{ system_metrics.metrics_profile|default("node-metrics.yml") }} --step={{ system_metrics.step }} volumeMounts: - name: system-metrics-collector diff --git a/roles/system-metrics/templates/node-metrics.yml b/roles/system-metrics/templates/node-metrics.yml index 3e9c7f7ef..eeb1128b9 100644 --- a/roles/system-metrics/templates/node-metrics.yml +++ b/roles/system-metrics/templates/node-metrics.yml @@ -1,32 +1,31 @@ -metrics: - - query: sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) > 0 - metricName: nodeCPU +- query: sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) > 0 + metricName: nodeCPU - - query: avg(node_memory_MemAvailable_bytes) by (instance) - metricName: nodeMemoryAvailable +- query: avg(node_memory_MemAvailable_bytes) by (instance) + metricName: nodeMemoryAvailable - - query: avg(node_memory_Active_bytes) by (instance) - metricName: nodeMemoryActive +- query: avg(node_memory_Active_bytes) by (instance) + metricName: nodeMemoryActive - - query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) - metricName: nodeMemoryCached+nodeMemoryBuffers +- query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) + metricName: nodeMemoryCached+nodeMemoryBuffers - - query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) - metricName: rxNetworkBytes +- query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: rxNetworkBytes - - query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) - metricName: txNetworkBytes +- query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: txNetworkBytes - - query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) - metricName: nodeDiskWrittenBytes +- query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskWrittenBytes - - query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) - metricName: nodeDiskReadBytes +- query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskReadBytes - - query: kube_node_role - metricName: nodeRoles - instant: true +- query: kube_node_role + metricName: nodeRoles + instant: true - - query: cluster_version{type="completed"} - metricName: clusterVersion - instant: true +- query: cluster_version{type="completed"} + metricName: clusterVersion + instant: true From 9187a21bfcc8d8f502857538279bedfab5cda6bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Tue, 3 Aug 2021 18:51:18 +0200 Subject: [PATCH 104/249] Container build mechanism (#613) * Benchmark-operator container release Signed-off-by: Raul Sevilla * Update dnf repository Signed-off-by: Raul Sevilla * Push manifests Signed-off-by: Raul Sevilla * Login into quay Signed-off-by: Raul Sevilla * Replace podman-* by image-* Signed-off-by: Raul Sevilla * Fix version name retrieval Signed-off-by: Raul Sevilla * Update helm chart and docs Signed-off-by: Raul Sevilla --- .github/workflows/release.yml | 60 ++++++++++++++++++++++++++ Dockerfile | 4 +- Makefile | 49 ++++++++++++++------- README.md | 5 +-- charts/benchmark-operator/README.md | 2 +- charts/benchmark-operator/values.yaml | 2 +- hack/tag_name.sh | 7 +++ image_resources/centos8-appstream.repo | 16 ++++++- tests/common.sh | 2 +- 9 files changed, 120 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100755 hack/tag_name.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..a04d8700c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,60 @@ +name: Release benchmark-operator container images +on: + push: + branches: + - master + tags: + - "*" # triggers only if push new tag version + +jobs: + containers: + name: Build container images + runs-on: ubuntu-latest + strategy: + matrix: + arch: + - arm64 + - amd64 + - ppc64le + env: + ORG: cloud-bulldozer + ARCH: ${{ matrix.arch }} + steps: + + - name: Install dependencies required for multi-arch builds + run: sudo apt-get install qemu-user-static podman fuse-overlayfs + + - name: Check out code + uses: actions/checkout@v2 + + - name: Login in quay + run: podman login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN} + env: + QUAY_USER: ${{ secrets.QUAY_USER }} + QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + + - name: Build benchmark-operator container + run: make image-build + + - name: Push benchmark-operator container + run: make image-push + + manifest: + name: Build container manifest + runs-on: ubuntu-latest + env: + MANIFEST_ARCHS: amd64 arm64 ppc64le + needs: containers + steps: + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Login in quay + run: podman login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN} + env: + QUAY_USER: ${{ secrets.QUAY_USER }} + QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + + - name: Build and push container manifest + run: make manifest diff --git a/Dockerfile b/Dockerfile index 59c30f046..e63517e58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,10 +6,10 @@ RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \ && chmod -R ug+rwx ${HOME}/.ansible COPY image_resources/centos8-appstream.repo /etc/yum.repos.d/centos8-appstream.repo -RUN dnf install -y --nodocs redis openssl --enablerepo=centos8-appstream && dnf clean all +RUN dnf install -y --nodocs redis openssl --enablerepo=centos8-appstream-* && dnf clean all COPY resources/kernel-cache-drop-daemonset.yaml /opt/kernel_cache_dropper/ COPY watches.yaml ${HOME}/watches.yaml COPY roles/ ${HOME}/roles/ COPY playbooks/ ${HOME}/playbooks/ -USER 1001 \ No newline at end of file +USER 1001 diff --git a/Makefile b/Makefile index 4bc5e712f..8cc6d88b7 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.0.1 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "preview,fast,stable") @@ -35,10 +34,26 @@ IMAGE_TAG_BASE ?= cloudbulldozer.io/ripsaw # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) +OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +ARCH ?= $(shell uname -m | sed 's/x86_64/amd64/') + # Image URL to use all building/pushing image targets -IMG ?= quay.io/benchmark-operator/benchmark-operator:master +REGISTRY ?= quay.io +ORG ?= cloud-bulldozer +# Get the current branch/tag name +# In case this is the master branch, rename it to latest +VERSION ?= $(shell hack/tag_name.sh) +IMG = $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) +MANIFEST_ARCHS ?= amd64 arm64 ppc64le + +# Containers +ifeq (, $(shell which podman)) + ENGINE := docker +else + ENGINE := podman +endif -all: docker-build +all: image-build ##@ General @@ -61,17 +76,21 @@ help: ## Display this help. run: ansible-operator ## Run against the configured Kubernetes cluster in ~/.kube/config $(ANSIBLE_OPERATOR) run -docker-build: ## Build docker image with the manager. - docker build -t ${IMG} . +image-build: ## Build container image with the manager. + ${ENGINE} build --arch=$(ARCH) -t ${IMG}-${ARCH} . -docker-push: ## Push docker image with the manager. - docker push ${IMG} +image-push: ## Push container image with the manager. + ${ENGINE} push ${IMG}-${ARCH} -podman-build: ## Build docker image with the manager. - podman build -t ${IMG} . +manifest: manifest-build ## Builds a container manifest and push it to the registry + $(ENGINE) manifest push $(IMG) $(IMG) -podman-push: ## Push docker image with the manager. - podman push ${IMG} +manifest-build: + $(ENGINE) manifest create $(IMG) + @for arch in $(MANIFEST_ARCHS); do \ + echo "Adding $(IMG)-$${arch} to manifest ${IMG}"; \ + $(ENGINE) manifest add $(IMG) $(IMG)-$${arch}; \ + done ##@ Deployment @@ -88,8 +107,6 @@ deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/c undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/default | kubectl delete -f - -OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') -ARCH := $(shell uname -m | sed 's/x86_64/amd64/') .PHONY: kustomize KUSTOMIZE = $(shell pwd)/bin/kustomize @@ -132,11 +149,11 @@ bundle: kustomize ## Generate bundle manifests and metadata, then validate gener .PHONY: bundle-build bundle-build: ## Build the bundle image. - docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . + ${ENGINE} build -f bundle.Dockerfile -t $(BUNDLE_IMG) . .PHONY: bundle-push bundle-push: ## Push the bundle image. - $(MAKE) docker-push IMG=$(BUNDLE_IMG) + $(MAKE) image-push IMG=$(BUNDLE_IMG) .PHONY: opm OPM = ./bin/opm @@ -176,4 +193,4 @@ catalog-build: opm ## Build a catalog image. # Push the catalog image. .PHONY: catalog-push catalog-push: ## Push a catalog image. - $(MAKE) docker-push IMG=$(CATALOG_IMG) + $(MAKE) image-push IMG=$(CATALOG_IMG) diff --git a/README.md b/README.md index 476b651af..5516e3307 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ If you wish to build a version of the operator from your local copy of the repo, ```bash git clone https://github.com/cloud-bulldozer/benchmark-operator -make podman-build podman-push deploy IMG=$YOUR_IMAGE +make image-build image-push deploy IMG=$YOUR_IMAGE ``` > Note: building the image requires podman @@ -157,9 +157,6 @@ spec: image: my.location/foo:latest ``` -## Installation -[Installation](docs/installation.md) - ## Contributing [Contributing](CONTRIBUTING.md) diff --git a/charts/benchmark-operator/README.md b/charts/benchmark-operator/README.md index aff72d471..da778f107 100644 --- a/charts/benchmark-operator/README.md +++ b/charts/benchmark-operator/README.md @@ -12,7 +12,7 @@ A Helm chart for Kubernetes | nameOverride | string | `""` | | | operator.affinity.nodeAffinity | object | `{"preferredDuringSchedulingIgnoredDuringExecution":[{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/workload","operator":"In","values":[""]}]},"weight":100}]}` | nodeAffinity is set to make the operator run on workload nodes, but it's not required | | operator.image.pullPolicy | string | `"Always"` | | -| operator.image.repository | string | `"quay.io/benchmark-operator/benchmark-operator"` | | +| operator.image.repository | string | `"quay.io/cloud-bulldozer/benchmark-operator"` | | | operator.image.tag | string | `""` | | | operator.redisImage.pullPolicy | string | `"Always"` | | | operator.redisImage.repository | string | `"k8s.gcr.io/redis"` | | diff --git a/charts/benchmark-operator/values.yaml b/charts/benchmark-operator/values.yaml index 379f03664..7b2a24779 100644 --- a/charts/benchmark-operator/values.yaml +++ b/charts/benchmark-operator/values.yaml @@ -18,7 +18,7 @@ operator: replicaCount: 1 image: - repository: quay.io/benchmark-operator/benchmark-operator + repository: quay.io/cloud-bulldozer/benchmark-operator pullPolicy: Always tag: master diff --git a/hack/tag_name.sh b/hack/tag_name.sh new file mode 100755 index 000000000..edf670fcc --- /dev/null +++ b/hack/tag_name.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +if [[ -z $(git branch --show-current) ]]; then + git describe --tags +else + git branch --show-current | sed 's/master/latest/g' +fi diff --git a/image_resources/centos8-appstream.repo b/image_resources/centos8-appstream.repo index 5f2902c0f..b35453861 100644 --- a/image_resources/centos8-appstream.repo +++ b/image_resources/centos8-appstream.repo @@ -1,5 +1,17 @@ -[centos8-appstream] -name=CentOS-8-Appstream +[centos8-appstream-x86_64] +name=CentOS-8-Appstream-x86_64 baseurl=http://mirror.centos.org/centos/8/AppStream/x86_64/os/ enabled=0 gpgcheck=0 + +[centos8-appstream-aarch64] +name=CentOS-8-Appstream-aarch64 +baseurl=http://mirror.centos.org/centos/8/AppStream/aarch64/os/ +enabled=0 +gpgcheck=0 + +[centos8-appstream-ppc64le] +name=CentOS-8-Appstream-ppc64le +baseurl=http://mirror.centos.org/centos/8/AppStream/ppc64le//os/ +enabled=0 +gpgcheck=0 diff --git a/tests/common.sh b/tests/common.sh index 2f07ed0f3..bf2e89c4e 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -145,7 +145,7 @@ function pod_count () { function apply_operator { tag_name="${NODE_NAME:-master}" - make podman-build podman-push deploy IMG=$image_location/$image_account/benchmark-operator:$tag_name + make image-build image-push deploy IMG=$image_location/$image_account/benchmark-operator:$tag_name kubectl wait --for=condition=available "deployment/benchmark-controller-manager" -n benchmark-operator --timeout=300s } From b36775ff2fbf6e3c9948ff80680862d9247c0d7d Mon Sep 17 00:00:00 2001 From: Gavin Jefferies Date: Tue, 3 Aug 2021 10:16:54 -0700 Subject: [PATCH 105/249] Backpack needs a RW root (#612) * backpack needs a RW root * backpack init needs a RW root --- playbooks/templates/metadata.yml.j2 | 1 + roles/backpack/templates/backpack.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/playbooks/templates/metadata.yml.j2 b/playbooks/templates/metadata.yml.j2 index e4fc6a7c9..663469d70 100644 --- a/playbooks/templates/metadata.yml.j2 +++ b/playbooks/templates/metadata.yml.j2 @@ -29,6 +29,7 @@ imagePullPolicy: Always securityContext: privileged: {{ metadata.privileged }} + readOnlyRootFilesystem: false runAsNonRoot: false env: - name: my_node_name diff --git a/roles/backpack/templates/backpack.yml b/roles/backpack/templates/backpack.yml index 8d4f74e6c..5729ef89f 100755 --- a/roles/backpack/templates/backpack.yml +++ b/roles/backpack/templates/backpack.yml @@ -61,6 +61,7 @@ spec: imagePullPolicy: Always securityContext: privileged: {{ metadata.privileged }} + readOnlyRootFilesystem: false runAsNonRoot: false readinessProbe: exec: From 7843d381ac2f263b76c1acee7a44a85f61d05094 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 4 Aug 2021 17:25:33 +0200 Subject: [PATCH 106/249] Use IMG to push the image Signed-off-by: Raul Sevilla --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8cc6d88b7..1eb39edfb 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ ORG ?= cloud-bulldozer # Get the current branch/tag name # In case this is the master branch, rename it to latest VERSION ?= $(shell hack/tag_name.sh) -IMG = $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) +IMG ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION)-$(ARCH) MANIFEST_ARCHS ?= amd64 arm64 ppc64le # Containers @@ -77,10 +77,10 @@ run: ansible-operator ## Run against the configured Kubernetes cluster in ~/.kub $(ANSIBLE_OPERATOR) run image-build: ## Build container image with the manager. - ${ENGINE} build --arch=$(ARCH) -t ${IMG}-${ARCH} . + ${ENGINE} build --arch=$(ARCH) -t $(IMG) . image-push: ## Push container image with the manager. - ${ENGINE} push ${IMG}-${ARCH} + ${ENGINE} push $(IMG) manifest: manifest-build ## Builds a container manifest and push it to the registry $(ENGINE) manifest push $(IMG) $(IMG) From a05ffac63467567ccc344331292d2d6f9d7c763f Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 5 Aug 2021 08:45:17 +0200 Subject: [PATCH 107/249] Fix manifest pushing Signed-off-by: Raul Sevilla --- Makefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1eb39edfb..cac528606 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,8 @@ ORG ?= cloud-bulldozer # In case this is the master branch, rename it to latest VERSION ?= $(shell hack/tag_name.sh) IMG ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION)-$(ARCH) +IMG_NOARCH ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) +MANIFEST ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) MANIFEST_ARCHS ?= amd64 arm64 ppc64le # Containers @@ -83,13 +85,13 @@ image-push: ## Push container image with the manager. ${ENGINE} push $(IMG) manifest: manifest-build ## Builds a container manifest and push it to the registry - $(ENGINE) manifest push $(IMG) $(IMG) + $(ENGINE) manifest push $(MANIFEST) $(MANIFEST) manifest-build: - $(ENGINE) manifest create $(IMG) + $(ENGINE) manifest create $(MANIFEST) @for arch in $(MANIFEST_ARCHS); do \ - echo "Adding $(IMG)-$${arch} to manifest ${IMG}"; \ - $(ENGINE) manifest add $(IMG) $(IMG)-$${arch}; \ + echo "Adding $(IMG_NOARCH)-$${arch} to manifest ${MANIFEST}"; \ + $(ENGINE) manifest add $(MANIFEST) $(IMG_NOARCH)-$${arch}; \ done ##@ Deployment From 0c88b6aa10789a5d9ff0cb5b035771f1c3f12b58 Mon Sep 17 00:00:00 2001 From: ebattat Date: Thu, 5 Aug 2021 10:22:01 +0300 Subject: [PATCH 108/249] add vm completed state --- roles/stressng/templates/stressng_workload_vm.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/stressng/templates/stressng_workload_vm.yml.j2 b/roles/stressng/templates/stressng_workload_vm.yml.j2 index 0d8435cf3..b9d1231a2 100644 --- a/roles/stressng/templates/stressng_workload_vm.yml.j2 +++ b/roles/stressng/templates/stressng_workload_vm.yml.j2 @@ -82,6 +82,7 @@ spec: - dnf install -y git redis stress-ng - pip install git+https://github.com/cloud-bulldozer/benchmark-wrapper - run_snafu --tool stressng -j /tmp/stressng-test/jobfile -u {{ uuid }} + - redis-cli -h {{bo.resources[0].status.podIP}} set complete true name: cloudinitdisk - configMap: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" From 4a79d3378fd89feb58e7cc6aa2c41e35228ca318 Mon Sep 17 00:00:00 2001 From: ebattat <73884315+ebattat@users.noreply.github.com> Date: Thu, 5 Aug 2021 13:17:04 +0300 Subject: [PATCH 109/249] add redis completed status --- roles/hammerdb/templates/db_creation_mariadb_vm.yml | 1 + roles/hammerdb/templates/db_creation_mssql_vm.yml | 1 + roles/hammerdb/templates/db_creation_postgres_vm.yml | 1 + roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 | 2 +- roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 | 2 +- roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 | 2 +- 6 files changed, 6 insertions(+), 3 deletions(-) diff --git a/roles/hammerdb/templates/db_creation_mariadb_vm.yml b/roles/hammerdb/templates/db_creation_mariadb_vm.yml index 98dd0d7c8..1e937ea92 100644 --- a/roles/hammerdb/templates/db_creation_mariadb_vm.yml +++ b/roles/hammerdb/templates/db_creation_mariadb_vm.yml @@ -97,6 +97,7 @@ spec: - systemctl start redis - systemctl enable redis - bash /tmp/hammerdb-mariadb-test/run_mariadb_script.sh + - redis-cli -h {{bo.resources[0].status.podIP}} set complete true name: cloudinitdisk - configMap: name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" diff --git a/roles/hammerdb/templates/db_creation_mssql_vm.yml b/roles/hammerdb/templates/db_creation_mssql_vm.yml index 6760b76e0..78f2b5a35 100644 --- a/roles/hammerdb/templates/db_creation_mssql_vm.yml +++ b/roles/hammerdb/templates/db_creation_mssql_vm.yml @@ -98,6 +98,7 @@ spec: - systemctl start redis - systemctl enable redis - bash /tmp/hammerdb-mssql-test/run_mssql_script.sh + - redis-cli -h {{bo.resources[0].status.podIP}} set complete true name: cloudinitdisk - configMap: name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" diff --git a/roles/hammerdb/templates/db_creation_postgres_vm.yml b/roles/hammerdb/templates/db_creation_postgres_vm.yml index c609ced87..10a8c7b77 100644 --- a/roles/hammerdb/templates/db_creation_postgres_vm.yml +++ b/roles/hammerdb/templates/db_creation_postgres_vm.yml @@ -97,6 +97,7 @@ spec: - systemctl start redis - systemctl enable redis - bash /tmp/hammerdb-postgres-test/run_postgres_script.sh + - redis-cli -h {{bo.resources[0].status.podIP}} set complete true name: cloudinitdisk - configMap: name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" diff --git a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 index 91f7f459a..6a34b9aec 100644 --- a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 @@ -59,4 +59,4 @@ cd /hammer; ./hammerdbcli auto /creator/createdb.tcl; redis-cli set db-creation-{{trunc_uuid}} true; run_snafu --tool hammerdb -u {{ uuid }}; -redis-cli set complete true; + diff --git a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 index a18470733..409a2b75f 100644 --- a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 @@ -63,4 +63,4 @@ cd /hammer; ./hammerdbcli auto /creator/createdb.tcl; redis-cli set db-creation-{{trunc_uuid}} true; run_snafu --tool hammerdb -u {{ uuid }}; -redis-cli set complete true; + diff --git a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 index 62580e6b4..e8544ef1e 100644 --- a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 @@ -68,4 +68,4 @@ cd /hammer; ./hammerdbcli auto /creator/createdb.tcl; redis-cli set db-creation-{{trunc_uuid}} true; run_snafu --tool hammerdb -u {{ uuid }}; -redis-cli set complete true; + From 8d7283560e81509325ddd47c403301a1b108c1b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Thu, 5 Aug 2021 19:07:00 +0200 Subject: [PATCH 110/249] --show-current flag is not available for git versions < 2.22 (#633) Signed-off-by: Raul Sevilla --- Makefile | 2 +- hack/tag_name.sh | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100755 hack/tag_name.sh diff --git a/Makefile b/Makefile index cac528606..2e11b124b 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ REGISTRY ?= quay.io ORG ?= cloud-bulldozer # Get the current branch/tag name # In case this is the master branch, rename it to latest -VERSION ?= $(shell hack/tag_name.sh) +VERSION ?= $(shell git describe --tags 2>/dev/null || git rev-parse --abbrev-ref HEAD | sed 's/master/latest/g') IMG ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION)-$(ARCH) IMG_NOARCH ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) MANIFEST ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) diff --git a/hack/tag_name.sh b/hack/tag_name.sh deleted file mode 100755 index edf670fcc..000000000 --- a/hack/tag_name.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -if [[ -z $(git branch --show-current) ]]; then - git describe --tags -else - git branch --show-current | sed 's/master/latest/g' -fi From 72b641be405fa12cb1b6c5ee8c463820fdb7b58f Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Sat, 7 Aug 2021 01:30:17 +0200 Subject: [PATCH 111/249] Add IMG_ARCH Signed-off-by: Raul Sevilla --- Makefile | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 2e11b124b..eb2750feb 100644 --- a/Makefile +++ b/Makefile @@ -43,9 +43,8 @@ ORG ?= cloud-bulldozer # Get the current branch/tag name # In case this is the master branch, rename it to latest VERSION ?= $(shell git describe --tags 2>/dev/null || git rev-parse --abbrev-ref HEAD | sed 's/master/latest/g') -IMG ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION)-$(ARCH) -IMG_NOARCH ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) -MANIFEST ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) +IMG ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) +IMG_ARCH = $(IMG)-$(ARCH) MANIFEST_ARCHS ?= amd64 arm64 ppc64le # Containers @@ -79,19 +78,19 @@ run: ansible-operator ## Run against the configured Kubernetes cluster in ~/.kub $(ANSIBLE_OPERATOR) run image-build: ## Build container image with the manager. - ${ENGINE} build --arch=$(ARCH) -t $(IMG) . + ${ENGINE} build --arch=$(ARCH) -t $(IMG_ARCH) . image-push: ## Push container image with the manager. - ${ENGINE} push $(IMG) + ${ENGINE} push $(IMG_ARCH) manifest: manifest-build ## Builds a container manifest and push it to the registry - $(ENGINE) manifest push $(MANIFEST) $(MANIFEST) + $(ENGINE) manifest push $(IMG) $(IMG) manifest-build: - $(ENGINE) manifest create $(MANIFEST) + $(ENGINE) manifest create $(IMG) @for arch in $(MANIFEST_ARCHS); do \ - echo "Adding $(IMG_NOARCH)-$${arch} to manifest ${MANIFEST}"; \ - $(ENGINE) manifest add $(MANIFEST) $(IMG_NOARCH)-$${arch}; \ + echo "Adding $(IMG)-$${arch} to manifest ${IMG}"; \ + $(ENGINE) manifest add $(IMG) $(IMG)-$${arch}; \ done ##@ Deployment From b8a336c55ac16372c7555f21d567feb0517f05ef Mon Sep 17 00:00:00 2001 From: dry923 Date: Wed, 4 Aug 2021 15:57:56 -0400 Subject: [PATCH 112/249] remove if from populate_test_list function --- tests/common.sh | 101 ++++++++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 54 deletions(-) diff --git a/tests/common.sh b/tests/common.sh index bf2e89c4e..302e3d144 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -9,60 +9,53 @@ echo "using container image location $image_location and account $image_account" function populate_test_list { rm -f tests/iterate_tests - if [ -n $1 ]; then - echo $1 >> tests/iterate_tests - else - - for item in $@ - do - # Check for changes in roles - if [[ $(echo ${item} | grep 'roles/fs-drift') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/uperf') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/fio_distributed') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/iperf3') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/byowl') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/sysbench') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/pgbench') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/ycsb') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/backpack') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/hammerdb') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/smallfile') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/vegeta') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/stressng') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/scale_openshift') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/kube-burner') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/flent') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/log_generator') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/image_pull') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi - - - # Check for changes in cr files - if [[ $(echo ${item} | grep 'valid_backpack*') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_byowl*') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_fiod*') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_fs_drift*') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_hammerdb*') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_iperf3*') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_pgbench*') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_smallfile*') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_sysbench*') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_uperf*') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_ycsb*') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_vegeta*') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_stressng*') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_scale*') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_kube-burner*') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_flent*') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_log_generator*') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_image_pull*') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi - - - # Check for changes in test scripts - test_check=`echo $item | awk -F / '{print $2}'` - - if [[ $(echo ${test_check} | grep 'test_.*.sh') ]]; then echo ${test_check} >> tests/iterate_tests; fi - done - fi + for item in $@ + do + # Check for changes in roles + if [[ $(echo ${item} | grep 'roles/fs-drift') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/uperf') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/fio_distributed') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/iperf3') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/byowl') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/sysbench') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/pgbench') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/ycsb') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/backpack') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/hammerdb') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/smallfile') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/vegeta') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/stressng') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/scale_openshift') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/kube-burner') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/flent') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/log_generator') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/image_pull') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi + + # Check for changes in cr files + if [[ $(echo ${item} | grep 'valid_backpack*') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_byowl*') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_fiod*') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_fs_drift*') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_hammerdb*') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_iperf3*') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_pgbench*') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_smallfile*') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_sysbench*') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_uperf*') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_ycsb*') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_vegeta*') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_stressng*') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_scale*') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_kube-burner*') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_flent*') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_log_generator*') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_image_pull*') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi + + # Check for changes in test scripts + test_check=`echo $item | awk -F / '{print $2}'` + + if [[ $(echo ${test_check} | grep 'test_.*.sh') ]]; then echo ${test_check} >> tests/iterate_tests; fi + done } function delete_benchmark { From 89dbce5f2de73cdd5912c6120dffa6c582b643d3 Mon Sep 17 00:00:00 2001 From: Ben England Date: Wed, 4 Aug 2021 13:01:03 -0400 Subject: [PATCH 113/249] fio prefill should use same numjobs value as fio samples move numjobs loop up above prefill section pass numjobs var into prefill fio job file --- roles/fio_distributed/templates/configmap.yml.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/fio_distributed/templates/configmap.yml.j2 b/roles/fio_distributed/templates/configmap.yml.j2 index f57de4649..06cf1463b 100644 --- a/roles/fio_distributed/templates/configmap.yml.j2 +++ b/roles/fio_distributed/templates/configmap.yml.j2 @@ -5,6 +5,8 @@ metadata: name: fio-test-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' data: +# FIXME: I don't think prefill works correctly for a list of numjobs values, only for 1 of them +{% for numjobs in workload_args.numjobs %} {% if workload_args.prefill is defined and workload_args.prefill is sameas true %} fiojob-prefill: | [global] @@ -22,7 +24,7 @@ data: {% endif %} iodepth=1 direct=1 - numjobs=1 + numjobs={{numjobs}} [write] rw=write @@ -40,7 +42,6 @@ data: {% set loopvar = workload_args.bs %} {% set loopvar_str = 'bs' %} {% endif %} -{% for numjobs in workload_args.numjobs %} {% for i in loopvar %} {% for job in workload_args.jobs %} fiojob-{{job}}-{{i}}-{{numjobs}}: | From 39b1a1b7726119972b7bf4ed054eb37614ba9b24 Mon Sep 17 00:00:00 2001 From: Ben England Date: Mon, 9 Aug 2021 15:17:17 -0400 Subject: [PATCH 114/249] Smf cleanup delay (#597) * add new smallfile parameter to CR * test as many smallfile params as you can * always pass through value specified in CR * shrink time required to run test --- .../smallfilejob.other_parameters.yaml.j2 | 16 ++++++++++------ tests/test_crs/valid_smallfile.yaml | 17 ++++++++++++++++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 b/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 index 5540a5fcb..cfc766eca 100644 --- a/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 +++ b/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 @@ -15,11 +15,11 @@ files: {{ workload_args.files }} {% endif %} {% if workload_args.fsync is defined %} -fsync: y +fsync: {{ workload_args.fsync }} {% endif %} {% if workload_args.response_times is defined %} -response-times: true +response-times: {{ workload_args.response_times }} {% endif %} {% if workload_args.network_sync_dir is defined %} @@ -79,17 +79,21 @@ verbose: {{workload_args.verbose}} {% endif %} {% if workload_args.permute_host_dirs is defined %} -permute-host-dirs: true +permute-host-dirs: {{ workload_args.permute_host_dirs }} {% endif %} {% if workload_args.record_ctime_size is defined %} -record-ctime-size: true +record-ctime-size: {{ workload_args.record_ctime_size }} {% endif %} {% if workload_args.verify_read is defined %} -verify-read: true +verify-read: {{ workload_args.verify_read }} {% endif %} {% if workload_args.incompressible is defined %} -incompressible: true +incompressible: {{ workload_args.incompressible }} +{% endif %} + +{% if workload_args.cleanup_delay_usec_per_file is defined %} +cleanup-delay-usec-per-file: {{workload_args.cleanup_delay_usec_per_file}} {% endif %} diff --git a/tests/test_crs/valid_smallfile.yaml b/tests/test_crs/valid_smallfile.yaml index 8d2dd480b..0541aa915 100644 --- a/tests/test_crs/valid_smallfile.yaml +++ b/tests/test_crs/valid_smallfile.yaml @@ -25,6 +25,21 @@ spec: clients: 2 operation: ["create", "read", "append", "delete"] threads: 1 - file_size: 0 + file_size: 2 + record_size: 1 + xattr_size: 50 + pause: 2 + stonewall: n + finish: y + prefix: abc + hash_into_dirs: true + same_dir: y + incompressible: y + verify_read: y + xattr_count: 10 + fsync: false + files_per_dir: 2000 + dirs_per_dir: 4 files: 100000 + cleanup_delay_usec_per_file: 200 debug: true From 43ea414e710d8141468dbf6830ca875d914ebf01 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 10 Aug 2021 09:28:46 +0200 Subject: [PATCH 115/249] Optional IMAGE_ARCH to build and push containers with different architectures Signed-off-by: Raul Sevilla --- .github/workflows/release.yml | 2 +- Makefile | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a04d8700c..46ba91369 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,7 @@ jobs: - ppc64le env: ORG: cloud-bulldozer - ARCH: ${{ matrix.arch }} + IMAGE_ARCH: ${{ matrix.arch }} steps: - name: Install dependencies required for multi-arch builds diff --git a/Makefile b/Makefile index eb2750feb..e7c1a03aa 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,10 @@ ORG ?= cloud-bulldozer # In case this is the master branch, rename it to latest VERSION ?= $(shell git describe --tags 2>/dev/null || git rev-parse --abbrev-ref HEAD | sed 's/master/latest/g') IMG ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) -IMG_ARCH = $(IMG)-$(ARCH) +ifdef IMAGE_ARCH +IMG := $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION)-$(IMAGE_ARCH) +BUILD_FLAGS := --arch=$(ARCH) +endif MANIFEST_ARCHS ?= amd64 arm64 ppc64le # Containers @@ -78,10 +81,10 @@ run: ansible-operator ## Run against the configured Kubernetes cluster in ~/.kub $(ANSIBLE_OPERATOR) run image-build: ## Build container image with the manager. - ${ENGINE} build --arch=$(ARCH) -t $(IMG_ARCH) . + ${ENGINE} build $(BUILD_FLAGS) -t $(IMG) . image-push: ## Push container image with the manager. - ${ENGINE} push $(IMG_ARCH) + ${ENGINE} push $(IMG) manifest: manifest-build ## Builds a container manifest and push it to the registry $(ENGINE) manifest push $(IMG) $(IMG) From f9388f762551c1327f76842a4ecd1e49ffb6f571 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 10 Aug 2021 09:36:02 +0200 Subject: [PATCH 116/249] Fix typo Signed-off-by: Raul Sevilla --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e7c1a03aa..9348e39ff 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ VERSION ?= $(shell git describe --tags 2>/dev/null || git rev-parse --abbrev-ref IMG ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) ifdef IMAGE_ARCH IMG := $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION)-$(IMAGE_ARCH) -BUILD_FLAGS := --arch=$(ARCH) +BUILD_FLAGS := --arch=$(IMAGE_ARCH) endif MANIFEST_ARCHS ?= amd64 arm64 ppc64le From b6e4b58cdee4d7b126216bd465ade8777eff0d65 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 10 Aug 2021 12:38:24 +0200 Subject: [PATCH 117/249] Deprecate cerberus integration Signed-off-by: Raul Sevilla --- README.md | 3 - .../ripsaw.cloudbulldozer.io_benchmarks.yaml | 10 --- docs/cerberus.md | 61 ------------------- docs/testpmd.md | 4 +- playbooks/benchmark.yml | 1 - roles/benchmark_state/tasks/failure.yml | 1 - roles/cerberus/tasks/main.yml | 29 --------- 7 files changed, 2 insertions(+), 107 deletions(-) delete mode 100644 docs/cerberus.md delete mode 100644 roles/cerberus/tasks/main.yml diff --git a/README.md b/README.md index 5516e3307..1a092b57f 100644 --- a/README.md +++ b/README.md @@ -163,9 +163,6 @@ spec: ## Metadata Collection [Metadata Collection](docs/metadata.md) -## Cerberus Integration -[Cerberus Integration](docs/cerberus.md) - ## Indexing to Elasticsearch [Indexing to Elasticsearch](docs/elastic.md) diff --git a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml index 5f24460c1..0b72502c0 100644 --- a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml +++ b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml @@ -121,10 +121,6 @@ spec: default: false type: boolean type: object - cerberus_url: - type: string - cerberus_validate_certs: - type: boolean cleanup: type: boolean test_user: @@ -150,8 +146,6 @@ spec: type: string uuid: type: string - cerberus: - type: string pod_hi_idx: type: string pod_low_idx: @@ -183,10 +177,6 @@ spec: type: string description: The state of metadata collection jsonPath: .status.metadata - - name: Cerberus - type: string - description: If Cerberus is connected or not - jsonPath: .status.cerberus - name: System metrics type: string description: System metrics collect status diff --git a/docs/cerberus.md b/docs/cerberus.md deleted file mode 100644 index 777d345c1..000000000 --- a/docs/cerberus.md +++ /dev/null @@ -1,61 +0,0 @@ -# Cerberus Integration - -How To: -- [Cerberus Integration](#cerberus-integration) -- [What is it](#what-is-it) -- [How it Works](#how-it-works) -- [How to enable Cerberus](#how-to-enable-cerberus) - -# What is it - -What is Cerberus? [Cerberus](https://github.com/openshift-scale/cerberus) is a project that will watch an Openshift/Kubernernetes cluster -for dead nodes, component failures, etc and provide a healthly/unhealthy (True/False) signal. - -# How it Works - -For installation and startup instructions for Cerberus please see [https://github.com/openshift-scale/cerberus](https://github.com/openshift-scale/cerberus) - -Ripsaw has been enabled to check a provided Cerberus url for a True/False signal. True being healthy and False being unhealthy (meaning a component -has failed). If ripsaw encounters a False signal it will set the State of the benchmark to Error and the benchmark will not proceed -further. - -It will NOT stop any running components or kill any pods when a failure signal is found. That means that any pods that are running will -continue to run. Ripsaw will simply not proceed to any next steps of the workload. Everything is left in this state to aid in any -potential debugging that may need to be done. - -Once an Error state is entered it will not go back to its previous state. This means that you will either need to restart the benchmark -entirely or manually change the state of the benchmark. - -# How to enable Cerberus - -Enabling Cerberus is very easy. Simply define the cerberus_url variable in your CR file with the url:port of the Cerberus service you wish -to use. - -For example, if Cerberus is running at 1.2.3.4:8080 - -``` -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: byowl-benchmark - namespace: benchmark-operator -spec: - elasticsearch: - url: "http://foo.bar.com:9200" - cerberus_url: "http://1.2.3.4:8080" - cerberus_validate_certs: true - workload: - name: byowl -... -``` - -NOTE: The cerberus url MUST BE in the format http://[address]:[port] . If the cerberus is not running with proper ssl certificates, set `cerberus_validate_certs` to `false`. - -Once Cerberus is enabled the connection status can be viewed by getting the benchmark status. If Cerberus is not -enabled the connection status will simply be "not connected" - -``` -$ kubectl -n benchmark-operator get benchmarks.ripsaw.cloudbulldozer.io -NAME TYPE STATE METADATA STATE CERBERUS UUID AGE -byowl-benchmark byowl Error Complete Connected 11faec65-4009-58e8-ac36-0233f0fc822d 10m -``` diff --git a/docs/testpmd.md b/docs/testpmd.md index 337daa22d..eb6aae8b7 100644 --- a/docs/testpmd.md +++ b/docs/testpmd.md @@ -182,8 +182,8 @@ This CR will create TestPMD pod and TRex pod using the SRIOV network provided an ```sh # oc get benchmark -NAME TYPE STATE METADATA STATE CERBERUS UUID AGE -testpmd-benchmark testpmd Complete not collected not connected 35daf5ac-2edf-5e34-a6cc-17fcda055937 4m36s +NAME TYPE STATE METADATA STATE UUID AGE +testpmd-benchmark testpmd Complete not collected 35daf5ac-2edf-5e34-a6cc-17fcda055937 4m36s # oc get pods NAME READY STATUS RESTARTS AGE diff --git a/playbooks/benchmark.yml b/playbooks/benchmark.yml index 3256b9a81..bb58f4cd9 100644 --- a/playbooks/benchmark.yml +++ b/playbooks/benchmark.yml @@ -47,7 +47,6 @@ complete: false suuid: "{{ trunc_uuid }}" metadata: "not collected" - cerberus: "not connected" rescue: - include_role: name: benchmark_state diff --git a/roles/benchmark_state/tasks/failure.yml b/roles/benchmark_state/tasks/failure.yml index 619205a39..c8ec38455 100644 --- a/roles/benchmark_state/tasks/failure.yml +++ b/roles/benchmark_state/tasks/failure.yml @@ -12,4 +12,3 @@ message: "Reconcile failed on {{ ansible_failed_task.name }} {{ ansible_failed_task.action }} \n ```{{ ansible_failed_result | to_nice_json }}```" suuid: "{{ trunc_uuid }}" metadata: "not collected" - cerberus: "not connected" \ No newline at end of file diff --git a/roles/cerberus/tasks/main.yml b/roles/cerberus/tasks/main.yml deleted file mode 100644 index 852af1617..000000000 --- a/roles/cerberus/tasks/main.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -- name: Get status from Cerberus - uri: - url: "{{ cerberus_url }}" - validate_certs: "{{ cerberus_validate_certs | default('False') }}" - return_content: yes - register: result - -- name: Update status if unhealthy - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - cerberus: "Connected" - state: "Error" - when: result.content == "False" - -- name: Update status if healthy - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - cerberus: "Connected" - when: result.content == "True" - From 2d3dc8dab1bd6543d33ff2f421f1befecf5853e7 Mon Sep 17 00:00:00 2001 From: Harshith u <44830597+Harshith-umesh@users.noreply.github.com> Date: Thu, 12 Aug 2021 16:27:45 +0530 Subject: [PATCH 118/249] Disable readonly fs in kube-burner pod (#639) Disable readonly fs in kube-burner pod --- roles/kube-burner/templates/kube-burner.yml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/kube-burner/templates/kube-burner.yml.j2 b/roles/kube-burner/templates/kube-burner.yml.j2 index 472360f7e..c3db198f9 100644 --- a/roles/kube-burner/templates/kube-burner.yml.j2 +++ b/roles/kube-burner/templates/kube-burner.yml.j2 @@ -30,6 +30,8 @@ spec: - name: kube-burner image: {{ workload_args.image | default('quay.io/cloud-bulldozer/kube-burner:latest') }} imagePullPolicy: Always + securityContext: + readOnlyRootFilesystem: false {% if workload_args.extra_env_vars is defined and workload_args.extra_env_vars is mapping %} env: {% for name, value in workload_args.extra_env_vars.items() %} From f7a57d9436ee3a7435710ee1fa3da0c2ee8cbca0 Mon Sep 17 00:00:00 2001 From: Robert Krawitz Date: Wed, 18 Aug 2021 12:34:55 -0400 Subject: [PATCH 119/249] make undeploy should depend on make kustomize (#647) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9348e39ff..9a894a1f6 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/c cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - -undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/default | kubectl delete -f - From d03bbb74005c6200fb1696ac9bcc90903aa3d65b Mon Sep 17 00:00:00 2001 From: Keith Whitley Date: Wed, 18 Aug 2021 13:29:00 -0400 Subject: [PATCH 120/249] Ripsaw CLI (#617) * adding ripsaw cli project * deleting pycache dirs * removing resources * working tests with kind * adding k8s tests * working tests with es mock * adding operator command tests * adding tox * adding github workflow cli and marking integration/unit tests * fixing github actions template * fixing github actions template * trying to fix knd * trying to fix kind pt 2 * trying to fix kind pt 3 * trying to fix kind pt 4 * trying to fix kind pt 5 * trying to fix kind pt 6 * making operator command tests an integratino test * making coverae depend on unit * fixing coverage report * fixing coverage report * fixing coverage report again * debugging coverage * ttesting github.workspace coverage * ttesting github.workspace coverage for all steps * fixing path for artifact * fixing path for artifact for real this time * fixing download artiact path * adding tox install * adding tox install * fixing coverage report path * removing debug commands * testing pre commit * testing pre commit * testing pre commit with incluion path * fixing name of unit test jobs * adding precommit config with linted code * moving python lint config files into cli dir * removing cli/commands from omit coverage * fixing operator group name * adding readme * adding docstrings * adding docstring to setup and fixing kustomize template * adding more docstrings * fixing .pylintrc * removing gha for pre-commit * fixing operator install when outside a git repo and making logs more accurate * fixing uninstall command --- .github/workflows/cli.unit.yml | 42 ++++ .gitignore | 147 +++++++++++++ .pre-commit-config.yaml | 75 +++++++ cli/.coveragerc | 4 + cli/.flake8 | 11 + cli/.pylintrc | 20 ++ cli/MANIFEST.in | 1 + cli/README.md | 26 +++ cli/pytest.ini | 4 + cli/ripsaw/__init__.py | 0 cli/ripsaw/cli.py | 33 +++ cli/ripsaw/clients/__init__.py | 0 cli/ripsaw/clients/elastic.py | 45 ++++ cli/ripsaw/clients/k8s.py | 197 ++++++++++++++++++ cli/ripsaw/commands/__init__.py | 0 cli/ripsaw/commands/benchmark.py | 77 +++++++ cli/ripsaw/commands/operator.py | 130 ++++++++++++ cli/ripsaw/models/__init__.py | 0 cli/ripsaw/models/benchmark.py | 77 +++++++ cli/ripsaw/models/workload.py | 63 ++++++ cli/ripsaw/util/__init__.py | 0 cli/ripsaw/util/exceptions.py | 56 +++++ cli/ripsaw/util/logging.py | 41 ++++ cli/setup.cfg | 52 +++++ cli/setup.py | 29 +++ cli/tests/__init__.py | 0 cli/tests/clients/test_elastic.py | 32 +++ cli/tests/clients/test_k8s.py | 117 +++++++++++ cli/tests/commands/test_benchmark_commands.py | 23 ++ cli/tests/commands/test_operator_commands.py | 32 +++ cli/tests/conftest.py | 91 ++++++++ cli/tests/models/test_benchmark.py | 29 +++ cli/tests/models/test_workload.py | 26 +++ cli/tests/resources/benchmark.yaml | 16 ++ cli/tests/resources/job.yaml | 18 ++ cli/tox.ini | 19 ++ cli/version.txt | 1 + config/manager/manager.yaml | 4 +- 38 files changed, 1537 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/cli.unit.yml mode change 100644 => 100755 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 cli/.coveragerc create mode 100644 cli/.flake8 create mode 100644 cli/.pylintrc create mode 100644 cli/MANIFEST.in create mode 100644 cli/README.md create mode 100644 cli/pytest.ini create mode 100755 cli/ripsaw/__init__.py create mode 100755 cli/ripsaw/cli.py create mode 100644 cli/ripsaw/clients/__init__.py create mode 100755 cli/ripsaw/clients/elastic.py create mode 100755 cli/ripsaw/clients/k8s.py create mode 100755 cli/ripsaw/commands/__init__.py create mode 100755 cli/ripsaw/commands/benchmark.py create mode 100755 cli/ripsaw/commands/operator.py create mode 100755 cli/ripsaw/models/__init__.py create mode 100755 cli/ripsaw/models/benchmark.py create mode 100755 cli/ripsaw/models/workload.py create mode 100755 cli/ripsaw/util/__init__.py create mode 100755 cli/ripsaw/util/exceptions.py create mode 100755 cli/ripsaw/util/logging.py create mode 100644 cli/setup.cfg create mode 100755 cli/setup.py create mode 100644 cli/tests/__init__.py create mode 100755 cli/tests/clients/test_elastic.py create mode 100755 cli/tests/clients/test_k8s.py create mode 100755 cli/tests/commands/test_benchmark_commands.py create mode 100755 cli/tests/commands/test_operator_commands.py create mode 100755 cli/tests/conftest.py create mode 100755 cli/tests/models/test_benchmark.py create mode 100755 cli/tests/models/test_workload.py create mode 100644 cli/tests/resources/benchmark.yaml create mode 100755 cli/tests/resources/job.yaml create mode 100644 cli/tox.ini create mode 100644 cli/version.txt diff --git a/.github/workflows/cli.unit.yml b/.github/workflows/cli.unit.yml new file mode 100644 index 000000000..197da5ae1 --- /dev/null +++ b/.github/workflows/cli.unit.yml @@ -0,0 +1,42 @@ + +name: CLI Unit Tests +on: + push: + paths: + - 'cli/**' + - 'workflows/cli.*.yml' + pull_request: + workflow_dispatch: + +defaults: + run: + shell: bash + working-directory: cli + +jobs: + unit_tests: + name: Run Unit Tests -- Python 3.${{ matrix.pyver }} + runs-on: ubuntu-latest + strategy: + matrix: + pyver: [8, 9] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.${{ matrix.pyver }} + - name: Install tox + run: pip install tox + - name: Install Kind Cluster + run: kind create cluster --name=pytest-kind --kubeconfig ./kubeconfig + - name: Run Tox + run: tox -e py3${{ matrix.pyver }}-unit -- --kind-bin=/usr/local/bin/kind --kubeconfig ./kubeconfig --keep-cluster + - name: Generate Coverage Report + if: matrix.pyver == 9 + run: tox -e coverage + - name: Upload Coverage Report + if: matrix.pyver == 9 + uses: codecov/codecov-action@v1 + with: + files: cli/coverage.xml + flags: unit,gha,python-3.${{ matrix.pyver }} \ No newline at end of file diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index 98ff18d96..ef9123ae7 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,150 @@ bin *.swp *.swo *~ + +.kube +.pytest-kind +.kind + + + +### Pulled from https://github.com/github/gitignore/blob/master/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..8762bb269 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,75 @@ +--- +files: ^cli/ +repos: + - repo: git://github.com/Lucas-C/pre-commit-hooks + rev: v1.1.10 + hooks: + - id: remove-tabs + + - repo: git://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: trailing-whitespace + - id: check-merge-conflict + - id: end-of-file-fixer + - id: check-case-conflict + - id: detect-private-key + - id: check-ast + + - repo: https://github.com/psf/black + rev: 21.7b0 + hooks: + - id: black + args: ['--line-length', '110', 'cli'] + + - repo: https://github.com/pycqa/isort + rev: 5.9.2 + hooks: + - id: isort + args: + - --profile=black + - --color + - --line-length=110 + + - repo: https://gitlab.com/PyCQA/flake8 + rev: '3.9.2' + hooks: + - id: flake8 + args: ['--config', 'cli/.flake8'] + additional_dependencies: ['pep8-naming', 'flake8-docstrings'] + + - repo: https://github.com/PyCQA/pylint + rev: v2.9.5 + hooks: + - id: pylint + args: ['--rcfile', 'cli/.pylintrc'] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.910 + hooks: + - id: mypy + args: + - --ignore-missing-imports + - --install-types + - --non-interactive + + - repo: https://github.com/asottile/pyupgrade + rev: v2.21.2 + hooks: + - id: pyupgrade + + - repo: https://github.com/PyCQA/bandit + rev: 1.7.0 + hooks: + - id: bandit + args: ["--exclude", "cli/tests/", "--skip", "B404,B603"] + +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit.com hooks + for more information, see https://pre-commit.ci + autofix_prs: true + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: weekly + skip: [] + submodules: false \ No newline at end of file diff --git a/cli/.coveragerc b/cli/.coveragerc new file mode 100644 index 000000000..7ae4f689f --- /dev/null +++ b/cli/.coveragerc @@ -0,0 +1,4 @@ +[run] +omit = + **/__init__.py + ripsaw/cli.py diff --git a/cli/.flake8 b/cli/.flake8 new file mode 100644 index 000000000..31cf5219b --- /dev/null +++ b/cli/.flake8 @@ -0,0 +1,11 @@ +[flake8] +ignore= + E123,E125,E731,E722,W503,W605, # Black takes care of these + D202, # no blank lines after docstring + D203, # blank line before class docstring + D212, # multi-line summary starts at first line + D401, # imperitive mood + D413, # blank line after last section +max-line-length=110 +docstring-convention=numpy +select=E,W,F diff --git a/cli/.pylintrc b/cli/.pylintrc new file mode 100644 index 000000000..c8abffeb5 --- /dev/null +++ b/cli/.pylintrc @@ -0,0 +1,20 @@ +[MASTER] +jobs=0 +ignore-paths= + cli/tests +[MESSAGES CONTROL] +disable= + duplicate-code, + import-error, + too-few-public-methods, + too-many-arguments, + too-many-instance-attributes, + fixme, + logging-fstring-interpolation, + no-self-use, + logging-format-interpolation + + + +[FORMAT] +max-line-length=110 diff --git a/cli/MANIFEST.in b/cli/MANIFEST.in new file mode 100644 index 000000000..48715ba55 --- /dev/null +++ b/cli/MANIFEST.in @@ -0,0 +1 @@ +include version.txt diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 000000000..64bcdde9c --- /dev/null +++ b/cli/README.md @@ -0,0 +1,26 @@ +# Ripsaw CLI + +## Overview + +Ripsaw CLI is a portable, lightweight CLI that can be used to install the benchmark-operator into a specified cluster and run user-defined benchmarks. + +## Installation + +> Note: Ripsaw CLI is only tested against Python 3.8+ + +You must first start by installing the python package like so: + +```bash +cd cli +pip install . +``` + +If you want to make changes to the underlying code, you can set it up as editable by running `pip install -e .` instead. + + +After that, you can run `ripsaw --help` to see the command options. + + +## Commands + +There are two top-level command groups: `operator` and `benchmark`. You can run `ripsaw operator --help` to see options for installing/deleting the operator, and `ripsaw benchmark --help` to see options for running benchmarks. diff --git a/cli/pytest.ini b/cli/pytest.ini new file mode 100644 index 000000000..6757ecf58 --- /dev/null +++ b/cli/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +markers = + integration: mark a test as integration tests + unit: mark a test as a unit test diff --git a/cli/ripsaw/__init__.py b/cli/ripsaw/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/cli/ripsaw/cli.py b/cli/ripsaw/cli.py new file mode 100755 index 000000000..1634c47e4 --- /dev/null +++ b/cli/ripsaw/cli.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Main Entrypoint Module for the CLI""" + + +import click +from ripsaw.commands import benchmark, operator +from ripsaw.util import logging + +logger = logging.get_logger(__name__) + + +@click.group() +def cli(): + """Top level Click CLI group that is used to add all the groups defined in the commands module.""" + + +cli.add_command(operator.operator_group) +cli.add_command(benchmark.benchmark_group) + +if __name__ == "__main__": + cli() diff --git a/cli/ripsaw/clients/__init__.py b/cli/ripsaw/clients/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cli/ripsaw/clients/elastic.py b/cli/ripsaw/clients/elastic.py new file mode 100755 index 000000000..6e07c5935 --- /dev/null +++ b/cli/ripsaw/clients/elastic.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Create portable functions around interacting with ES hosting benchmark data""" + + +import ssl + +import elasticsearch +import urllib3 +from ripsaw.util import logging + +logger = logging.get_logger(__name__) + + +def check_index(server, uuid, index, es_ssl=False): + """Checks index on ES Server for benchmark with a specific uuid""" + + if es_ssl: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + ssl_ctx = ssl.create_default_context() + ssl_ctx.check_hostname = False + ssl_ctx.verify_mode = ssl.CERT_NONE + es_client = elasticsearch.Elasticsearch( + [server], send_get_body_as="POST", ssl_context=ssl_ctx, use_ssl=True + ) + else: + es_client = elasticsearch.Elasticsearch([server], send_get_body_as="POST") + es_client.indices.refresh(index=index) + results = es_client.search(index=index, body={"query": {"term": {"uuid.keyword": uuid}}}, size=1) + if results["hits"]["total"]["value"] > 0: + return True + + print("No result found in ES") + return False diff --git a/cli/ripsaw/clients/k8s.py b/cli/ripsaw/clients/k8s.py new file mode 100755 index 000000000..b42fa43e0 --- /dev/null +++ b/cli/ripsaw/clients/k8s.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create portable definition of a Kubernetes/Openshift Cluster""" + + +import time + +from kubernetes import client, config +from ripsaw.util import logging +from ripsaw.util.exceptions import BenchmarkFailedError, BenchmarkTimeoutError, PodsNotFoundError + +logger = logging.get_logger(__name__) + +# how many seconds to sleep before looping back in wait commands +DEFAULT_WAIT_TIME = 1 + + +class Cluster: + """ + Class definition of a Kubernetes/Openshift Cluster + + Attributes: + api_client (ApiClient) + core_client (CoreV1Api) + batch_client (BatchV1Api) + crd_client (CustomObjectsApi) + """ + + def __init__(self, kubeconfig_path=None): + """Initialize object and create clients from specified kubeconfig""" + config.load_kube_config(config_file=kubeconfig_path) + self.api_client = client.ApiClient() + self.core_client = client.CoreV1Api() + self.batch_client = client.BatchV1Api() + self.crd_client = client.CustomObjectsApi() + + def get_pods(self, label_selector, namespace): + """Get pods matching specified label selector in specified namespace""" + return self.core_client.list_namespaced_pod(namespace, label_selector=label_selector, watch=False) + + def get_pod_logs(self, label_selector, namespace, container): + """Get container logs from pods with labels in namespace""" + pods = self.get_pods(label_selector, namespace).items + return [ + self.core_client.read_namespaced_pod_log( + name=pod.metadata.name, namespace=namespace, container=container + ) + for pod in pods + ] + + def get_jobs(self, label_selector, namespace): + """Get jobs with specified labels in namespace""" + return self.batch_client.list_namespaced_job(namespace, label_selector=label_selector, watch=False) + + def get_nodes(self, label_selector=None): + """Get all nodes with label selector, default is to get all nodes""" + return self.core_client.list_node(label_selector=label_selector) + + def get_node_names(self, label_selector=None): + """Get names of all nodes with label selector""" + return [node.metadata.name for node in self.get_nodes(label_selector).items] + + def get_namespaces(self, label_selector=None): + """Get namespaces with labels""" + return self.core_client.list_namespace(label_selector=label_selector) + + def get_benchmark(self, name, namespace="benchmark-operator"): + """Get benchmark resource""" + return self.crd_client.get_namespaced_custom_object( + group="ripsaw.cloudbulldozer.io", + version="v1alpha1", + namespace=namespace, + plural="benchmarks", + name=name, + ) + + def get_benchmark_metadata(self, name, namespace="benchmark-operator"): + """Get benchmark metadata from the cluster resource""" + benchmark = self.get_benchmark(name, namespace) + return { + "name": benchmark["metadata"]["name"], + "namespace": benchmark["metadata"]["namespace"], + "uuid": benchmark.get("status", {}).get("uuid", "Not Assigned Yet"), + "suuid": benchmark.get("status", {}).get("suuid", "Not Assigned Yet"), + "status": benchmark.get("status", {}).get("state", ""), + } + + # Waiters + def wait_for_pods(self, label_selector, namespace, timeout=300): + """wait for pods with label in namespace for specified timeout""" + waiting_for_pods = True + timeout_interval = 0 + while waiting_for_pods: + if timeout_interval >= timeout: + raise TimeoutError() + pods = self.get_pods(label_selector, namespace).items + if len(pods) == 0 and timeout_interval < 60: + continue + + if len(pods) == 0 and timeout_interval > 60: + raise PodsNotFoundError(f"Found no pods with label selector {label_selector}") + + _ = [ + logger.info(f"{pod.metadata.namespace}\t{pod.metadata.name}\t{pod.status.phase}") + for pod in pods + ] + waiting_for_pods = any(pod.status.phase != "Running" for pod in pods) + + time.sleep(DEFAULT_WAIT_TIME) + timeout_interval += DEFAULT_WAIT_TIME + + def wait_for_benchmark( + self, name, namespace="benchmark-operator", desired_state="Completed", timeout=300 + ): + """Wait for benchmark to hit desired state for specified timeout""" + waiting_for_benchmark = True + logger.info(f"Waiting for state: {desired_state}") + timeout_interval = 0 + logger.info("BENCHMARK\tUUID\t\t\t\t\tSTATE") + while waiting_for_benchmark: + if timeout_interval >= timeout: + raise BenchmarkTimeoutError(name) + benchmark = self.get_benchmark(name, namespace) + bench_status = benchmark.get("status", {}) + uuid = bench_status.get("uuid", "Not Assigned Yet") + current_state = bench_status.get("state", "") + logger.info(f"{benchmark['metadata']['name']}\t{uuid}\t{current_state}") + if current_state == "Failed": + raise BenchmarkFailedError(benchmark["metadata"]["name"], benchmark["status"]["uuid"]) + + waiting_for_benchmark = current_state != desired_state + time.sleep(DEFAULT_WAIT_TIME) + + timeout_interval += DEFAULT_WAIT_TIME + logger.info( + f"{benchmark['metadata']['name']} with uuid {uuid} has reached the desired state {desired_state}" + ) + + # Create Functions + + def create_benchmark(self, benchmark): + """Create benchmark in the cluster""" + self.crd_client.create_namespaced_custom_object( + group="ripsaw.cloudbulldozer.io", + version="v1alpha1", + namespace=benchmark["metadata"]["namespace"], + plural="benchmarks", + body=benchmark, + ) + + # Delete Functions + + def delete_benchmark(self, name, namespace="benchmark-operator"): + """Delete the benchmark from the cluster""" + logger.info(f"Deleting benchmark {name} in namespace {namespace}") + self.crd_client.delete_namespaced_custom_object( + group="ripsaw.cloudbulldozer.io", + version="v1alpha1", + namespace=namespace, + plural="benchmarks", + name=name, + ) + logger.info(f"Deleted benchmark {name} in namespace {namespace}") + + def delete_all_benchmarks(self, namespace="benchmark-operator"): + """Delete all the benchmarks from the cluster""" + all_benchmarks = self.crd_client.list_namespaced_custom_object( + group="ripsaw.cloudbulldozer.io", version="v1alpha1", namespace=namespace, plural="benchmarks" + ) + + _ = [ + self.delete_benchmark(benchmark["metadata"]["name"], namespace) + for benchmark in all_benchmarks.get("items", []) + ] + + def delete_namespace(self, namespace): + """Delete namespace from the cluster""" + return self.core_client.delete_namespace(namespace) + + def delete_namespaces_with_label(self, label_selector): + """Delete all namespaces with label from the cluster.""" + return [ + self.core_client.delete_namespace(namespace.metadata.name) + for namespace in self.get_namespaces(label_selector=label_selector).items + ] diff --git a/cli/ripsaw/commands/__init__.py b/cli/ripsaw/commands/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/cli/ripsaw/commands/benchmark.py b/cli/ripsaw/commands/benchmark.py new file mode 100755 index 000000000..c0535540f --- /dev/null +++ b/cli/ripsaw/commands/benchmark.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CLI Commands for benchmark operations""" + + +import json +from urllib.parse import urlparse + +import click +from ripsaw.clients import elastic +from ripsaw.clients.k8s import Cluster +from ripsaw.models.benchmark import Benchmark + + +@click.group("benchmark") +def benchmark_group(): + """Function to define Click CLI group, used as a decorator to declare functions as group commands + This function is used as a decorator for the benchmark commands (eg. ripsaw benchmark ) + """ + + +@benchmark_group.command("run") +@click.option("-f", "--file", "file", required=True, type=click.File("rb")) +@click.option("-t", "--timeout", "timeout", default=300, type=int) +@click.option("--check-es", is_flag=True) +def run_benchmark(file, timeout, check_es): + """Create benchmark class from file and submit the resource to configured cluster.""" + cluster = Cluster() + benchmark = Benchmark(file, cluster) + click.echo(f"Starting Benchmark {benchmark.name}, timeout set to {timeout} seconds") + benchmark.run(timeout=timeout) + click.secho(f"Benchmark {benchmark.name} Complete", fg="green") + run_metadata = benchmark.get_metadata() + click.echo(f"Run Metadata: \n{json.dumps(run_metadata, indent=4)}\n") + if check_es: + results_found = _check_es(benchmark) + if results_found: + click.secho("Results Found, Benchmark Complete", fg="green") + benchmark.cleanup() + else: + click.secho("No Results Found", fg="red") + + +@benchmark_group.command("delete") +@click.argument("name") +@click.option("-n", "--namespace", "namespace", default="benchmark-operator") +def delete_benchmark(name, namespace): + """delete benchmark resource from cluster""" + cluster = Cluster() + cluster.delete_benchmark(name, namespace) + click.echo(f"Benchmark {name} deleted") + + +def _check_es(benchmark): + """check es for benchmark results""" + run_metadata = benchmark.get_metadata() + es_url = benchmark.yaml["spec"]["elasticsearch"]["url"] + index_name = benchmark.yaml["spec"]["elasticsearch"]["index_name"] + click.echo(f"Checking ES: \n Host: {_get_es_hostname(es_url)} \n Index: {index_name} \n") + return elastic.check_index(es_url, run_metadata["uuid"], f"{index_name}-results") + + +def _get_es_hostname(es_url): + """parse es url and get the hostname""" + parsed_url = urlparse(es_url) + return parsed_url.hostname diff --git a/cli/ripsaw/commands/operator.py b/cli/ripsaw/commands/operator.py new file mode 100755 index 000000000..92cad8a60 --- /dev/null +++ b/cli/ripsaw/commands/operator.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CLI Commands for benchmark operations""" + +import os +import subprocess +import sys +import tempfile + +import click +import git +from ripsaw.clients.k8s import Cluster +from ripsaw.util import logging + +logger = logging.get_logger(__name__) +DEFAULT_REPO = "https://github.com/cloud-bulldozer/benchmark-operator" +DEFAULT_BRANCH = "master" + + +@click.group("operator") +def operator_group(): + """Function to define Click CLI group, used as a decorator to declare functions as group commands + This function is used as a decorator for the operator commands (eg. ripsaw operator ) + """ + + +@operator_group.command("install") +@click.option("--repo", help="Repo URL for benchmark-operator", show_default=True, default=DEFAULT_REPO) +@click.option("--branch", help="Branch to checkout", show_default=True, default=DEFAULT_BRANCH) +@click.option("--force-remote", help="Forcibly use remote repo for install", is_flag=True) +@click.option("--kubeconfig", help=" path to kubeconfig of cluster", default=None) +def install( + repo=DEFAULT_REPO, branch=DEFAULT_BRANCH, force_remote=False, command="make deploy", kubeconfig=None +): + """installs the operator from repo into specified cluster""" + _perform_operator_action(repo, branch, force_remote, command, kubeconfig) + wait_for_operator(kubeconfig=kubeconfig) + click.secho("Operator Running", fg="green") + + +@operator_group.command("delete") +@click.option("--repo", help="Repo URL for benchmark-operator", show_default=True, default=DEFAULT_REPO) +@click.option("--branch", help="Branch to checkout", show_default=True, default=DEFAULT_BRANCH) +@click.option("--force-remote", help="Forcibly use remote repo for install", is_flag=True) +@click.option("--kubeconfig", help="kubeconfig of cluster", default=None) +def delete( + repo=DEFAULT_REPO, + branch=DEFAULT_BRANCH, + force_remote=False, + command="make kustomize undeploy", + kubeconfig=None, +): + """delete the operator from configured cluster""" + click.echo("Deleting Operator") + _perform_operator_action(repo, branch, force_remote, command, kubeconfig) + click.secho("Operator Deleted", fg="green") + + +def wait_for_operator(namespace="benchmark-operator", kubeconfig=None): + """wait for operator pods to be ready""" + cluster = Cluster(kubeconfig_path=kubeconfig) + label_selector = "control-plane=controller-manager" + cluster.wait_for_pods(label_selector, namespace, timeout=120) + + +def _perform_operator_action(repo, branch, force_remote, command, kubeconfig=None): + """run command from operator repo""" + shell_env = os.environ.copy() + if kubeconfig is not None: + shell_env["KUBECONFIG"] = kubeconfig + local_git_repo = _find_git_repo(os.getcwd()) + result = None + if not force_remote and local_git_repo is not None and "benchmark-operator" in local_git_repo: + shell_env["VERSION"] = "latest" + logger.info("Detected CLI is running from local repo, using that instead of remote") + click.echo(f"Installing Operator from local repo at {local_git_repo}") + result = _run_command(command, shell_env, local_git_repo) + else: + with tempfile.TemporaryDirectory(dir=".") as temp_dir: + click.echo(f"Installing Operator from repo {repo} and branch {branch}") + _clone_repo(repo, branch, temp_dir) + result = _run_command(command, shell_env, temp_dir) + + if result.stderr: + logger.warning(result.stderr) + if result.stdout: + logger.debug("Command Result: {}".format(result.stdout)) + + +def _run_command(command, shell_env, cwd): + try: + return subprocess.run( + command.split(" "), + shell=False, + env=shell_env, + cwd=cwd, + capture_output=True, + check=True, + encoding="utf-8", + ) + except subprocess.CalledProcessError as err: + logger.error(f"Something went wrong when running '{command}'") + logger.error(f"Return Code {err.returncode}") + logger.error(f"stdout -- {err.output}") + logger.error(f"stderr -- {err.stderr}") + sys.exit(1) + + +def _clone_repo(repo, branch, directory): + """clone git repo into directory""" + git.Repo.clone_from(url=repo, single_branch=True, depth=1, to_path=directory, branch=branch) + + +def _find_git_repo(path): + """get working dir of first git root on tree, return None otherwise""" + try: + return git.Repo(path, search_parent_directories=True).working_dir + except git.exc.InvalidGitRepositoryError: + return None diff --git a/cli/ripsaw/models/__init__.py b/cli/ripsaw/models/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/cli/ripsaw/models/benchmark.py b/cli/ripsaw/models/benchmark.py new file mode 100755 index 000000000..e230422d7 --- /dev/null +++ b/cli/ripsaw/models/benchmark.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines a Class representing a Benchmark""" + +import yaml +from benedict import benedict +from ripsaw.clients.k8s import Cluster +from ripsaw.util import logging +from ripsaw.util.exceptions import BenchmarkNotStartedError + +logger = logging.get_logger(__name__) + + +class Benchmark: + """ + This class represents a Benchmark to run against a specified cluster + + Attributes: + file (file): Benchmark CR + cluster (ripsaw.clients.k8s.Cluster): Kubernetes/Openshift Cluster + name (str): Name of the benchmark, defined in the file + namespace (str): Namespace for the benchmark, defined in the file + metadata (dict): Metadata from the benchmark + + """ + + def __init__(self, file, cluster=None): + """Initialize Benchmark Class""" + self.yaml = benedict(yaml.full_load(file), keypath_separator=",") + if cluster is None: + self.cluster = Cluster() + else: + self.cluster = cluster + self.name = self.yaml["metadata"]["name"] + self.namespace = self.yaml["metadata"]["namespace"] + self.metadata = {} + + def update_spec(self, key_path, new_value): + """Update YAML Key Path with new value""" + self.yaml[key_path] = new_value + + def run(self, timeout=600): + """Run benchmark with specified timeout in seconds (default: 600)""" + self.metadata = self.cluster.create_benchmark(self.yaml) + logger.info(f"Benchmark {self.name} created") + self.wait(timeout=timeout) + return self.metadata + + def wait(self, desired_state="Complete", timeout=600): + """Wait for benchmark to enter desired state with specified timeout in seconds (default: 600)""" + if self.metadata == {}: + raise BenchmarkNotStartedError(self.name) + self.cluster.wait_for_benchmark( + self.name, self.namespace, desired_state=desired_state, timeout=timeout + ) + self.metadata = self.cluster.get_benchmark_metadata(self.name, self.namespace) + return self.metadata + + def cleanup(self): + """Delete this benchmark from the cluster""" + self.cluster.delete_benchmark(self.name, self.namespace) + + def get_metadata(self): + """Fetch Benchmark Metadata from the cluster""" + return self.cluster.get_benchmark_metadata(self.name, self.namespace) diff --git a/cli/ripsaw/models/workload.py b/cli/ripsaw/models/workload.py new file mode 100755 index 000000000..694519411 --- /dev/null +++ b/cli/ripsaw/models/workload.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines a Class representing a Workload""" + + +from os import listdir +from os.path import isfile, join + +from ripsaw.clients.k8s import Cluster +from ripsaw.models.benchmark import Benchmark +from ripsaw.util import logging + +logger = logging.get_logger(__name__) + + +class Workload: + """Defines a Class representing a Workload""" + + def __init__(self, name, cluster=None, benchmark_dir="."): + self.name = name + if cluster is None: + self.cluster = Cluster() + else: + self.cluster = cluster + self.workload_dir = f"{benchmark_dir}/{name}" + self.benchmarks = [ + Benchmark(join(self.workload_dir, f), self.cluster) + for f in listdir(self.workload_dir) + if isfile(join(self.workload_dir, f)) and f.endswith(".yaml") + ] + + def run_all(self): + """Run all benchmarks in workload""" + runs = [] + for run in self.benchmarks: + run.start() + run.wait() + runs.append(run.metadata) + return runs + + def run(self, run_name): + """Run a benchmark""" + run = next((run for run in self.benchmarks if run.name == run_name), None) + run.start() + run.wait() + return run.metadata + + def inject_overrides(self, overrides): + """Override all benchmark specs with overrides""" + for run in self.benchmarks: + _ = [run.update_spec(key, value) for key, value in overrides.items()] diff --git a/cli/ripsaw/util/__init__.py b/cli/ripsaw/util/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/cli/ripsaw/util/exceptions.py b/cli/ripsaw/util/exceptions.py new file mode 100755 index 000000000..8dac49999 --- /dev/null +++ b/cli/ripsaw/util/exceptions.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines Custom Exceptions""" + + +class BenchmarkFailedError(Exception): + """Exception raised when Benchmark hits a Failed State""" + + def __init__(self, name, uuid, msg=None): + if msg is None: + msg = f"The benchmark {name} with uuid {uuid} failed" + super().__init__(msg) + self.name = name + self.uuid = uuid + + +class BenchmarkNotStartedError(Exception): + """Exception raised when Benchmark doesn't hit the Running state before timeout""" + + def __init__(self, name, msg=None): + if msg is None: + msg = f"The benchmark {name} has not started yet" + + super().__init__(msg) + + +class BenchmarkTimeoutError(Exception): + """Exception raised when Benchmark doesn't hit the Completed state before timeout""" + + def __init__(self, name, msg=None): + if msg is None: + msg = f"The benchmark {name} timed out" + + super().__init__(msg) + self.name = name + + +class PodsNotFoundError(Exception): + """Exception raised when query for pods returns none""" + + def __init__(self, msg=None): + if msg is None: + msg = "No Pods found" + super().__init__(msg) diff --git a/cli/ripsaw/util/logging.py b/cli/ripsaw/util/logging.py new file mode 100755 index 000000000..71a92e25d --- /dev/null +++ b/cli/ripsaw/util/logging.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Defines Custom Logging Resources""" + +import logging + + +class DuplicateFilter(logging.Filter): + """Filter class to apply to loggers to prevent duplicate log lines""" + + def filter(self, record): + """Filters our duplicate log lines""" + # add other fields if you need more granular comparison, depends on your app + current_log = (record.module, record.levelno, record.msg) + if current_log != getattr(self, "last_log", None): + self.last_log = current_log # pylint: disable=attribute-defined-outside-init + return True + return False + + +logging.basicConfig(level=logging.INFO, format="ripsaw-cli:%(name)s:%(levelname)s :: %(message)s") +logger = logging.getLogger() # get the root logger +logger.addFilter(DuplicateFilter()) + + +def get_logger(name): + """Get new logger with name""" + new_logger = logging.getLogger(name) + new_logger.addFilter(DuplicateFilter()) + return new_logger diff --git a/cli/setup.cfg b/cli/setup.cfg new file mode 100644 index 000000000..5e7fb573f --- /dev/null +++ b/cli/setup.cfg @@ -0,0 +1,52 @@ +# This file is used to configure your project. +# Read more about the various options under: +# http://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files + +[metadata] +name = ripsaw-cli +description = To run benchmarks against openshift/kubernetes from the CLI +author = red-hat-performance +author_email = perf-dept@redhat.com +license = Apache License 2.0 +long_description = file: README.md +long_description_content-type = text/markdown; charset=UTF-8 +classifiers = + Development Status :: 4 - Beta + Programming Language :: Python +version = file: version.txt +home-page = https://github.com/cloud-bulldozer/benchmark-operator/cli + +[options] +zip_safe = False +packages = find: +include_package_data = True +# Add here dependencies of your project (semicolon/line-separated), e.g. +install_requires = + python-benedict==0.24.0 + click==8.0.1 + kubernetes==17.17.0 + urllib3==1.25.7 + PyYAML==5.4.1 + elasticsearch==7.13.4 + GitPython==3.1.18 +python_requires = >=3.8 + +[options.extras_require] +tests = + pytest + pytest-cov + tox + pytest-kind + ElasticMock + pytest-env +[options.entry_points] +# Add here console scripts like: +console_scripts = + ripsaw = ripsaw.cli:cli + +[aliases] +dists = bdist_wheel + +[bdist_wheel] +# Use this option if your package is pure-python +universal = 1 diff --git a/cli/setup.py b/cli/setup.py new file mode 100755 index 000000000..fc2d23911 --- /dev/null +++ b/cli/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Setup ripsaw package""" + +import sys + +from pkg_resources import VersionConflict, require +from setuptools import setup + +try: + require("setuptools>=38.3") +except VersionConflict: + print("Error: version of setuptools is too old (<38.3)!") + sys.exit(1) + + +if __name__ == "__main__": + setup() diff --git a/cli/tests/__init__.py b/cli/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cli/tests/clients/test_elastic.py b/cli/tests/clients/test_elastic.py new file mode 100755 index 000000000..9e8c60de8 --- /dev/null +++ b/cli/tests/clients/test_elastic.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import elasticsearch +import pytest +from elasticmock import elasticmock +from ripsaw.clients import elastic + + +@pytest.mark.unit +class TestElastic: + @elasticmock + def test_check_index(self): + server_url = "http://localhost:9200" + index = "test-index" + document = {"uuid": "foo", "data": "bar"} + + elastic_client = elasticsearch.Elasticsearch(hosts=[{"host": "localhost", "port": 9200}]) + elastic_client.index(index, document) + assert elastic.check_index(server_url, document["uuid"], index) + assert not elastic.check_index(server_url, "random-uuid", index) diff --git a/cli/tests/clients/test_k8s.py b/cli/tests/clients/test_k8s.py new file mode 100755 index 000000000..ddc5211aa --- /dev/null +++ b/cli/tests/clients/test_k8s.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import kubernetes +import pytest +from ripsaw.util.exceptions import BenchmarkTimeoutError + + +@pytest.mark.unit +class TestCluster: + def test_get_pods(self, cluster): + label_selector = "component=etcd" + pods = cluster.get_pods(label_selector=label_selector, namespace="kube-system") + assert len(pods.items) == 1 + assert "etcd-pytest-kind-control-plane" in pods.items[0].metadata.name + + def test_get_pod_logs(self, cluster): + label_selector = "component=etcd" + logs = cluster.get_pod_logs(label_selector=label_selector, namespace="kube-system", container="etcd") + assert len(logs) > 0 + + def test_get_jobs(self, cluster, test_job): + label_selector = "app=test-job" + jobs = cluster.get_jobs(label_selector=label_selector, namespace="default") + assert len(jobs.items) == 1 + assert jobs.items[0].metadata.name == "busybox" + + def test_get_nodes(self, cluster): + nodes = cluster.get_nodes() + assert len(nodes.items) == 1 + + def test_get_node_names(self, cluster): + names = cluster.get_node_names() + assert len(names) == 1 + assert type(names[0]) is str + assert names[0] == "pytest-kind-control-plane" + + def test_get_namespaces(self, cluster): + namespaces = cluster.get_namespaces() + names = [namespace.metadata.name for namespace in namespaces.items] + assert "kube-system" in names + + def test_get_benchmark(self, cluster, test_benchmark): + benchmark = cluster.get_benchmark(name="byowl") + assert benchmark["metadata"]["name"] == "byowl" + + def test_get_benchmark_metadata(self, cluster, test_benchmark): + benchmark_metadata = cluster.get_benchmark_metadata(name="byowl") + assert benchmark_metadata == { + "name": "byowl", + "namespace": "benchmark-operator", + "uuid": "Not Assigned Yet", + "suuid": "Not Assigned Yet", + "status": "", + } + + def test_wait_for_pods(self, cluster, test_job): + label_selector = "job-name=busybox" + namespace = "default" + cluster.wait_for_pods(label_selector, namespace) + pods = cluster.get_pods(label_selector, namespace) + assert len(pods.items) == 1 + assert pods.items[0].status.phase == "Running" + + def test_create_benchmark_async(self, cluster, test_benchmark_model): + name = test_benchmark_model.name + namespace = test_benchmark_model.namespace + cluster.create_benchmark(test_benchmark_model.yaml) + assert name == cluster.get_benchmark(name, namespace)["metadata"]["name"] + + def test_wait_for_benchmark_timeout(self, cluster, test_benchmark): + name = "byowl" + namespace = "benchmark-operator" + with pytest.raises(BenchmarkTimeoutError): + cluster.wait_for_benchmark(name, namespace, timeout=5) + + def test_delete_benchmark(self, cluster, test_benchmark): + name = "byowl" + namespace = "benchmark-operator" + cluster.delete_benchmark(name, namespace) + with pytest.raises(kubernetes.client.exceptions.ApiException): + cluster.get_benchmark(name, namespace) + + def test_delete_all_benchmarks(self, cluster, test_benchmark): + name = "byowl" + namespace = "benchmark-operator" + cluster.delete_all_benchmarks(namespace) + with pytest.raises(kubernetes.client.exceptions.ApiException): + cluster.get_benchmark(name, namespace) + + def test_delete_namespace(self, cluster, test_namespace): + expected_name = test_namespace[0] + label_selector = test_namespace[1] + namespace = cluster.get_namespaces(label_selector=label_selector).items[0] + assert namespace.metadata.name == expected_name + response = cluster.delete_namespace(namespace.metadata.name) + assert response.status == "{'phase': 'Terminating'}" + + def test_delete_namespaces_with_label(self, cluster, test_multiple_namespaces): + expected_namespaces = test_multiple_namespaces[0] + label_selector = test_multiple_namespaces[1] + namespaces = cluster.get_namespaces(label_selector=label_selector).items + responses = cluster.delete_namespaces_with_label(label_selector=label_selector) + assert expected_namespaces == [namespace.metadata.name for namespace in namespaces] + assert len(responses) == len(namespaces) + assert all([response.status == "{'phase': 'Terminating'}" for response in responses]) diff --git a/cli/tests/commands/test_benchmark_commands.py b/cli/tests/commands/test_benchmark_commands.py new file mode 100755 index 000000000..c2b2ecb1c --- /dev/null +++ b/cli/tests/commands/test_benchmark_commands.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + + +@pytest.mark.integration +class TestBenchmarkCommands: + def test_run_benchmark(self): + pass + + def test_delete_benchmark(self): + pass diff --git a/cli/tests/commands/test_operator_commands.py b/cli/tests/commands/test_operator_commands.py new file mode 100755 index 000000000..478a5f44f --- /dev/null +++ b/cli/tests/commands/test_operator_commands.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from ripsaw.commands import operator + + +@pytest.mark.integration +class TestOperatorCommands: + def test_operator_commands(self, kind_kubeconfig, cluster, benchmark_namespace): + operator.install(kubeconfig=kind_kubeconfig) + pods = cluster.get_pods( + label_selector="control-plane=controller-manager", namespace="benchmark-operator" + ) + assert len(pods.items) == 1 + assert pods.items[0].status.phase == "Running" + operator.delete(kubeconfig=kind_kubeconfig) + pods = cluster.get_pods( + label_selector="control-plane=controller-manager", namespace="benchmark-operator" + ) + assert len(pods.items) == 0 diff --git a/cli/tests/conftest.py b/cli/tests/conftest.py new file mode 100755 index 000000000..04c37ee32 --- /dev/null +++ b/cli/tests/conftest.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import pytest +import pytest_kind # noqa: F401 +from ripsaw.clients.k8s import Cluster +from ripsaw.models.benchmark import Benchmark + +# Kind Cluster Fixtures + + +@pytest.fixture(scope="session") +def cluster(kind_cluster): + time.sleep(15) + return Cluster(kubeconfig_path=str(kind_cluster.kubeconfig_path.resolve())) + + +@pytest.fixture(scope="session") +def kind_kubeconfig(kind_cluster): + return str(kind_cluster.kubeconfig_path.resolve()) + + +@pytest.fixture(scope="function") +def test_job(kind_cluster): + yield kind_cluster.kubectl("apply", "-f", "tests/resources/job.yaml") + kind_cluster.kubectl("delete", "-f", "tests/resources/job.yaml", "--ignore-not-found") + + +@pytest.fixture(scope="session") +def benchmark_crd(kind_cluster): + yield kind_cluster.kubectl("apply", "-f", "../config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml") + kind_cluster.kubectl( + "delete", "-f", "../config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml", "--ignore-not-found" + ) + + +@pytest.fixture(scope="session") +def benchmark_namespace(kind_cluster): + yield kind_cluster.kubectl("create", "namespace", "benchmark-operator") + kind_cluster.kubectl("delete", "namespace", "benchmark-operator", "--ignore-not-found") + + +@pytest.fixture(scope="function") +def test_benchmark(kind_cluster, benchmark_crd, benchmark_namespace): + yield kind_cluster.kubectl("apply", "-f", "tests/resources/benchmark.yaml") + kind_cluster.kubectl("delete", "-f", "tests/resources/benchmark.yaml", "--ignore-not-found") + + +@pytest.fixture(scope="function") +def test_benchmark_path(kind_cluster, benchmark_crd, benchmark_namespace): + return "tests/resources/benchmark.yaml" + + +@pytest.fixture(scope="function") +def test_namespace(kind_cluster): + name = "test-namespace" + label = "test=true" + kind_cluster.kubectl("create", "namespace", name) + kind_cluster.kubectl("label", "namespaces", name, label) + yield name, label + kind_cluster.kubectl("delete", "namespace", name, "--ignore-not-found") + + +@pytest.fixture(scope="function") +def test_multiple_namespaces(kind_cluster): + name = "test-namespace" + label = "multi-namespace=true" + namespaces = f"{name}-1 {name}-2 {name}-3".split(" ") + [kind_cluster.kubectl("create", "namespace", namespace) for namespace in namespaces] + [kind_cluster.kubectl("label", "namespaces", namespace, label) for namespace in namespaces] + yield namespaces, label + [kind_cluster.kubectl("delete", "namespace", namespace, "--ignore-not-found") for namespace in namespaces] + + +@pytest.fixture(scope="function") +def test_benchmark_model(kind_cluster, cluster, test_benchmark_path): + yield Benchmark(test_benchmark_path, cluster) + kind_cluster.kubectl("delete", "benchmark", "byowl", "-n", "benchmark-operator", "--ignore-not-found") diff --git a/cli/tests/models/test_benchmark.py b/cli/tests/models/test_benchmark.py new file mode 100755 index 000000000..a68e7a178 --- /dev/null +++ b/cli/tests/models/test_benchmark.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + + +@pytest.mark.integration +class TestBenchmark: + def test_run(self): + pass + + def test_wait(self): + pass + + def test_cleanup(self): + pass + + def test_get_metadata(self): + pass diff --git a/cli/tests/models/test_workload.py b/cli/tests/models/test_workload.py new file mode 100755 index 000000000..5a9a9166d --- /dev/null +++ b/cli/tests/models/test_workload.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + + +@pytest.mark.integration +class TestWorkload: + def test_run(self): + pass + + def test_run_all(self): + pass + + def test_inject_overrides(self): + pass diff --git a/cli/tests/resources/benchmark.yaml b/cli/tests/resources/benchmark.yaml new file mode 100644 index 000000000..bc226fd1b --- /dev/null +++ b/cli/tests/resources/benchmark.yaml @@ -0,0 +1,16 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: byowl + namespace: benchmark-operator +spec: + system_metrics: + collection: false + metadata: + collection: false + workload: + name: byowl + args: + image: "quay.io/jtaleric/uperf:testing" + clients: 1 + commands: "echo Test" diff --git a/cli/tests/resources/job.yaml b/cli/tests/resources/job.yaml new file mode 100755 index 000000000..976700ca1 --- /dev/null +++ b/cli/tests/resources/job.yaml @@ -0,0 +1,18 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: busybox + namespace: default + labels: + app: test-job +spec: + template: + spec: + containers: + - image: busybox + command: + - sleep + - "3600" + imagePullPolicy: IfNotPresent + name: busybox + restartPolicy: Never diff --git a/cli/tox.ini b/cli/tox.ini new file mode 100644 index 000000000..c8b771b20 --- /dev/null +++ b/cli/tox.ini @@ -0,0 +1,19 @@ +[tox] +envlist = + py{38,39}{-unit} +skip_missing_interpreters = true + +[testenv:py{38,39}-unit] +extras = + tests +setenv = + py{38,39}-unit: COVERAGE_FILE = .coverage.{envname} +commands = + pytest -m unit --cov-config=.coveragerc --cov=ripsaw --cov-report term-missing {posargs} + +[testenv:coverage] +skip_install = true +deps = coverage +commands = + coverage combine + coverage xml diff --git a/cli/version.txt b/cli/version.txt new file mode 100644 index 000000000..8acdd82b7 --- /dev/null +++ b/cli/version.txt @@ -0,0 +1 @@ +0.0.1 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 225839355..d719ee940 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -63,10 +63,12 @@ spec: initialDelaySeconds: 5 periodSeconds: 10 - name: redis-master - image: k8s.gcr.io/redis:v1 + image: bitnami/redis:latest env: - name: MASTER value: "true" + - name: ALLOW_EMPTY_PASSWORD + value: "yes" ports: - containerPort: 6379 resources: From 7df4c9a4c7fd93c11e996756fba8af938a378320 Mon Sep 17 00:00:00 2001 From: Sebastian Jug Date: Mon, 16 Aug 2021 09:06:27 -0400 Subject: [PATCH 121/249] Fix CR links in docs and standardize - Small changes to iperf3 CR --- CONTRIBUTING.md | 6 +++--- Makefile | 2 +- README.md | 4 +++- config/manager/kustomization.yaml | 10 ++++++++-- config/samples/iperf3/cr.yaml | 2 +- docs/cyclictest.md | 3 ++- docs/elastic.md | 2 +- docs/fio_distributed.md | 5 +++-- docs/flent.md | 2 +- docs/fs-drift.md | 7 ++++--- docs/{iperf.md => iperf3.md} | 10 +++++----- docs/oslat.md | 3 ++- docs/smallfile.md | 6 +++--- docs/sysbench.md | 2 +- docs/uperf.md | 2 +- docs/ycsb.md | 2 +- 16 files changed, 40 insertions(+), 28 deletions(-) rename docs/{iperf.md => iperf3.md} (98%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e6317012..3dae03a37 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ Example `playbook.yml`: ### Workload triggers [CRD](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/) holds the definition of the resource. -The operator triggers roles based on the conditions defined in a CR ([example](resources/crds/ripsaw_v1alpha1_uperf_cr.yaml)) which will influence which roles the +The operator triggers roles based on the conditions defined in a CR ([example](config/samples/uperf/cr.yaml) which will influence which roles the [playbook](playbook.yml) executes. Other vars may be defined that can modify the workload run conditions. @@ -154,7 +154,7 @@ Redefine CRD ``` Apply a new CR ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_uperf_cr.yaml +# kubectl apply -f config/samples/uperf/cr.yaml ``` ## CI @@ -166,7 +166,7 @@ To ensure that adding new a workload will not break other workloads and its behavior can be predicted, we've mandated writing tests before PR can be merged. If a new workload is added, please follow the instructions to add a testcase to -[test.sh](test,sh): +[test.sh](test.sh): * copy an existing test like [uperf test](tests/test_uperf.sh) * Add commands needed to setup the workload specific requirements if any * Add a valid cr file to [test_crs](tests/test_crs/) directory for your workload diff --git a/Makefile b/Makefile index 9a894a1f6..bb4701448 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ ARCH ?= $(shell uname -m | sed 's/x86_64/amd64/') # Image URL to use all building/pushing image targets REGISTRY ?= quay.io ORG ?= cloud-bulldozer -# Get the current branch/tag name +# Get the current branch name # In case this is the master branch, rename it to latest VERSION ?= $(shell git describe --tags 2>/dev/null || git rev-parse --abbrev-ref HEAD | sed 's/master/latest/g') IMG ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) diff --git a/README.md b/README.md index 1a092b57f..9d299d054 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ The easiest way to install the operator is through the operator-sdk methods prov ```bash git clone https://github.com/cloud-bulldozer/benchmark-operator +cd benchmark-operator make deploy ``` @@ -16,6 +17,7 @@ If you wish to build a version of the operator from your local copy of the repo, ```bash git clone https://github.com/cloud-bulldozer/benchmark-operator +cd benchmark-operator make image-build image-push deploy IMG=$YOUR_IMAGE ``` @@ -49,7 +51,7 @@ helm uninstall benchmark-operator -n benchmark-operator | Workload | Use | ElasticSearch indexing | Reconciliation usage | VM support (kubevirt) | Kata Containers | CI Tested | | ------------------------------ | ---------------------- | ------------------ | -------------------------- | --------------------- | --------------- | ------------ | | [UPerf](docs/uperf.md) | Network Performance | Yes | Used, default : 3second | Working | Working | Yes | -| [Iperf3](docs/iperf.md) | Network Performance | No | Used, default : 3second | Not Supported | Preview | Yes | +| [Iperf3](docs/iperf3.md) | Network Performance | No | Used, default : 3second | Not Supported | Preview | Yes | | [fio](docs/fio_distributed.md) | Storage IO | Yes | Used, default : 3second | Working | Working | Yes | | [Sysbench](docs/sysbench.md) | System Performance | No | Used, default : 3second | Not Supported | Preview | Yes | | [YCSB](docs/ycsb.md) | Database Performance | Yes | Used, default : 3second | Not Supported | Preview | Yes | diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 2bcd3eeaa..f6241927a 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,6 +5,12 @@ generatorOptions: disableNameSuffixHash: true configMapGenerator: -- name: manager-config - files: +- files: - controller_manager_config.yaml + name: manager-config +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/cloud-bulldozer/benchmark-operator + newTag: latest diff --git a/config/samples/iperf3/cr.yaml b/config/samples/iperf3/cr.yaml index a14c8a081..2a95a56ae 100644 --- a/config/samples/iperf3/cr.yaml +++ b/config/samples/iperf3/cr.yaml @@ -12,7 +12,7 @@ spec: transmit_type: time transmit_value: 60 omit_start: 0 - length_buffer: 128K +# length_buffer: 128K window_size: 64k ip_tos: 0 mss: 900 diff --git a/docs/cyclictest.md b/docs/cyclictest.md index 0e2d6a23e..e1c1b9c40 100644 --- a/docs/cyclictest.md +++ b/docs/cyclictest.md @@ -43,7 +43,8 @@ spec: You can run it by: ```bash -oc apply -f resources/crds/ripsaw_v1alpha1_cyclictest_cr.yaml # if edited the original one +# kubectl apply -f config/samples/cyclictest/cr.yaml # if edited the original one +# kubectl apply -f # if created a new cr file ``` ## Looking at results diff --git a/docs/elastic.md b/docs/elastic.md index 3e11ce0c1..feaa398f0 100644 --- a/docs/elastic.md +++ b/docs/elastic.md @@ -29,7 +29,7 @@ Current supported ES + Prometheus integrated workloads: | Workload | Status | | ------------------------------ | ---------------------- | | [UPerf](docs/uperf.md) | Not Supported | -| [Iperf3](docs/iperf.md) | Not Supported | +| [Iperf3](docs/iperf3.md) | Not Supported | | [fio](docs/fio_distributed.md) | Supported | | [Sysbench](docs/sysbench.md) | Not Supported | | [YCSB](docs/ycsb.md) | Not Supported | diff --git a/docs/fio_distributed.md b/docs/fio_distributed.md index 73a967e4b..d8728a629 100644 --- a/docs/fio_distributed.md +++ b/docs/fio_distributed.md @@ -256,7 +256,8 @@ the cache drop by URL. **You must deploy this privileged pod in order for the drop caches requests in the workload to function.** ```bash -kubectl apply -f config/samples/fio/cr.yaml +# kubectl apply -f config/samples/fio/cr.yaml # if edited the original one +# kubectl apply -f # if created a new cr file ``` *Note: If Ceph is in a namespace other than `rook-ceph` you will need to modify the provided YAML accordingly.* @@ -265,7 +266,7 @@ Since the cache drop pod is deployed with host networking, the pod will take on of the node on which it is running. You will need to use this IP address in the CR file as described above. ```bash -kubectl get pod -n rook-ceph rook-ceph-osd-cache-drop --template={{.status.podIP}} +# kubectl get pod -n rook-ceph rook-ceph-osd-cache-drop --template={{.status.podIP}} ``` ## Indexing in elasticsearch and visualization through Grafana diff --git a/docs/flent.md b/docs/flent.md index 1ff19cc53..f87b793ae 100644 --- a/docs/flent.md +++ b/docs/flent.md @@ -233,7 +233,7 @@ To enable Multus in Ripsaw, here is the relevant config. Once done creating/editing the resource file, you can run it by: ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_flent_cr.yaml # if edited the original one +# kubectl apply -f config/samples/flent/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` diff --git a/docs/fs-drift.md b/docs/fs-drift.md index 678c2690f..2fc7f6c9b 100644 --- a/docs/fs-drift.md +++ b/docs/fs-drift.md @@ -9,7 +9,7 @@ tree - see the benchmark documentation for details. fs-drift requires that a st ## Running fs-drift Benchmark -Once the operator has been installed following the instructions, one needs to modify the [cr.yaml](../resources/crds/ripsaw_v1alpha1_fs-drift_cr.yaml) to customize workload parameters - the defaults are selected to demonstrate its operation and are not intended to specify a long-duration test. +Once the operator has been installed following the instructions, one needs to modify the [cr.yaml](../config/samples/fs-drift/cr.yaml) to customize workload parameters - the defaults are selected to demonstrate its operation and are not intended to specify a long-duration test. The parameters in [cr.yaml](../config/samples/fs-drift/cr.yaml) would look similar to this example: @@ -37,7 +37,7 @@ specified in a YAML input file also. YAML input style is a little different -- is omitted, and single dashes are converted to underscores. So "--parameter-foo bar" becomes "parameter_foo: bar". Operator CRs apparently also do not allow dashes in key names. So for the above example, use the -syntax "parameter_foo" instead of "--parameter-foo". See resources/crds/ for an example of fs-drift CR. +syntax "parameter_foo" instead of "--parameter-foo". See `config/samples/fs-drift/` for an example of fs-drift CR. The following fs-drift parameters will be overridden when fs-drift is used in ripsaw - do not specify these parameters yourself! @@ -56,7 +56,8 @@ intended for Kata containers. Once done creating/editing the CR file below, one can run it by: ```bash -kubectl apply -f resources/crds/ripsaw_v1alpha1_fs-drift_cr.yaml +# kubectl apply -f config/samples/fs-drift/cr.yaml # if edited the original one +# kubectl apply -f # if created a new cr file ``` Deploying the above(assuming worker_pods set to 2) would result in diff --git a/docs/iperf.md b/docs/iperf3.md similarity index 98% rename from docs/iperf.md rename to docs/iperf3.md index fb4a205db..51c035a4d 100644 --- a/docs/iperf.md +++ b/docs/iperf3.md @@ -19,14 +19,14 @@ spec: args: pairs: 1 hostnetwork: false - pin: true - pin_server: "master-0" - pin_client: "master-1" + #pin: true + #pin_server: "master-0" + #pin_client: "master-1" port: 5201 transmit_type: time transmit_value: 60 omit_start: 0 - length_buffer: 128K + #length_buffer: 128K window_size: 64k ip_tos: 0 mss: 900 @@ -116,7 +116,7 @@ $ oc adm policy add-scc-to-user privileged -z benchmark-operator Once done creating/editing the resource file, you can run it by: ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_iperf3_cr.yaml # if edited the original one +# kubectl apply -f config/samples/iperf3/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` diff --git a/docs/oslat.md b/docs/oslat.md index 9cd3db504..70c3b450a 100644 --- a/docs/oslat.md +++ b/docs/oslat.md @@ -42,7 +42,8 @@ spec: You can run it by: ```bash -oc apply -f resources/crds/ripsaw_v1alpha1_oslat_cr.yaml # if edited the original one +# kubectl apply -f config/samples/oslat/cr.yaml # if edited the original one +# kubectl apply -f # if created a new cr file ``` ## Looking at results diff --git a/docs/smallfile.md b/docs/smallfile.md index 7b3955803..b4ca9c627 100644 --- a/docs/smallfile.md +++ b/docs/smallfile.md @@ -3,7 +3,7 @@ [Smallfile](https://github.com/distributed-system-analysis/smallfile) is a python-based distributed POSIX workload generator which can be used to quickly measure performance for a variety of metadata-intensive workloads across an entire cluster. ## Running Smallfile Benchmark using Ripsaw -Once the operator has been installed following the instructions, one needs to modify the clients parameter(which is currently set to 0), to value greater than 0 in [ripsaw_v1alpha1_smallfile_cr.yaml](../config/samples/smallfile/cr.yaml) to run default "create" the test. Also, in addtion to that, smallfile operator is completely dependent on storageclass and storagesize. Please make sure to double check the parameters in CRD file. +Once the operator has been installed following the instructions, one needs to modify the clients parameter(which is currently set to 0), to value greater than 0 in [cr.yaml](../config/samples/smallfile/cr.yaml) to run default "create" the test. Also, in addtion to that, smallfile operator is completely dependent on storageclass and storagesize. Please make sure to double check the parameters in CRD file. ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 @@ -64,7 +64,7 @@ Smallfile also comes with a variety of configurable options for running tests, f the file operation duration down to microsecond resolution. * **file_size_distribution** – only supported value today is exponential. * **record_size** -- record size in KB, how much data is transferred in a single - read or write system call. If 0 then it is set to the minimum of the file + read or write system call. If 0 then it is set to the minimum of the file size and 1-MiB record size limit. * **files_per_dir** -- maximum number of files contained in any one directory. * **dirs_per_dir** -- maximum number of subdirectories contained in any one @@ -107,7 +107,7 @@ previous runs for example) Once done creating/editing the resource file, one can run it by: ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml # if edited the original one +# kubectl apply -f config/samples/smallfile/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` diff --git a/docs/sysbench.md b/docs/sysbench.md index 46e5ccc56..3575a2383 100644 --- a/docs/sysbench.md +++ b/docs/sysbench.md @@ -44,7 +44,7 @@ You can find more information at [sysbench documentation](https://github.com/ako Once done creating/editing the resource file, you can run it by: ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml # if edited the original one +# kubectl apply -f config/samples/sysbench/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` diff --git a/docs/uperf.md b/docs/uperf.md index 799fa4f00..1a186283a 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -251,7 +251,7 @@ combinations of UPERF data points to exercise datapath performance from many ang Once done creating/editing the resource file, you can run it by: ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_uperf_cr.yaml # if edited the original one +# kubectl apply -f config/samples/uperf/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` diff --git a/docs/ycsb.md b/docs/ycsb.md index 0a6162a66..c3a5fb3ab 100644 --- a/docs/ycsb.md +++ b/docs/ycsb.md @@ -71,6 +71,6 @@ so that the ycsb pod can access the API of database. Once done creating/editing the resource file, you can run it by: ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_ycsb_cr.yaml # if edited the original one +# kubectl apply -f config/samples/ycsb/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` From ef4b6638c806969db99f1dae4cfc9578fc7db3c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Fri, 20 Aug 2021 17:00:26 +0200 Subject: [PATCH 122/249] Benchmark status handler (#638) * Add benchmark-uuid label Signed-off-by: Raul Sevilla * Functions to set benchmark status Signed-off-by: Raul Sevilla * Get benchmark status in pre_tasks Signed-off-by: Raul Sevilla * Reuse benchmark_state Signed-off-by: Raul Sevilla * Append trunc_uuid to redis key Signed-off-by: Raul Sevilla * Append uuid to publisher channel Signed-off-by: Raul Sevilla * set_state mechanism Signed-off-by: Raul Sevilla * CRD Fixes Signed-off-by: Raul Sevilla --- .../ripsaw.cloudbulldozer.io_benchmarks.yaml | 41 +++- docs/flent.md | 12 +- max-namespaces.yml | 45 ---- playbooks/benchmark.yml | 50 +++-- playbooks/templates/metadata_pod.yml.j2 | 43 ++++ roles/backpack/tasks/main.yml | 33 +-- roles/backpack/templates/backpack.yml | 1 + roles/benchmark_state/tasks/completed.yml | 41 ++++ roles/benchmark_state/tasks/set_state.yml | 17 ++ roles/byowl/tasks/main.yml | 69 +----- roles/byowl/templates/workload.yml | 6 +- roles/cyclictest/tasks/main.yml | 91 +------- roles/cyclictest/templates/cyclictestjob.yaml | 5 +- roles/fio_distributed/tasks/main.yml | 113 +++------- roles/fio_distributed/templates/client.yaml | 4 +- .../templates/server_vm.yml.j2 | 1 + roles/fio_distributed/templates/servers.yaml | 99 ++++---- roles/flent/tasks/main.yml | 212 ++++++------------ roles/flent/templates/server.yml.j2 | 50 ++--- roles/flent/templates/workload.yml.j2 | 5 +- roles/fs-drift/tasks/main.yml | 71 +----- roles/fs-drift/templates/publisher.py.j2 | 5 +- roles/fs-drift/templates/subscriber.py.j2 | 3 +- roles/fs-drift/templates/workload_job.yml.j2 | 3 +- roles/hammerdb/tasks/main.yaml | 193 +++++----------- roles/hammerdb/templates/db_creation.yml | 8 +- .../templates/db_creation_mariadb_vm.yml | 2 +- .../templates/db_creation_mssql_vm.yml | 2 +- .../templates/db_creation_postgres_vm.yml | 2 +- .../templates/db_mariadb_workload.yml.j2 | 7 +- .../templates/db_mssql_workload.yml.j2 | 7 +- .../templates/db_postgres_workload.yml.j2 | 7 +- roles/image_pull/tasks/main.yml | 89 +------- roles/image_pull/templates/image_pull.yml | 3 + roles/iperf3/tasks/main.yml | 117 +++------- roles/iperf3/templates/client.yml.j2 | 4 +- roles/iperf3/templates/client_store.yml.j2 | 1 + roles/iperf3/templates/server.yml.j2 | 69 +++--- roles/kube-burner/tasks/main.yml | 79 +------ .../kube-burner/templates/kube-burner.yml.j2 | 1 + roles/log_generator/tasks/main.yml | 89 +------- .../log_generator/templates/log_generator.yml | 3 + roles/oslat/tasks/main.yml | 91 +------- roles/oslat/templates/oslatjob.yaml | 8 +- roles/pgbench/tasks/check_clients.yml | 13 +- roles/pgbench/tasks/init.yml | 13 +- roles/pgbench/tasks/main.yml | 34 +-- roles/pgbench/tasks/prep_workload.yml | 13 +- roles/pgbench/tasks/run_workload.yml | 13 +- roles/pgbench/templates/workload.yml.j2 | 4 +- roles/scale_openshift/tasks/main.yml | 66 +----- roles/scale_openshift/templates/scale.yml | 4 +- roles/servicemesh/tasks/main.yml | 152 +++---------- roles/smallfile/tasks/main.yml | 70 +----- roles/smallfile/templates/publisher.py.j2 | 5 +- .../smallfilejob.other_parameters.yaml.j2 | 16 +- roles/smallfile/templates/subscriber.py.j2 | 3 +- roles/smallfile/templates/workload_job.yml.j2 | 5 +- roles/stressng/tasks/main.yaml | 87 ++----- .../templates/stressng_workload.yml.j2 | 7 +- .../templates/stressng_workload_vm.yml.j2 | 2 +- roles/sysbench/tasks/main.yml | 68 +----- roles/sysbench/templates/workload.yml | 4 +- roles/sysbench/templates/workload_vm.yml | 1 + roles/system-metrics/tasks/main.yml | 2 +- .../templates/kube-burner.yml.j2 | 3 +- roles/testpmd/tasks/main.yml | 111 +++------ roles/testpmd/templates/testpmd.yml.j2 | 1 + roles/testpmd/templates/trex.yml.j2 | 1 + roles/uperf/tasks/main.yml | 30 +-- roles/uperf/tasks/next_set.yml | 10 +- roles/uperf/tasks/run_a_set.yml | 2 +- roles/uperf/tasks/send_client_run_signal.yml | 2 +- roles/uperf/tasks/setup.yml | 27 --- roles/uperf/tasks/wait_client_done.yml | 2 +- roles/uperf/tasks/wait_client_ready.yml | 2 +- roles/uperf/tasks/wait_server_ready.yml | 2 +- roles/uperf/tasks/wait_set_done.yml | 2 +- roles/uperf/templates/server.yml.j2 | 2 +- roles/uperf/templates/server_vm.yml.j2 | 2 +- roles/uperf/templates/service.yml.j2 | 2 +- roles/uperf/templates/workload.yml.j2 | 2 +- roles/uperf/templates/workload_vm.yml.j2 | 2 +- roles/vegeta/tasks/main.yml | 71 +----- roles/vegeta/templates/vegeta.yml.j2 | 3 + roles/ycsb/tasks/main.yml | 142 ++++-------- 86 files changed, 836 insertions(+), 1969 deletions(-) delete mode 100644 max-namespaces.yml create mode 100644 playbooks/templates/metadata_pod.yml.j2 create mode 100644 roles/benchmark_state/tasks/completed.yml create mode 100644 roles/benchmark_state/tasks/set_state.yml diff --git a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml index 0b72502c0..90d4a716d 100644 --- a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml +++ b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml @@ -24,6 +24,7 @@ spec: uuid: type: string elasticsearch: + default: {} type: object properties: url: @@ -32,8 +33,10 @@ spec: type: string parallel: type: boolean + default: false verify_cert: type: boolean + default: true prometheus: type: object properties: @@ -46,6 +49,7 @@ spec: prom_token: type: string system_metrics: + default: {} type: object properties: collection: @@ -73,6 +77,22 @@ spec: workload: x-kubernetes-preserve-unknown-fields: true type: object + properties: + name: + type: string + description: Workload name + args: + x-kubernetes-preserve-unknown-fields: true + type: object + properties: + debug: + type: boolean + default: false + description: Set debug log level in the workload + job_timeout: + type: integer + description: Timeout used for jobs deployed by this benchmark + default: 3600 job_params: type: array items: @@ -134,36 +154,41 @@ spec: drop_cache_rook_ceph: type: boolean status: + default: {} type: object properties: complete: type: boolean + default: false + description: Benchmark is completed metadata: type: string + description: Metadata collection status state: type: string + description: Benchmark status suuid: type: string + description: Benchmark short UUID uuid: type: string + description: Benchmark UUID pod_hi_idx: type: string pod_low_idx: - type: string + type: string node_hi_idx: type: string node_low_idx: - type: string + type: string pod_idx: - type: string + type: string node_idx: - type: string + type: string system_metrics: - type: string - default: Not collected - message: type: string - default: None + default: Not collected + description: System metrics collection status additionalPrinterColumns: - name: Type type: string diff --git a/docs/flent.md b/docs/flent.md index f87b793ae..e4af62416 100644 --- a/docs/flent.md +++ b/docs/flent.md @@ -16,8 +16,7 @@ metadata: spec: clustername: myk8scluster #elasticsearch: - # server: elk.server.com - # port: 9200 + # url: http://my.elasticsearch.server:80 #test_user: username_to_attach_to_metadata workload: # cleanup: true @@ -266,13 +265,8 @@ spec: runtime: 30 ``` -The new fields : - -`elasticsearch.server` this is the elasticsearch cluster ip you want to send the result data to for long term storage. - -`elasticsearch.port` port which elasticsearch is listening, typically `9200`. - -`user` provide a user id to the metadata that will be sent to Elasticsearch, this makes finding the results easier. +- `elasticsearch.url` Elastic search instance with full URL format. https://elastic.apps.org:9200 +- `user` provide a user id to the metadata that will be sent to Elasticsearch, this makes finding the results easier. By default we will utilize the `flent-results` index for Elasticsearch. diff --git a/max-namespaces.yml b/max-namespaces.yml deleted file mode 100644 index 927379ed2..000000000 --- a/max-namespaces.yml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: kube-burner-benchmark - namespace: my-ripsaw -spec: - metadata: - collection: false - force: false - image: quay.io/cloud-bulldozer/backpack:latest - privileged: false - serviceaccount: default - ssl: false - stockpileSkipTags: [] - stockpileTags: - - common - - k8s - - openshift - targeted: true - workload: - args: - burst: 50 - cleanup: true - default_index: ripsaw-kube-burner - error_on_verify: true - image: quay.io/cloud-bulldozer/kube-burner:latest - job_iterations: 1 - job_timeout: 300 - log_level: info - metrics_profile: metrics.yaml - node_selector: - key: node-role.kubernetes.io/worker - value: null - pin_server: - node-role.kubernetes.io/worker: "" - pod_wait: false - qps: 25 - step: 30s - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - verify_objects: true - wait_when_finished: true - workload: max-namespaces - name: kube-burner diff --git a/playbooks/benchmark.yml b/playbooks/benchmark.yml index bb58f4cd9..e4366aa44 100644 --- a/playbooks/benchmark.yml +++ b/playbooks/benchmark.yml @@ -2,6 +2,15 @@ collections: - operator_sdk.util gather_facts: no + pre_tasks: + + - name: Get current state + k8s_info: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ operator_namespace }}' + register: benchmark_state tasks: - include_role: @@ -12,15 +21,6 @@ name: "kernel_cache_drop" when: workload.args.drop_cache_kernel is defined - - - name: Get state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: cr_state - - name: Capture operator information k8s_facts: kind: Pod @@ -52,16 +52,16 @@ name: benchmark_state tasks_from: failure - when: workload is defined and (cr_state.resources[0].status is not defined or cr_state.resources[0].status.uuid is not defined) + when: workload is defined and (benchmark_state.resources[0].status is not defined or benchmark_state.resources[0].status.uuid is not defined) - set_fact: - uuid: "{{ cr_state.resources[0].status.uuid }}" - trunc_uuid: "{{ cr_state.resources[0].status.suuid }}" + uuid: "{{ benchmark_state.resources[0].status.uuid }}" + trunc_uuid: "{{ benchmark_state.resources[0].status.suuid }}" when: - - cr_state.resources[0].status is defined - - cr_state.resources[0].status.uuid is defined - - cr_state.resources[0].status.uuid != "" + - benchmark_state.resources[0].status is defined + - benchmark_state.resources[0].status.uuid is defined + - benchmark_state.resources[0].status.uuid != "" - name: Run Workload rescue: @@ -82,23 +82,31 @@ namespace: "{{ operator_namespace }}" status: metadata: "Collecting" - when: metadata is defined and metadata.targeted | default('true') | bool and not cr_state.resources[0].status.state is defined + when: metadata is defined and metadata.targeted | default('true') | bool and not benchmark_state.resources[0].status.state is defined - when: metadata is defined and metadata.collection | default('false') | bool and (cr_state.resources[0].status.metadata is defined and cr_state.resources[0].status.metadata != "Complete") + when: metadata is defined and metadata.collection | default('false') | bool and (benchmark_state.resources[0].status.metadata is defined and benchmark_state.resources[0].status.metadata != "Complete") - block: - include_role: name: "common" + - name: Set Building state + include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Building + when: benchmark_state.resources[0].status.state is not defined and workload.name != "backpack" + - include_role: name: "{{ workload.name }}" vars: workload_args: "{{ workload.args }}" - when: metadata is not defined or not metadata.collection | default('false') | bool or (cr_state.resources[0].status.metadata is defined and cr_state.resources[0].status.metadata == "Complete") or metadata.targeted | default('true') | bool + when: metadata is not defined or not metadata.collection | default('false') | bool or (benchmark_state.resources[0].status.metadata is defined and benchmark_state.resources[0].status.metadata == "Complete") or metadata.targeted | default('true') | bool - when: cr_state is defined and cr_state.resources[0].status is defined and not cr_state.resources[0].status.complete|bool and (cr_state.resources[0].status.state is not defined or cr_state.resources[0].status.state != "Error") + when: benchmark_state is defined and benchmark_state.resources[0].status is defined and not benchmark_state.resources[0].status.complete|bool and (benchmark_state.resources[0].status.state is not defined or benchmark_state.resources[0].status.state != "Error") - include_role: name: system-metrics @@ -106,7 +114,7 @@ vars: workload_args: "{{ workload.args }}" when: - - cr_state.resources[0].status.state is defined - - cr_state.resources[0].status.state == "Complete" + - benchmark_state.resources[0].status.state is defined + - benchmark_state.resources[0].status.state == "Complete" - system_metrics.collection | bool - workload.name not in ["kube-burner", "backpack"] diff --git a/playbooks/templates/metadata_pod.yml.j2 b/playbooks/templates/metadata_pod.yml.j2 new file mode 100644 index 000000000..6464e86d5 --- /dev/null +++ b/playbooks/templates/metadata_pod.yml.j2 @@ -0,0 +1,43 @@ +{% if metadata.collection is sameas true and metadata.targeted is sameas true %} +{% if metadata.serviceaccount != "default" %} + serviceAccountName: {{ metadata.serviceaccount }} +{% endif %} + initContainers: + - name: backpack + image: {{ metadata.image }} + command: ["/bin/sh", "-c"] + args: + - > + python3 + stockpile-wrapper.py + -s={{ elasticsearch.url }} + -u={{ uuid }} + -n=${my_node_name} + -N=${my_pod_name} + --redisip={{ bo.resources[0].status.podIP }} + --redisport=6379 +{% if metadata.force is sameas true %} + --force +{% endif %} + --tags={{ metadata.stockpileTags|default(["common", "k8s", "openshift"])|join(",") }} +{% if metadata.stockpileSkipTags|length > 0 %} + --skip-tags={{ metadata.stockpileSkipTags|join(",") }} +{% endif %} +{% if metadata.ssl is sameas true %} + --sslskipverify True +{% endif %} + imagePullPolicy: Always + securityContext: + privileged: {{ metadata.privileged }} + readOnlyRootFilesystem: false + runAsNonRoot: false + env: + - name: my_node_name + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: my_pod_name + valueFrom: + fieldRef: + fieldPath: metadata.name +{% endif %} diff --git a/roles/backpack/tasks/main.yml b/roles/backpack/tasks/main.yml index 68fcfa6c4..8a1ff169e 100644 --- a/roles/backpack/tasks/main.yml +++ b/roles/backpack/tasks/main.yml @@ -1,12 +1,4 @@ --- -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state - - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -58,25 +50,6 @@ set_fact: backpack_cur_gen: "{{ my_daemonset | json_query('resources[].metadata.generation')|first }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Metadata Collecting" - complete: false - metadata: "Collecting" - when: backpack_orig_gen != backpack_cur_gen - - - name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state - - block: - name: Get initial pod list @@ -95,9 +68,9 @@ name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: - state: null - complete: false - metadata: "Complete" + state: Complete + complete: true + metadata: Complete - name: Get benchmark state k8s_facts: diff --git a/roles/backpack/templates/backpack.yml b/roles/backpack/templates/backpack.yml index 5729ef89f..ad48ff7e9 100755 --- a/roles/backpack/templates/backpack.yml +++ b/roles/backpack/templates/backpack.yml @@ -16,6 +16,7 @@ spec: labels: name: backpack-{{ trunc_uuid }} app: backpack-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: tolerations: - key: node-role.kubernetes.io/master diff --git a/roles/benchmark_state/tasks/completed.yml b/roles/benchmark_state/tasks/completed.yml new file mode 100644 index 000000000..eff33efbd --- /dev/null +++ b/roles/benchmark_state/tasks/completed.yml @@ -0,0 +1,41 @@ +--- +- name: Get job status + k8s_info: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - benchmark-uuid = {{ uuid }} + register: job_state + +# Set Benchmark as Complete when all jobs succeed +- name: Setting benchmark state - Complete + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + when: job_state|json_query('resources[].status[]')|selectattr('succeeded', 'defined')|list|length == job_state.resources|length + +# Set Benchmark as failed when one of the jobs fails +- name: Setting benchmark state - Failed + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Failed + complete: true + when: job_state|json_query('resources[].status[]')|selectattr('failed', 'defined')|list|length > 0 + +- name: Get current state + k8s_info: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ operator_namespace }}' + register: benchmark_state diff --git a/roles/benchmark_state/tasks/set_state.yml b/roles/benchmark_state/tasks/set_state.yml new file mode 100644 index 000000000..936f47a4d --- /dev/null +++ b/roles/benchmark_state/tasks/set_state.yml @@ -0,0 +1,17 @@ +--- +- name: Setting benchmark state - Building + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "{{ state }}" + +- name: Get current benchmark state + k8s_info: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ operator_namespace }}' + register: benchmark_state diff --git a/roles/byowl/tasks/main.yml b/roles/byowl/tasks/main.yml index 05042d6d4..4ce2fd544 100644 --- a/roles/byowl/tasks/main.yml +++ b/roles/byowl/tasks/main.yml @@ -1,69 +1,20 @@ --- - -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: Start byowl job k8s: state: present definition: "{{ lookup('template', 'workload.yml') | from_yaml }}" - when: workload_args.kind is not defined - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false - - when: resource_state.resources[0].status.state == "Starting" - -- block: - - name: Waiting for pods to complete.... - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = byowl-{{ trunc_uuid }} - register: client_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "workload_args.clients == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: benchmark_state.resources[0].status.state == "Building" - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/byowl/templates/workload.yml b/roles/byowl/templates/workload.yml index effc34d31..214403169 100755 --- a/roles/byowl/templates/workload.yml +++ b/roles/byowl/templates/workload.yml @@ -1,15 +1,17 @@ --- +kind: Job apiVersion: batch/v1 -kind: "job" metadata: name: "byowl-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: byowl-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/cyclictest/tasks/main.yml b/roles/cyclictest/tasks/main.yml index a9fd46996..cbd25ddfb 100644 --- a/roles/cyclictest/tasks/main.yml +++ b/roles/cyclictest/tasks/main.yml @@ -1,31 +1,4 @@ --- - -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state if changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: Add cyclic test scripts @@ -40,64 +13,20 @@ cyclictest.sh: "{{ lookup('file', 'cyclictest.sh') }}" functions.sh: "{{ lookup('file', 'functions.sh') }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: ConfigMaps Created - complete: false - - when: resource_state.resources[0].status.state == "Building" - -- block: - name: Start cyclitest job k8s: state: present definition: "{{ lookup('template', 'cyclictestjob.yaml') | from_yaml }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Benchmark Running - complete: false - - when: resource_state.resources[0].status.state == "ConfigMaps Created" - -- block: - - - name: Wait for benchmark to complete - k8s_facts: - kind: Job - api_version: batch/v1 - name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' - namespace: "{{ operator_namespace }}" - register: cyclictest_status - - - name: Set Complete Status to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Benchmark Complete - complete: true - when: cyclictest_status.resources[0].status.succeeded is defined and (cyclictest_status.resources[0].status.succeeded | int) > 0 + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - name: Set failed state to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Failed - complete: true - when: cyclictest_status.resources[0].status.failed is defined and (cyclictest_status.resources[0].status.failed | int) > 0 + when: benchmark_state.resources[0].status.state == "Building" - when: resource_state.resources[0].status.state == "Benchmark Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index 7eb02f23f..38f0682e0 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -8,6 +8,7 @@ spec: metadata: labels: app: cyclictest-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -44,9 +45,7 @@ spec: value: "{{ clustername }}" {% if elasticsearch is defined %} - name: es - value: "{{ elasticsearch.server }}" - - name: es_port - value: "{{ elasticsearch.port }}" + value: "{{ elasticsearch.url}}" - name: es_index value: "{{ elasticsearch.index_name | default("ripsaw-cyclictest") }}" - name: parallel diff --git a/roles/fio_distributed/tasks/main.yml b/roles/fio_distributed/tasks/main.yml index f74ae4b94..17397e297 100644 --- a/roles/fio_distributed/tasks/main.yml +++ b/roles/fio_distributed/tasks/main.yml @@ -1,30 +1,4 @@ --- -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: Provide path for fio - PV set_fact: @@ -72,18 +46,15 @@ when: workload_args.servers|default('1')|int > 0 and (workload.args.kind | default('pod')) == "vm" with_sequence: start=1 count={{ workload_args.servers|default('1')|int }} - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: StartingServers - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: StartingServers - when: resource_state.resources[0].status.state == "Building" + when: benchmark_state.resources[0].status.state == "Building" -- block : +- block: - name: Capture pod list k8s_info: @@ -94,15 +65,12 @@ - app = fio-benchmark-{{ trunc_uuid }} register: server_pods - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: StartingClient - complete: false - when : "workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.podIP')|length) and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) and resource_state.resources[0].status.state == 'StartingServers'" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: StartingClient + when: "workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.podIP')|length) and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) and benchmark_state.resources[0].status.state == 'StartingServers'" - name: Create IP list and nodes set_fact: @@ -122,15 +90,12 @@ - app = fio-benchmark-{{ trunc_uuid }} register: server_pods - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: StartingClient - complete: false - when : "workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.interfaces[].ipAddress')|length) and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) and resource_state.resources[0].status.state == 'StartingServers'" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: StartingClient + when : "workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.interfaces[].ipAddress')|length) and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) and benchmark_state.resources[0].status.state == 'StartingServers'" - name: Create IP list and nodes set_fact: @@ -166,35 +131,15 @@ k8s: definition: "{{ lookup('template', 'client.yaml') | from_yaml }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - when: resource_state.resources[0].status.state == "StartingClient" + when: benchmark_state.resources[0].status.state == "StartingClient" -- block: - - name: Capture Client pod list - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = fiod-client-{{ trunc_uuid }} - register: client_pods - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when : "client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length > 0 " - - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/fio_distributed/templates/client.yaml b/roles/fio_distributed/templates/client.yaml index 372cca38e..df4e1def9 100644 --- a/roles/fio_distributed/templates/client.yaml +++ b/roles/fio_distributed/templates/client.yaml @@ -5,10 +5,12 @@ metadata: name: 'fio-client-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: + benchmark-uuid: {{ uuid }} app: fiod-client-{{ trunc_uuid }} spec: {% if workload_args.runtime_class is defined %} diff --git a/roles/fio_distributed/templates/server_vm.yml.j2 b/roles/fio_distributed/templates/server_vm.yml.j2 index 1da5d7e66..c302516d3 100644 --- a/roles/fio_distributed/templates/server_vm.yml.j2 +++ b/roles/fio_distributed/templates/server_vm.yml.j2 @@ -6,6 +6,7 @@ metadata: namespace: '{{ operator_namespace }}' labels: app: fio-benchmark-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: domain: cpu: diff --git a/roles/fio_distributed/templates/servers.yaml b/roles/fio_distributed/templates/servers.yaml index 213dacc6a..062c2f42c 100644 --- a/roles/fio_distributed/templates/servers.yaml +++ b/roles/fio_distributed/templates/servers.yaml @@ -1,71 +1,68 @@ --- -kind: Job -apiVersion: batch/v1 +kind: Pod +apiVersion: v1 metadata: name: 'fio-server-{{ item }}-benchmark-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + app: fio-benchmark-{{ trunc_uuid }} spec: - ttlSecondsAfterFinished: 600 - template: - metadata: - labels: - app: fio-benchmark-{{ trunc_uuid }} - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - fio-benchmark-{{ trunc_uuid }} - topologyKey: "kubernetes.io/hostname" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - fio-benchmark-{{ trunc_uuid }} + topologyKey: "kubernetes.io/hostname" {% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" + runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} - containers: - - name: fio-server + containers: + - name: fio-server {% if hostpath is defined %} - securityContext: - privileged: true + securityContext: + privileged: true {% endif %} - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/fio:latest') }} - imagePullPolicy: Always - ports: - - containerPort: 8765 - command: ["/bin/sh", "-c"] - args: - - "cd /tmp; fio --server" + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/fio:latest') }} + imagePullPolicy: Always + ports: + - containerPort: 8765 + command: ["/bin/sh", "-c"] + args: + - "cd /tmp; fio --server" {% if workload_args.storageclass is defined or hostpath is defined %} - volumeMounts: - - name: data-volume - mountPath: "{{ fio_path }}" + volumeMounts: + - name: data-volume + mountPath: "{{ fio_path }}" {% endif %} - restartPolicy: Never - serviceAccountName: benchmark-operator + restartPolicy: Never + serviceAccountName: benchmark-operator {% if workload_args.nodeselector is defined %} - nodeSelector: + nodeSelector: {% for label, value in workload_args.nodeselector.items() %} - {{ label | replace ("_", "-" )}}: {{ value }} + {{ label | replace ("_", "-" )}}: {{ value }} {% endfor %} {% endif %} {% if workload_args.tolerations is defined %} - tolerations: - {{ workload_args.tolerations }} + tolerations: + {{ workload_args.tolerations }} {% endif %} {% if workload_args.storageclass is defined %} - volumes: - - name: data-volume - persistentVolumeClaim: - claimName: claim-{{ item }}-{{ trunc_uuid }} + volumes: + - name: data-volume + persistentVolumeClaim: + claimName: claim-{{ item }}-{{ trunc_uuid }} {% elif hostpath is defined %} - volumes: - - name: data-volume - hostPath: - path: {{ hostpath }} - type: DirectoryOrCreate + volumes: + - name: data-volume + hostPath: + path: {{ hostpath }} + type: DirectoryOrCreate {% endif %} -{% include "metadata.yml.j2" %} +{% include "metadata_pod.yml.j2" %} diff --git a/roles/flent/tasks/main.yml b/roles/flent/tasks/main.yml index f71db76bb..d167d35f9 100644 --- a/roles/flent/tasks/main.yml +++ b/roles/flent/tasks/main.yml @@ -1,31 +1,4 @@ --- - -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: Start Server(s) @@ -43,16 +16,13 @@ - type = flent-bench-server-{{ trunc_uuid }} register: server_pods - - name: Update resource state to starting servers - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Servers" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting Servers - when: resource_state.resources[0].status.state == "Building" + when: benchmark_state.resources[0].status.state == "Building" - block: @@ -65,18 +35,15 @@ - type = flent-bench-server-{{ trunc_uuid }} register: server_pods - - name: Update resource state to starting clients - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting Clients when: "workload_args.pair|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" - when: resource_state.resources[0].status.state == "Starting Servers" + when: benchmark_state.resources[0].status.state == "Starting Servers" - block: @@ -95,117 +62,70 @@ definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" with_items: "{{ server_pods.resources }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Waiting for Clients + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Waiting for Clients - when: resource_state.resources[0].status.state == "Starting Clients" + when: benchmark_state.resources[0].status.state == "Starting Clients" - block: + - name: Get client pod status + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = flent-bench-client-{{ trunc_uuid }} + register: client_pods - block: - - name: Get client pod status - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = flent-bench-client-{{ trunc_uuid }} - register: client_pods - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Clients Running - when: "workload_args.pair|default('1')|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" - - when: resource_state.resources[0].status.state == "Waiting for Clients" -- block: + - name: Signal workload + command: "redis-cli set start-{{ trunc_uuid }} true" - - name: Signal workload - command: "redis-cli set start true" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running + when: "workload_args.pair|default('1')|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" - - name: Update resource state to Running - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running" + when: benchmark_state.resources[0].status.state == "Waiting for Clients" - when: resource_state.resources[0].status.state == "Clients Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" - block: - - block: - - name: Waiting for pods to complete.... - k8s_info: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = flent-bench-client-{{ trunc_uuid }} - register: client_pods - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false - when: "workload_args.pair|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" - - when: resource_state.resources[0].status.state == "Running" -- block: + - name: Get Server Pods + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = flent-bench-server-{{ trunc_uuid }} + register: server_pods - - block: - - name: Get Server Pods - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = flent-bench-server-{{ trunc_uuid }} - register: server_pods - - - name: Pod names - to clean - set_fact: - clean_pods: | - [ - {% for item in server_pods.resources %} - "{{ item['metadata']['name'] }}", - {% endfor %} - ] - - - name: Cleanup run - k8s: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - state: absent - name: "{{ item }}" - with_items: "{{ clean_pods }}" - when: cleanup - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - - when: resource_state.resources[0].status.state == "Cleanup" + - name: Pod names - to clean + set_fact: + clean_pods: | + [ + {% for item in server_pods.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + + - name: Cleanup run + k8s: + kind: pod + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_pods }}" + + when: benchmark_state.resources[0].status.complete and cleanup diff --git a/roles/flent/templates/server.yml.j2 b/roles/flent/templates/server.yml.j2 index 31f3754b8..18ffa49af 100644 --- a/roles/flent/templates/server.yml.j2 +++ b/roles/flent/templates/server.yml.j2 @@ -1,41 +1,37 @@ --- -kind: Job -apiVersion: batch/v1 +kind: Pod +apiVersion: v1 metadata: name: 'flent-server-{{ item }}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' -spec: - ttlSecondsAfterFinished: 600 - backoffLimit: 0 - template: - metadata: - labels: - app : flent-bench-server-{{item}}-{{ trunc_uuid }} - type : flent-bench-server-{{ trunc_uuid }} + labels: + app : flent-bench-server-{{item}}-{{ trunc_uuid }} + type : flent-bench-server-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} {% if workload_args.multus.enabled is sameas true %} - annotations: - k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} + annotations: + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} {% endif %} - spec: +spec: {% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" + runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} {% if workload_args.hostnetwork is sameas true %} - hostNetwork: true - serviceAccountName: benchmark-operator + hostNetwork: true + serviceAccountName: benchmark-operator {% endif %} - containers: - - name: benchmark - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/flent:latest') }} + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/flent:latest') }} {% if workload_args.server_resources is defined %} - resources: {{ workload_args.server_resources | to_json }} + resources: {{ workload_args.server_resources | to_json }} {% endif %} - imagePullPolicy: Always - command: ["/bin/sh","-c"] - args: ["netserver -D"] - restartPolicy: OnFailure + imagePullPolicy: Always + command: ["/bin/sh","-c"] + args: ["netserver -D"] + restartPolicy: OnFailure {% if workload_args.pin is sameas true %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_server }}' + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_server }}' {% endif %} -{% include "metadata.yml.j2" %} +{% include "metadata_pod.yml.j2" %} diff --git a/roles/flent/templates/workload.yml.j2 b/roles/flent/templates/workload.yml.j2 index 421318d7d..1f9791d66 100644 --- a/roles/flent/templates/workload.yml.j2 +++ b/roles/flent/templates/workload.yml.j2 @@ -10,6 +10,7 @@ spec: labels: app: flent-bench-client-{{ trunc_uuid }} clientfor: {{ item.metadata.labels.app }} + benchmark-uuid: {{ uuid }} {% if workload_args.multus.enabled is sameas true %} annotations: k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} @@ -67,7 +68,7 @@ spec: export hostnet={{workload_args.hostnetwork}}; cd /tmp/flent-test; while true; do - if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start) =~ 'true' ]]; then + if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{trunc_uuid}}) =~ 'true' ]]; then {% for test in workload_args.test_types %} run_snafu --tool flent --ftest {{test}} -u {{ uuid }} -r $h --user {{test_user | default("ripsaw")}} \ {% if workload_args.debug is defined and workload_args.debug %} @@ -81,7 +82,7 @@ spec: break; done; - redis-cli -h {{bo.resources[0].status.podIP}} set start false" + redis-cli -h {{bo.resources[0].status.podIP}} set start-{{trunc_uuid}} false" volumeMounts: - name: flent-tmp mountPath: "/tmp/flent-test" diff --git a/roles/fs-drift/tasks/main.yml b/roles/fs-drift/tasks/main.yml index d40dc8e40..cded37064 100644 --- a/roles/fs-drift/tasks/main.yml +++ b/roles/fs-drift/tasks/main.yml @@ -1,32 +1,4 @@ --- - -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- name: Set state to starting - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - set_fact: @@ -92,7 +64,7 @@ command: ["/bin/sh", "-c"] workingDir: /root/fs-drift-master/ args: - - python /tmp/publisher {{bo.resources[0].status.podIP}} {{workload_args.worker_pods|default('1')|int}} + - python /tmp/publisher {{bo.resources[0].status.podIP}} {{workload_args.worker_pods|default('1')|int}} {{ trunc_uuid }} volumeMounts: - name: config-volume mountPath: "/tmp" @@ -109,36 +81,15 @@ with_sequence: start=1 count={{workload_args.worker_pods|default('1')|int}} when: workload_args.worker_pods|default('1')|int > 0 - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false - - when: resource_state.resources[0].status.state == "Starting" - -- block: - - - name: Waiting for pods to complete.... - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = fs-drift-benchmark-{{ trunc_uuid }} - register: client_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "workload_args.worker_pods|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: benchmark_state.resources[0].status.state == "Building" - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/fs-drift/templates/publisher.py.j2 b/roles/fs-drift/templates/publisher.py.j2 index 9a31395be..a831cac48 100644 --- a/roles/fs-drift/templates/publisher.py.j2 +++ b/roles/fs-drift/templates/publisher.py.j2 @@ -8,16 +8,17 @@ import sys redis_host = sys.argv[1] number_of_pods = sys.argv[2] +trunc_uuid = sys.argv[3] try: r = redis.StrictRedis(host=redis_host, port=6379) count = 0 while int(count) != int(number_of_pods): - count = r.execute_command("PUBSUB NUMSUB fs-drift")[1] + count = r.execute_command(f"PUBSUB NUMSUB fs-drift-{trunc_uuid}")[1] print(count, file=sys.stdout) time.sleep(1) print("All Pods are ready to run. Triggering the run ........\(^_^)/", file=sys.stdout) - r.publish("fs-drift","run") + r.publish(f"fs-drift-{trunc_uuid}","run") except Exception as e: print("******Exception Occured*******") print(str(e)) diff --git a/roles/fs-drift/templates/subscriber.py.j2 b/roles/fs-drift/templates/subscriber.py.j2 index 4cf0d29e5..bd685c9ab 100644 --- a/roles/fs-drift/templates/subscriber.py.j2 +++ b/roles/fs-drift/templates/subscriber.py.j2 @@ -6,12 +6,13 @@ import sys import redis redis_host = sys.argv[1] +trunc_uuid = sys.argv[2] try: r = redis.StrictRedis(host=redis_host, port=6379) p = r.pubsub() - p.subscribe('fs-drift') + p.subscribe(f'fs-drift-{trunc_uuid}') STATE = True while STATE: diff --git a/roles/fs-drift/templates/workload_job.yml.j2 b/roles/fs-drift/templates/workload_job.yml.j2 index 0878fb28f..80d437098 100644 --- a/roles/fs-drift/templates/workload_job.yml.j2 +++ b/roles/fs-drift/templates/workload_job.yml.j2 @@ -11,6 +11,7 @@ spec: metadata: labels: app: fs-drift-benchmark-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -74,7 +75,7 @@ spec: rm -rf ${TMPDIR}; mkdir -pv ${TMPDIR}/RESULTS; mkdir -pv {{fs_drift_path}}/fs_drift_test_data; - python /tmp/fs-drift/subscriber {{bo.resources[0].status.podIP}}; + python /tmp/fs-drift/subscriber {{bo.resources[0].status.podIP}} {{ trunc_uuid }}; cat /tmp/fs-drift/params; run_snafu --tool fs-drift diff --git a/roles/hammerdb/tasks/main.yaml b/roles/hammerdb/tasks/main.yaml index 13f9892fb..1b631ea38 100644 --- a/roles/hammerdb/tasks/main.yaml +++ b/roles/hammerdb/tasks/main.yaml @@ -1,30 +1,4 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - name: Create PVC(s) k8s: definition: @@ -154,37 +128,28 @@ run_postgres_script.sh: "{{ lookup('template', 'db_postgres_workload_vm.sh.j2') }}" when: workload_args.db_type == "pg" and resource_kind == "vm" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting DB - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting DB when: workload_args.db_init is defined and workload_args.db_init - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting DB Workload - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting DB Workload when: workload_args.db_benchmark is defined and workload_args.db_benchmark and (workload_args.db_init is not defined or not workload_args.db_init) - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: ConfigMaps Created - complete: true + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: ConfigMaps Created when: (workload_args.db_benchmark is not defined or not workload_args.db_benchmark) and (workload_args.db_init is not defined or not workload_args.db_init) - when: resource_state.resources[0].status.state == "Building" + when: benchmark_state.resources[0].status.state == "Building" - block: - block: @@ -219,16 +184,13 @@ when: resource_kind == "vm" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: DB Creating - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: DB Creating - when: resource_state.resources[0].status.state == "Starting DB" + when: benchmark_state.resources[0].status.state == "Starting DB" - block: @@ -241,52 +203,40 @@ namespace: "{{ operator_namespace }}" register: hammerdb_creator_pod - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting DB Workload - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting DB Workload when: hammerdb_creator_pod | json_query('resources[].status.succeeded') and workload_args.db_benchmark is defined and workload_args.db_benchmark - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: DB Created - complete: true + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: DB Created when: hammerdb_creator_pod | json_query('resources[].status.succeeded') and (workload_args.db_benchmark is not defined or not workload_args.db_benchmark) when: resource_kind == "pod" - block: - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting DB Workload - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting DB Workload - name: get complete command: "redis-cli get db-creation-{{trunc_uuid}}" register: db_creation_status - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: DB Created - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: DB Created when: db_creation_status.stdout == "true" when: resource_kind == "vm" - when: resource_state.resources[0].status.state == "DB Creating" + when: benchmark_state.resources[0].status.state == "DB Creating" - block: - name: set complete to false @@ -311,52 +261,15 @@ definition: "{{ lookup ('template', 'db_postgres_workload.yml.j2') | from_yaml }}" when: workload_args.db_type == "pg" and resource_kind == "pod" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: DB workload running - complete: false - - when: resource_state.resources[0].status.state == "Starting DB Workload" - -- block: - - block: - - name: wait for db workload job to finish - k8s_facts: - kind: Job - api_version: batch/v1 - name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' - namespace: "{{ operator_namespace }}" - register: hammerdb_workload_pod - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "hammerdb_workload_pod | json_query('resources[].status.succeeded')" - when: resource_kind == "pod" - - - block: - - name: get complete - command: "redis-cli get complete" - register: complete_status + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: complete_status.stdout == "true" - when: resource_kind == "vm" + when: benchmark_state.resources[0].status.state == "Starting DB Workload" - when: resource_state.resources[0].status.state == "DB workload running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/hammerdb/templates/db_creation.yml b/roles/hammerdb/templates/db_creation.yml index ed4cdea79..d09731a50 100644 --- a/roles/hammerdb/templates/db_creation.yml +++ b/roles/hammerdb/templates/db_creation.yml @@ -1,16 +1,18 @@ +--- apiVersion: batch/v1 -kind: "job" +kind: Job metadata: name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: hammerdb_creator-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-creator-{{ trunc_uuid }} - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.pin is sameas true %} nodeSelector: diff --git a/roles/hammerdb/templates/db_creation_mariadb_vm.yml b/roles/hammerdb/templates/db_creation_mariadb_vm.yml index 1e937ea92..ddbc0007f 100644 --- a/roles/hammerdb/templates/db_creation_mariadb_vm.yml +++ b/roles/hammerdb/templates/db_creation_mariadb_vm.yml @@ -7,7 +7,7 @@ metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} spec: domain: cpu: diff --git a/roles/hammerdb/templates/db_creation_mssql_vm.yml b/roles/hammerdb/templates/db_creation_mssql_vm.yml index 78f2b5a35..c6a018d84 100644 --- a/roles/hammerdb/templates/db_creation_mssql_vm.yml +++ b/roles/hammerdb/templates/db_creation_mssql_vm.yml @@ -7,7 +7,7 @@ metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} spec: domain: cpu: diff --git a/roles/hammerdb/templates/db_creation_postgres_vm.yml b/roles/hammerdb/templates/db_creation_postgres_vm.yml index 10a8c7b77..ad3c4582c 100644 --- a/roles/hammerdb/templates/db_creation_postgres_vm.yml +++ b/roles/hammerdb/templates/db_creation_postgres_vm.yml @@ -7,7 +7,7 @@ metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} spec: domain: cpu: diff --git a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 index 0e4fdc6c2..621f7118a 100644 --- a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 @@ -1,17 +1,18 @@ --- apiVersion: batch/v1 -kind: "job" +kind: Job metadata: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.pin is sameas true %} nodeSelector: diff --git a/roles/hammerdb/templates/db_mssql_workload.yml.j2 b/roles/hammerdb/templates/db_mssql_workload.yml.j2 index 64743a9ff..ddaf94283 100644 --- a/roles/hammerdb/templates/db_mssql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mssql_workload.yml.j2 @@ -1,17 +1,18 @@ --- apiVersion: batch/v1 -kind: "job" +kind: Job metadata: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.pin is sameas true %} nodeSelector: diff --git a/roles/hammerdb/templates/db_postgres_workload.yml.j2 b/roles/hammerdb/templates/db_postgres_workload.yml.j2 index dbff4ab60..9e1ef570b 100644 --- a/roles/hammerdb/templates/db_postgres_workload.yml.j2 +++ b/roles/hammerdb/templates/db_postgres_workload.yml.j2 @@ -1,17 +1,18 @@ --- apiVersion: batch/v1 -kind: "job" +kind: Job metadata: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.pin is sameas true %} nodeSelector: diff --git a/roles/image_pull/tasks/main.yml b/roles/image_pull/tasks/main.yml index c34eb4bcf..793d90d76 100644 --- a/roles/image_pull/tasks/main.yml +++ b/roles/image_pull/tasks/main.yml @@ -1,30 +1,4 @@ --- -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Image Pull Pods" - complete: false - when: benchmark_state.resources[0].status.state is not defined - -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state - - block: - name: Image pull pods @@ -32,58 +6,15 @@ state: present definition: "{{ lookup('template', 'image_pull.yml') }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Building Pods" - - when: benchmark_state.resources[0].status.state == "Starting Image Pull Pods" - -- block: - - - name: Get server pods - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = image-pull-{{ trunc_uuid }} - register: image_pods - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Image Pull Pods Running" - when: "workload_args.pod_count|default('1')|int == image_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" - - when: benchmark_state.resources[0].status.state == "Building Pods" - -- block: - - - name: Get server pods - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = image-pull-{{ trunc_uuid }} - register: image_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "workload_args.pod_count|default('1')|int == (image_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: benchmark_state.resources[0].status.state == "Building" - when: benchmark_state.resources[0].status.state == "Image Pull Pods Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/image_pull/templates/image_pull.yml b/roles/image_pull/templates/image_pull.yml index 5299761e0..c0a100196 100644 --- a/roles/image_pull/templates/image_pull.yml +++ b/roles/image_pull/templates/image_pull.yml @@ -5,11 +5,14 @@ metadata: name: 'image-pull-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} parallelism: {{ workload_args.pod_count | default(1) | int }} template: metadata: labels: app: image-pull-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/iperf3/tasks/main.yml b/roles/iperf3/tasks/main.yml index db2d34d8e..b722d0843 100644 --- a/roles/iperf3/tasks/main.yml +++ b/roles/iperf3/tasks/main.yml @@ -1,30 +1,4 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: Start Server(s) @@ -32,21 +6,18 @@ definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" with_sequence: start=1 count={{ workload_args.pairs|default('1')|int }} - - name: Update state to Starting Server - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Server" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Server" - when: resource_state.resources[0].status.state == "Starting" + when: benchmark_state.resources[0].status.state == "Building" - block: - name: Wait for pods to be running.... - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -54,22 +25,19 @@ - app = iperf3-bench-server-{{ trunc_uuid }} register: server_pods - - name: Update state to Starting Clients - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Clients" when: "workload_args.pairs|default('1')|int == (server_pods | json_query('resources[].status.podIP')|length) and (server_pods | json_query('resources[].status.phase') is defined and workload_args.pairs|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length)" - when: resource_state.resources[0].status.state == "Starting Server" + when: benchmark_state.resources[0].status.state == "Starting Server" - block: - name: Get server pods - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -82,21 +50,18 @@ definition: "{{ lookup('template', 'client.yml.j2') | from_yaml }}" with_items: "{{ server_pods.resources }}" - - name: Update state to Waiting for Clients - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Waiting for Clients" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Waiting for Clients" - when: resource_state.resources[0].status.state == "Starting Clients" + when: benchmark_state.resources[0].status.state == "Starting Clients" - block: - name: Wait for pods to be running.... - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -104,36 +69,16 @@ - app = iperf3-bench-client-{{ trunc_uuid }} register: client_pods - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running when: client_pods.resources|length > 0 - when: ( workload_args.pairs|default('1')|int > 0 ) and resource_state.resources[0].status.state == "Waiting for Clients" + when: ( workload_args.pairs|default('1')|int > 0 ) and benchmark_state.resources[0].status.state == "Waiting for Clients" -- block: - - name: Waiting for Jobs to complete.... - k8s_facts: - kind: Job - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = iperf3-bench-client-{{ trunc_uuid }} - register: client_jobs - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "workload_args.pairs|default('1')|int == (client_jobs | json_query('resources[].status.succeeded')|length)" - - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/iperf3/templates/client.yml.j2 b/roles/iperf3/templates/client.yml.j2 index acae9610b..90e63dabc 100644 --- a/roles/iperf3/templates/client.yml.j2 +++ b/roles/iperf3/templates/client.yml.j2 @@ -5,11 +5,13 @@ metadata: name: 'iperf3-client-{{ item.status.podIP }}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: iperf3-bench-client-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: affinity: podAntiAffinity: diff --git a/roles/iperf3/templates/client_store.yml.j2 b/roles/iperf3/templates/client_store.yml.j2 index bd47b8a60..219df6c53 100644 --- a/roles/iperf3/templates/client_store.yml.j2 +++ b/roles/iperf3/templates/client_store.yml.j2 @@ -10,6 +10,7 @@ spec: metadata: labels: app: iperf3-bench-client-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: affinity: podAntiAffinity: diff --git a/roles/iperf3/templates/server.yml.j2 b/roles/iperf3/templates/server.yml.j2 index 77558c253..95f6e5128 100644 --- a/roles/iperf3/templates/server.yml.j2 +++ b/roles/iperf3/templates/server.yml.j2 @@ -1,47 +1,44 @@ --- -kind: Job -apiVersion: batch/v1 +kind: Pod +apiVersion: v1 metadata: name: 'iperf3-server-{{ item }}-{{ trunc_uuid }}' namespace: "{{ operator_namespace }}" + labels: + benchmark-uuid: {{ uuid }} + app: iperf3-bench-server-{{ trunc_uuid }} spec: - ttlSecondsAfterFinished: 600 - template: - metadata: - labels: - app: iperf3-bench-server-{{ trunc_uuid }} - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - iperf3-bench-client-{{ trunc_uuid }} - topologyKey: kubernetes.io/hostname + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - iperf3-bench-client-{{ trunc_uuid }} + topologyKey: kubernetes.io/hostname {% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" + runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} {% if workload_args.hostnetwork is sameas true %} - hostNetwork: true - serviceAccountName: benchmark-operator + hostNetwork: true + serviceAccountName: benchmark-operator {% endif %} - containers: - - name: benchmark - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/iperf3:latest') }} - imagePullPolicy: Always - command: ["/bin/sh", "-c"] - args: - - "iperf3 -s - {% if workload_args.port is defined %} -p {{ workload_args.port }} {% endif %} - {% if workload_args.extra_options_server is defined %} {{ workload_args.extra_options_server }} {% endif %}" - restartPolicy: OnFailure + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/iperf3:latest') }} + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + args: + - "iperf3 -s + {% if workload_args.port is defined %} -p {{ workload_args.port }} {% endif %} + {% if workload_args.extra_options_server is defined %} {{ workload_args.extra_options_server }} {% endif %}" + restartPolicy: OnFailure {% if workload_args.pin_server is defined %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_server }}' + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_server }}' {% endif %} -{% include "metadata.yml.j2" %} +{% include "metadata_pod.yml.j2" %} diff --git a/roles/kube-burner/tasks/main.yml b/roles/kube-burner/tasks/main.yml index e09238f93..0cf7e0eab 100644 --- a/roles/kube-burner/tasks/main.yml +++ b/roles/kube-burner/tasks/main.yml @@ -1,32 +1,4 @@ --- - -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - - block: - name: Create metrics profile for remote configuration job @@ -209,46 +181,15 @@ k8s: definition: "{{ lookup('template', 'kube-burner.yml.j2') | from_yaml }}" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - when: resource_state.resources[0].status.state == "Building" - -- block: + when: benchmark_state.resources[0].status.state == "Building" - - name: Get job status - k8s_info: - kind: Job - api_version: v1 - namespace: '{{ operator_namespace }}' - name: kube-burner-{{ trunc_uuid }} - register: job_state - - - name: Set complete state to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: job_state.resources[0].status.succeeded is defined and (job_state.resources[0].status.succeeded | int) > 0 - - - name: Set failed state to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Failed - complete: true - when: job_state.resources[0].status.failed is defined and (job_state.resources[0].status.failed | int) > 0 - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/kube-burner/templates/kube-burner.yml.j2 b/roles/kube-burner/templates/kube-burner.yml.j2 index c3db198f9..f81bda7fc 100644 --- a/roles/kube-burner/templates/kube-burner.yml.j2 +++ b/roles/kube-burner/templates/kube-burner.yml.j2 @@ -11,6 +11,7 @@ spec: metadata: labels: app: kube-burner-benchmark-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/log_generator/tasks/main.yml b/roles/log_generator/tasks/main.yml index 17eac1060..bbf93d717 100644 --- a/roles/log_generator/tasks/main.yml +++ b/roles/log_generator/tasks/main.yml @@ -1,30 +1,4 @@ --- -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Log Generator Pods" - complete: false - when: benchmark_state.resources[0].status.state is not defined - -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state - - block: - name: Create log generator pods @@ -32,58 +6,15 @@ state: present definition: "{{ lookup('template', 'log_generator.yml') }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Building Pods" - - when: benchmark_state.resources[0].status.state == "Starting Log Generator Pods" - -- block: - - - name: Get server pods - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = log-generator-{{ trunc_uuid }} - register: log_pods - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Log Generator Running" - when: "workload_args.pod_count|default('1')|int == log_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" - - when: benchmark_state.resources[0].status.state == "Building Pods" - -- block: - - - name: Get server pods - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = log-generator-{{ trunc_uuid }} - register: log_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "workload_args.pod_count|default('1')|int == (log_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: benchmark_state.resources[0].status.state == "Building" - when: benchmark_state.resources[0].status.state == "Log Generator Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/log_generator/templates/log_generator.yml b/roles/log_generator/templates/log_generator.yml index e622be8ac..c180911e0 100644 --- a/roles/log_generator/templates/log_generator.yml +++ b/roles/log_generator/templates/log_generator.yml @@ -6,10 +6,13 @@ metadata: namespace: '{{ operator_namespace }}' spec: parallelism: {{ workload_args.pod_count | default(1) | int }} + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: log-generator-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/oslat/tasks/main.yml b/roles/oslat/tasks/main.yml index 03afb5c88..64c20dafc 100644 --- a/roles/oslat/tasks/main.yml +++ b/roles/oslat/tasks/main.yml @@ -1,31 +1,4 @@ --- - -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state if changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: Add oslat test scripts @@ -40,64 +13,20 @@ oslat.sh: "{{ lookup('file', 'oslat.sh') }}" functions.sh: "{{ lookup('file', 'functions.sh') }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: ConfigMaps Created - complete: false - - when: resource_state.resources[0].status.state == "Building" - -- block: - name: Start cyclitest job k8s: state: present definition: "{{ lookup('template', 'oslatjob.yaml') | from_yaml }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Benchmark Running - complete: false - - when: resource_state.resources[0].status.state == "ConfigMaps Created" - -- block: - - - name: Wait for benchmark to complete - k8s_facts: - kind: Job - api_version: batch/v1 - name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' - namespace: "{{ operator_namespace }}" - register: oslat_status - - - name: Set Complete Status to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Benchmark Complete - complete: true - when: oslat_status.resources[0].status.succeeded is defined and (oslat_status.resources[0].status.succeeded | int) > 0 + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - name: Set failed state to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Failed - complete: true - when: oslat_status.resources[0].status.failed is defined and (oslat_status.resources[0].status.failed | int) > 0 + when: benchmark_state.resources[0].status.state == "Building" - when: resource_state.resources[0].status.state == "Benchmark Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index 348d35883..e98b31682 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -4,15 +4,17 @@ metadata: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: oslat-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} - ttlSecondsAfterFinished: 600 containers: - name: oslat image: {{ workload_args.image | default('quay.io/cloud-bulldozer/oslat:latest') }} @@ -44,9 +46,7 @@ spec: value: "{{ clustername }}" {% if elasticsearch is defined %} - name: es - value: "{{ elasticsearch.server }}" - - name: es_port - value: "{{ elasticsearch.port }}" + value: "{{ elasticsearch.url}}" - name: es_index value: "{{ elasticsearch.index_name | default("ripsaw-oslat") }}" - name: parallel diff --git a/roles/pgbench/tasks/check_clients.yml b/roles/pgbench/tasks/check_clients.yml index 102b03690..7d87b7028 100644 --- a/roles/pgbench/tasks/check_clients.yml +++ b/roles/pgbench/tasks/check_clients.yml @@ -3,12 +3,9 @@ shell: "redis-cli --raw llen pgb_client_ready" register: client_ready -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Run Workload" - complete: false +- include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Run Workload" when: client_ready.stdout|int == dbnum|int diff --git a/roles/pgbench/tasks/init.yml b/roles/pgbench/tasks/init.yml index f3e144f7c..28198f89c 100644 --- a/roles/pgbench/tasks/init.yml +++ b/roles/pgbench/tasks/init.yml @@ -2,11 +2,8 @@ - name: Initialize dbnum_item in redis command: "redis-cli set dbnum_item 0" -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Prep Workload" - complete: false +- include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Prep Workload" diff --git a/roles/pgbench/tasks/main.yml b/roles/pgbench/tasks/main.yml index ecf657537..80a89eae2 100644 --- a/roles/pgbench/tasks/main.yml +++ b/roles/pgbench/tasks/main.yml @@ -1,36 +1,10 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Init" - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - name: Generate list of database counts include_tasks: generate_num_databases.yml - name: Init benchmark environment include_tasks: init.yml - when: resource_state.resources[0].status.state == "Init" + when: benchmark_state.resources[0].status.state == "Building" - name: Get dbnum_item from redis command: "redis-cli get dbnum_item" @@ -46,15 +20,15 @@ - name: Prep pgbench workload include_tasks: prep_workload.yml - when: resource_state.resources[0].status.state == "Prep Workload" + when: benchmark_state.resources[0].status.state == "Prep Workload" - name: Check pgbench clients include_tasks: check_clients.yml - when: resource_state.resources[0].status.state == "Check Clients" + when: benchmark_state.resources[0].status.state == "Check Clients" - name: Run pgbench workloads include_tasks: run_workload.yml - when: resource_state.resources[0].status.state == "Run Workload" + when: benchmark_state.resources[0].status.state == "Run Workload" #TODO: #- database passwords are currently provided in plain text in the CR diff --git a/roles/pgbench/tasks/prep_workload.yml b/roles/pgbench/tasks/prep_workload.yml index fa4f17ae8..94a2bf679 100644 --- a/roles/pgbench/tasks/prep_workload.yml +++ b/roles/pgbench/tasks/prep_workload.yml @@ -11,11 +11,8 @@ with_indexed_items: "{{ workload_args.databases }}" when: item.0|int in range(dbnum|int) -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Check Clients" - complete: false +- include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Check Clients" diff --git a/roles/pgbench/tasks/run_workload.yml b/roles/pgbench/tasks/run_workload.yml index 3d1d365c7..7b183a69c 100644 --- a/roles/pgbench/tasks/run_workload.yml +++ b/roles/pgbench/tasks/run_workload.yml @@ -15,14 +15,11 @@ command: "redis-cli incr dbnum_item" when: "'Succeeded' in (pgbench_pods | json_query('resources[].status.phase'))" -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Prep Workload" - complete: false +- include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Prep Workload" when: "'Succeeded' in (pgbench_pods | json_query('resources[].status.phase')) and dbnum_item|int < (num_databases|length - 1)" - operator_sdk.util.k8s_status: diff --git a/roles/pgbench/templates/workload.yml.j2 b/roles/pgbench/templates/workload.yml.j2 index 30854d300..540feb5f3 100644 --- a/roles/pgbench/templates/workload.yml.j2 +++ b/roles/pgbench/templates/workload.yml.j2 @@ -5,11 +5,13 @@ metadata: name: 'pgbench-{{ dbnum }}-dbs-client-{{ item.0|int + 1 }}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: pgbench-client-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/scale_openshift/tasks/main.yml b/roles/scale_openshift/tasks/main.yml index e2ade80e9..cabb1fc2e 100644 --- a/roles/scale_openshift/tasks/main.yml +++ b/roles/scale_openshift/tasks/main.yml @@ -1,30 +1,4 @@ --- -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Scale Pod" - complete: false - when: benchmark_state.resources[0].status.state is not defined - -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state - - block: - name: Create scale pod @@ -32,35 +6,15 @@ state: present definition: "{{ lookup('template', 'scale.yml') }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Scaling Cluster" - - when: benchmark_state.resources[0].status.state == "Starting Scale Pod" - -- block: - - - name: Waiting for scaling pod to complete - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = scale-{{ trunc_uuid }} - register: scale_pod + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "scale_pod|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length > 0" + when: benchmark_state.resources[0].status.state == "Building" - when: benchmark_state.resources[0].status.state == "Scaling Cluster" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/scale_openshift/templates/scale.yml b/roles/scale_openshift/templates/scale.yml index 64f06460b..c9ea17575 100755 --- a/roles/scale_openshift/templates/scale.yml +++ b/roles/scale_openshift/templates/scale.yml @@ -5,11 +5,13 @@ metadata: name: 'scale-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: scale-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/servicemesh/tasks/main.yml b/roles/servicemesh/tasks/main.yml index 71371e653..da2099e4a 100644 --- a/roles/servicemesh/tasks/main.yml +++ b/roles/servicemesh/tasks/main.yml @@ -1,12 +1,4 @@ --- -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - name: Define variables set_fact: servicemesh: "{{ servicemesh_defaults | combine(workload_args, recursive=True) }}" @@ -17,9 +9,9 @@ kind: Benchmark name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" - uid: "{{ resource_state.resources[0].metadata.uid }}" + uid: "{{ benchmark_state.resources[0].metadata.uid }}" -- when: resource_state.resources[0].status.state is not defined +- when: benchmark_state.resources[0].status.state is not defined block: - name: Create control plane namespace k8s: @@ -65,26 +57,13 @@ spec: version: "{{ servicemesh.hyperfoil_version }}" - - name: Set state to Init - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Init" - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Init -# Note: register is invoked even if the task is skipped; therefore we'll update the state non-conditionally -- name: Update current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- when: resource_state.resources[0].status.state == "Init" +- when: benchmark_state.resources[0].status.state == "Init" block: - name: Check Hyperfoil-status k8s_info: @@ -105,24 +84,12 @@ - set_fact: smcp_ready: "{{ smcp_state | json_query('resources[0].status.conditions[]') | selectattr('type','match','Ready') | selectattr('status', 'match', 'True') | list | length > 0 }}" - - name: Progress to deploy + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Deploy when: hf_state.resources[0].status.status == "Ready" and smcp_ready - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Deploy" - complete: false - -- name: Update current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - name: Check ingress route k8s_info: @@ -138,7 +105,7 @@ #TODO: maybe use trunc_uuid in the hostname? app_route: "{{ ansible_operator_meta.name }}-app-{{ trunc_uuid}}{{ ingress_route.resources[0].status.ingress[0].host | regex_replace('^[^.]*', '')}}" -- when: resource_state.resources[0].status.state == "Deploy" +- when: benchmark_state.resources[0].status.state == "Deploy" block: - name: Generate certificates command: openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /tmp/app.key -out /tmp/app.crt -subj "/CN={{ app_route }}/O=app" @@ -195,26 +162,14 @@ ready_containers: "{{ pods.resources | map(attribute='status.containerStatuses') | flatten | selectattr('ready', 'equalto', true) | list | length }}" expected_containers: "{{ 2 * servicemesh.deployments * servicemesh.pods_per_deployment }}" - - name: Progress to ready + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Ready when: ready_containers == expected_containers - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Ready" - complete: false - -- name: Update current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state -- when: resource_state.resources[0].status.state == "Ready" +- when: benchmark_state.resources[0].status.state == "Ready" block: - name: "Expand test template into config map" k8s: @@ -244,65 +199,20 @@ k8s: definition: "{{ lookup('template', 'job.yaml.j2') | from_yaml }}" - - name: Progress to running - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running" - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running -- name: Update current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- when: resource_state.resources[0].status.state == "Running" +- when: benchmark_state.resources[0].status.state == "Running" block: - - name: Get job status - k8s_info: - api_version: v1 - kind: Job - namespace: '{{ operator_namespace }}' - name: "{{ ansible_operator_meta.name }}-{{ trunc_uuid }}" - register: job_state - - - name: Complete benchmark - when: job_state.resources[0].status.succeeded is defined and (job_state.resources[0].status.succeeded | int) > 0 - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Complete" - complete: true - - - name: Fail benchmark - when: job_state.resources[0].status.failed is defined and (job_state.resources[0].status.failed | int) > 0 - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Failed" - complete: true - -- name: Update current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state + - name: Set completed state + include_role: + name: benchmark_state + tasks_from: completed.yml -- when: resource_state.resources[0].status.state == "Complete" and cleanup +- when: benchmark_state.resources[0].status.state == "Complete" and cleanup block: - name: Delete Hyperfoil resource k8s: diff --git a/roles/smallfile/tasks/main.yml b/roles/smallfile/tasks/main.yml index 01ca474e2..5fba6ada7 100644 --- a/roles/smallfile/tasks/main.yml +++ b/roles/smallfile/tasks/main.yml @@ -1,31 +1,4 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- name: Update current state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - set_fact: @@ -90,7 +63,7 @@ command: ["/bin/sh", "-c"] workingDir: /root/smallfile-master/ args: - - python /tmp/publisher {{bo.resources[0].status.podIP}} {{workload_args.clients|default('1')|int}} + - python /tmp/publisher {{bo.resources[0].status.podIP}} {{workload_args.clients|default('1')|int}} {{trunc_uuid}} volumeMounts: - name: config-volume mountPath: "/tmp" @@ -108,36 +81,15 @@ with_sequence: start=1 count={{ workload_args.clients|default('1')|int }} when: workload_args.clients|default('1')|int > 0 - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false - - when: resource_state.resources[0].status.state == "Starting" - -- block: - - - name: Waiting for pods to complete.... - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = smallfile-benchmark-{{ trunc_uuid }} - register: client_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "workload_args.clients|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: benchmark_state.resources[0].status.state == "Building" - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/smallfile/templates/publisher.py.j2 b/roles/smallfile/templates/publisher.py.j2 index 23f41dac1..b393cfc72 100644 --- a/roles/smallfile/templates/publisher.py.j2 +++ b/roles/smallfile/templates/publisher.py.j2 @@ -8,16 +8,17 @@ import sys redis_host = sys.argv[1] number_of_pods = sys.argv[2] +uuid = sys.argv[3] try: r = redis.StrictRedis(host=redis_host, port=6379) count = 0 while int(count) != int(number_of_pods): - count = r.execute_command("PUBSUB NUMSUB smallfile")[1] + count = r.execute_command(f"PUBSUB NUMSUB smallfile-{uuid}")[1] print(count, file=sys.stdout) time.sleep(1) print("All Pods are ready to run. Triggering the run ........\(^_^)/", file=sys.stdout) - r.publish("smallfile","run") + r.publish(f"smallfile-{uuid}","run") except Exception as e: print("******Exception Occured*******") print(str(e)) diff --git a/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 b/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 index cfc766eca..5540a5fcb 100644 --- a/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 +++ b/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 @@ -15,11 +15,11 @@ files: {{ workload_args.files }} {% endif %} {% if workload_args.fsync is defined %} -fsync: {{ workload_args.fsync }} +fsync: y {% endif %} {% if workload_args.response_times is defined %} -response-times: {{ workload_args.response_times }} +response-times: true {% endif %} {% if workload_args.network_sync_dir is defined %} @@ -79,21 +79,17 @@ verbose: {{workload_args.verbose}} {% endif %} {% if workload_args.permute_host_dirs is defined %} -permute-host-dirs: {{ workload_args.permute_host_dirs }} +permute-host-dirs: true {% endif %} {% if workload_args.record_ctime_size is defined %} -record-ctime-size: {{ workload_args.record_ctime_size }} +record-ctime-size: true {% endif %} {% if workload_args.verify_read is defined %} -verify-read: {{ workload_args.verify_read }} +verify-read: true {% endif %} {% if workload_args.incompressible is defined %} -incompressible: {{ workload_args.incompressible }} -{% endif %} - -{% if workload_args.cleanup_delay_usec_per_file is defined %} -cleanup-delay-usec-per-file: {{workload_args.cleanup_delay_usec_per_file}} +incompressible: true {% endif %} diff --git a/roles/smallfile/templates/subscriber.py.j2 b/roles/smallfile/templates/subscriber.py.j2 index b69babf06..08b7c7dba 100644 --- a/roles/smallfile/templates/subscriber.py.j2 +++ b/roles/smallfile/templates/subscriber.py.j2 @@ -6,12 +6,13 @@ import sys import redis redis_host = sys.argv[1] +uuid = sys.argv[2] try: r = redis.StrictRedis(host=redis_host, port=6379) p = r.pubsub() - p.subscribe('smallfile') + p.subscribe(f'smallfile-{uuid}') STATE = True while STATE: diff --git a/roles/smallfile/templates/workload_job.yml.j2 b/roles/smallfile/templates/workload_job.yml.j2 index 8d28b82ec..4b983c6a4 100644 --- a/roles/smallfile/templates/workload_job.yml.j2 +++ b/roles/smallfile/templates/workload_job.yml.j2 @@ -5,12 +5,13 @@ metadata: name: "smallfile-client-{{ item }}-benchmark-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: smallfile-benchmark-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -86,7 +87,7 @@ spec: - name: clients value: "{{ workload_args.clients|default('1')|int }}" args: - - python /tmp/smallfile/subscriber {{bo.resources[0].status.podIP}}; + - python /tmp/smallfile/subscriber {{bo.resources[0].status.podIP}} {{trunc_uuid}}; export TMPDIR=/tmp arr=$(python <<< "print(','.join({{workload_args.operation}}))"); rm -rf {{smallfile_path}}/RESULTS ; diff --git a/roles/stressng/tasks/main.yaml b/roles/stressng/tasks/main.yaml index 3dfeb0d5c..bb1e72f66 100644 --- a/roles/stressng/tasks/main.yaml +++ b/roles/stressng/tasks/main.yaml @@ -1,30 +1,4 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: template stressng config file @@ -38,20 +12,17 @@ data: jobfile: "{{ lookup ('template', 'jobfile.j2') }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: ConfigMaps Created - complete: false - - when: resource_state.resources[0].status.state == "Building" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: ConfigMaps Created + when: benchmark_state.resources[0].status.state == "Building" - block: + - name: set complete to false - command: "redis-cli set complete false" + command: "redis-cli set complete-{{ trunc_uuid }} false" - name: start stressng workload job k8s: @@ -65,42 +36,28 @@ register: jobs when: resource_kind == "vm" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Benchmark running - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - when: resource_state.resources[0].status.state == "ConfigMaps Created" + when: benchmark_state.resources[0].status.state == "ConfigMaps Created" - block: - - block: - - name: wait for benchmark job to finish - k8s_facts: - kind: Job - api_version: batch/v1 - name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' - namespace: "{{ operator_namespace }}" - register: stressng_workload_pod - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: stressng_workload_pod | json_query('resources[].status.succeeded') + - include_role: + name: benchmark_state + tasks_from: completed.yml when: resource_kind == "pod" + - block: + - name: get complete - command: "redis-cli get complete" + command: "redis-cli get complete-{{ trunc_uuid }}" register: complete_status + - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -111,4 +68,4 @@ complete: true when: complete_status.stdout == "true" when: resource_kind == "vm" - when: resource_state.resources[0].status.state == "Benchmark running" + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/stressng/templates/stressng_workload.yml.j2 b/roles/stressng/templates/stressng_workload.yml.j2 index dec80fa1f..c65b651af 100644 --- a/roles/stressng/templates/stressng_workload.yml.j2 +++ b/roles/stressng/templates/stressng_workload.yml.j2 @@ -1,18 +1,19 @@ --- apiVersion: batch/v1 -kind: "job" +kind: Job metadata: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: parallelism: {{ workload_args.instances }} - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: stressng_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} benchmark-operator-workload: stressng spec: {% if workload_args.pin is sameas true %} diff --git a/roles/stressng/templates/stressng_workload_vm.yml.j2 b/roles/stressng/templates/stressng_workload_vm.yml.j2 index b9d1231a2..bc59dcad0 100644 --- a/roles/stressng/templates/stressng_workload_vm.yml.j2 +++ b/roles/stressng/templates/stressng_workload_vm.yml.j2 @@ -7,7 +7,7 @@ metadata: labels: app: stressng_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} benchmark-operator-workload: stressng spec: domain: diff --git a/roles/sysbench/tasks/main.yml b/roles/sysbench/tasks/main.yml index 0d270ee89..559159eb2 100644 --- a/roles/sysbench/tasks/main.yml +++ b/roles/sysbench/tasks/main.yml @@ -1,31 +1,4 @@ --- - -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: template sysbench script k8s: @@ -50,36 +23,15 @@ definition: "{{ lookup('template', 'workload_vm.yml') | from_yaml }}" when: workload_args.kind is defined and workload_args.kind == "vm" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false - - when: resource_state.resources[0].status.state == "Starting" - -- block: - - - name: Waiting for pods to complete.... - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = sysbench-{{ trunc_uuid }} - register: client_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "1 == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: benchmark_state.resources[0].status.state == "Building" - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/sysbench/templates/workload.yml b/roles/sysbench/templates/workload.yml index 404310d84..10cd834a6 100644 --- a/roles/sysbench/templates/workload.yml +++ b/roles/sysbench/templates/workload.yml @@ -6,11 +6,13 @@ metadata: name: "sysbench-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: sysbench-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/sysbench/templates/workload_vm.yml b/roles/sysbench/templates/workload_vm.yml index 148e49c97..e3416a999 100644 --- a/roles/sysbench/templates/workload_vm.yml +++ b/roles/sysbench/templates/workload_vm.yml @@ -7,6 +7,7 @@ metadata: namespace: "{{ operator_namespace }}" labels: kubevirt-vm: vm-sysbench-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: domain: devices: diff --git a/roles/system-metrics/tasks/main.yml b/roles/system-metrics/tasks/main.yml index 09af09a6c..fd642228f 100644 --- a/roles/system-metrics/tasks/main.yml +++ b/roles/system-metrics/tasks/main.yml @@ -35,7 +35,7 @@ status: system_metrics: Collecting - when: cr_state.resources[0].status.system_metrics == "Not collected" + when: benchmark_state.resources[0].status.system_metrics == "Not collected" - name: Get job status k8s_info: diff --git a/roles/system-metrics/templates/kube-burner.yml.j2 b/roles/system-metrics/templates/kube-burner.yml.j2 index 03f8cb8e7..665f10f99 100644 --- a/roles/system-metrics/templates/kube-burner.yml.j2 +++ b/roles/system-metrics/templates/kube-burner.yml.j2 @@ -11,6 +11,7 @@ spec: metadata: labels: app: system-metrics-collector-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -29,7 +30,7 @@ spec: -c index.yml -u {{ system_metrics.prom_url }} -t {{ system_metrics.prom_token }} - --start={{ (cr_state.resources[0].metadata.creationTimestamp|to_datetime('%Y-%m-%dT%H:%M:%SZ')).strftime('%s') }} + --start={{ (benchmark_state.resources[0].metadata.creationTimestamp|to_datetime('%Y-%m-%dT%H:%M:%SZ')).strftime('%s') }} -m {{ system_metrics.metrics_profile|default("node-metrics.yml") }} --step={{ system_metrics.step }} volumeMounts: diff --git a/roles/testpmd/tasks/main.yml b/roles/testpmd/tasks/main.yml index f1d093c66..e2d38b8aa 100644 --- a/roles/testpmd/tasks/main.yml +++ b/roles/testpmd/tasks/main.yml @@ -1,30 +1,4 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - name: "Parse testpmd network" include_tasks: 10_network_parser.yml loop: "{{ workload_args.networks.testpmd }}" @@ -51,16 +25,13 @@ k8s: definition: "{{ lookup('template', 'testpmd.yml.j2') | from_yaml }}" - - name: Update state to Starting Server - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting TestPMD" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting TestPMD" - when: resource_state.resources[0].status.state == "Starting" + when: benchmark_state.resources[0].status.state == "Building" - block: @@ -73,18 +44,15 @@ - app = testpmd-application-pod-{{ trunc_uuid }} register: application_pods - - name: Update state to Running - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting TRex" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting TRex" when: - "(application_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) == 1" - when: resource_state.resources[0].status.state == "Starting TestPMD" + when: benchmark_state.resources[0].status.state == "Starting TestPMD" - name: Set trex network fact set_fact: @@ -110,16 +78,13 @@ definition: "{{ lookup('template', 'trex.yml.j2') | from_yaml }}" with_items: "{{ application_pods.resources }}" - - name: Update state to Waiting for TRex - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Waiting for TRex" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Waiting for TRex" - when: resource_state.resources[0].status.state == "Starting TRex" + when: benchmark_state.resources[0].status.state == "Starting TRex" - block: @@ -132,36 +97,16 @@ - app = trex-traffic-gen-pod-{{ trunc_uuid }} register: trex_pods - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running when: trex_pods.resources|length > 0 - when: resource_state.resources[0].status.state == "Waiting for TRex" - -- block: - - name: Waiting for Jobs to complete.... - k8s_facts: - kind: Job - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = trex-traffic-gen-pod-{{ trunc_uuid }} - register: trex_pods - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "(trex_pods | json_query('resources[].status.succeeded')|length) == 1" - when: resource_state.resources[0].status.state == "Running" + when: benchmark_state.resources[0].status.state == "Waiting for TRex" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/testpmd/templates/testpmd.yml.j2 b/roles/testpmd/templates/testpmd.yml.j2 index bfc9e38bd..e3f7a2bd4 100644 --- a/roles/testpmd/templates/testpmd.yml.j2 +++ b/roles/testpmd/templates/testpmd.yml.j2 @@ -10,6 +10,7 @@ spec: metadata: labels: app: "testpmd-application-pod-{{ trunc_uuid }}" + benchmark-uuid: {{ uuid }} annotations: k8s.v1.cni.cncf.io/networks: '[ {% set i = namespace(value=0) %} diff --git a/roles/testpmd/templates/trex.yml.j2 b/roles/testpmd/templates/trex.yml.j2 index fa30a941d..309896f42 100644 --- a/roles/testpmd/templates/trex.yml.j2 +++ b/roles/testpmd/templates/trex.yml.j2 @@ -10,6 +10,7 @@ spec: metadata: labels: app: "trex-traffic-gen-pod-{{ trunc_uuid }}" + benchmark-uuid: {{ uuid }} annotations: k8s.v1.cni.cncf.io/networks: '[ {% set i = namespace(value=0) %} diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index 435a9b6b9..43dbdc4eb 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -3,41 +3,41 @@ - include_tasks: setup.yml - include_tasks: start_server.yml - when: resource_state.resources[0].status.state == "Building" + when: benchmark_state.resources[0].status.state == "Building" - block: - include_tasks: wait_server_ready.yml - when: resource_state.resources[0].status.state == "Starting Servers" + when: benchmark_state.resources[0].status.state == "Starting Servers" - include_tasks: start_client.yml - when: resource_state.resources[0].status.state == "Starting Clients" + when: benchmark_state.resources[0].status.state == "Starting Clients" - include_tasks: wait_client_ready.yml - when: resource_state.resources[0].status.state == "Waiting for Clients" + when: benchmark_state.resources[0].status.state == "Waiting for Clients" # LOOP BEGIN # This loop iterates density_range[] and node_range[] for "scale" mode - include_tasks: run_a_set.yml - when: resource_state.resources[0].status.state == "Clients Running" + when: benchmark_state.resources[0].status.state == "Clients Running" - include_tasks: wait_set_done.yml - when: resource_state.resources[0].status.state == "Set Running" + when: benchmark_state.resources[0].status.state == "Set Running" - include_tasks: next_set.yml - when: resource_state.resources[0].status.state == "Run Next Set" + when: benchmark_state.resources[0].status.state == "Run Next Set" # will loop back to "Client Running" state, or FALLTHRU to "Running" # state below and finish # LOOP END - include_tasks: wait_client_done.yml - when: resource_state.resources[0].status.state == "Running" + when: benchmark_state.resources[0].status.state == "Running" - include_tasks: cleanup.yml - when: resource_state.resources[0].status.state == "Cleanup" + when: benchmark_state.resources[0].status.state == "Cleanup" when: resource_kind == "pod" @@ -48,22 +48,22 @@ - block: - include_tasks: wait_server_ready.yml - when: resource_state.resources[0].status.state == "Starting Servers" + when: benchmark_state.resources[0].status.state == "Starting Servers" - include_tasks: start_client.yml - when: resource_state.resources[0].status.state == "Starting Clients" + when: benchmark_state.resources[0].status.state == "Starting Clients" - include_tasks: wait_client_ready.yml - when: resource_state.resources[0].status.state == "Waiting for Clients" + when: benchmark_state.resources[0].status.state == "Waiting for Clients" - include_tasks: send_client_run_signal.yml - when: resource_state.resources[0].status.state == "Clients Running" + when: benchmark_state.resources[0].status.state == "Clients Running" - include_tasks: wait_client_done.yml - when: resource_state.resources[0].status.state == "Running" + when: benchmark_state.resources[0].status.state == "Running" - include_tasks: cleanup.yml - when: resource_state.resources[0].status.state == "Cleanup" + when: benchmark_state.resources[0].status.state == "Cleanup" when: resource_kind == "vm" diff --git a/roles/uperf/tasks/next_set.yml b/roles/uperf/tasks/next_set.yml index 3f3503d62..c8c74f07b 100644 --- a/roles/uperf/tasks/next_set.yml +++ b/roles/uperf/tasks/next_set.yml @@ -15,8 +15,8 @@ all_run_done: False inc: "{{workload_args.step_size|default('add1')}}" amount: 0 - pod_idx: "{{resource_state.resources[0].status.pod_idx|int}}" - node_idx: "{{resource_state.resources[0].status.node_idx|int}}" + pod_idx: "{{benchmark_state.resources[0].status.pod_idx|int}}" + node_idx: "{{benchmark_state.resources[0].status.node_idx|int}}" - name: Extract add amount set_fact: @@ -44,7 +44,7 @@ - name: Check node loop for ending condition set_fact: all_run_done: True - when: "node_idx|int > resource_state.resources[0].status.node_hi_idx|int" + when: "node_idx|int > benchmark_state.resources[0].status.node_hi_idx|int" # # Reset pod_idx AFTER node_idx tasks above, else cond change @@ -52,9 +52,9 @@ # - name: Reset pod_idx to pod_low_idx set_fact: - pod_idx: "{{resource_state.resources[0].status.pod_low_idx}}" + pod_idx: "{{benchmark_state.resources[0].status.pod_low_idx}}" - when: "pod_idx|int > resource_state.resources[0].status.pod_hi_idx|int" + when: "pod_idx|int > benchmark_state.resources[0].status.pod_hi_idx|int" - block: # diff --git a/roles/uperf/tasks/run_a_set.yml b/roles/uperf/tasks/run_a_set.yml index 997a0271a..ca0e44f76 100644 --- a/roles/uperf/tasks/run_a_set.yml +++ b/roles/uperf/tasks/run_a_set.yml @@ -9,7 +9,7 @@ # - name: Signal group to run - command: "redis-cli set start-{{trunc_uuid}} true-{{resource_state.resources[0].status.node_idx|int}}-{{resource_state.resources[0].status.pod_idx|int}}" + command: "redis-cli set start-{{trunc_uuid}} true-{{benchmark_state.resources[0].status.node_idx|int}}-{{benchmark_state.resources[0].status.pod_idx|int}}" - name: Update state to "Set Running" operator_sdk.util.k8s_status: diff --git a/roles/uperf/tasks/send_client_run_signal.yml b/roles/uperf/tasks/send_client_run_signal.yml index 739b5ceef..ce833492a 100644 --- a/roles/uperf/tasks/send_client_run_signal.yml +++ b/roles/uperf/tasks/send_client_run_signal.yml @@ -14,5 +14,5 @@ status: state: "Running" - when: resource_state.resources[0].status.state == "Clients Running" + when: benchmark_state.resources[0].status.state == "Clients Running" diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml index 2270fe2b9..9ce9e30a9 100644 --- a/roles/uperf/tasks/setup.yml +++ b/roles/uperf/tasks/setup.yml @@ -1,31 +1,4 @@ --- - -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: # # This block is for scale mode where client and server pods are spreaded diff --git a/roles/uperf/tasks/wait_client_done.yml b/roles/uperf/tasks/wait_client_done.yml index 9c2f1e200..bc745a86d 100644 --- a/roles/uperf/tasks/wait_client_done.yml +++ b/roles/uperf/tasks/wait_client_done.yml @@ -24,7 +24,7 @@ - block: ### kind - name: get complete - command: "redis-cli get complete" + command: "redis-cli get complete-{{ trunc_uuid }}" register: complete_status - operator_sdk.util.k8s_status: diff --git a/roles/uperf/tasks/wait_client_ready.yml b/roles/uperf/tasks/wait_client_ready.yml index ad06014f0..a66ba3b89 100644 --- a/roles/uperf/tasks/wait_client_ready.yml +++ b/roles/uperf/tasks/wait_client_ready.yml @@ -28,7 +28,7 @@ ### kind - name: set complete to false - command: "redis-cli set complete false" + command: "redis-cli set complete-{{ trunc_uuid }} false" - name: Get count of clients ready command: "redis-cli get clients-{{ trunc_uuid }}" diff --git a/roles/uperf/tasks/wait_server_ready.yml b/roles/uperf/tasks/wait_server_ready.yml index dca5b4baf..3bb11cecb 100644 --- a/roles/uperf/tasks/wait_server_ready.yml +++ b/roles/uperf/tasks/wait_server_ready.yml @@ -46,7 +46,7 @@ when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - name: blocking client from running uperf - command: "redis-cli set start false" + command: "redis-cli set start-{{ trunc_uuid }} false" with_items: "{{ server_vms.resources }}" when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" diff --git a/roles/uperf/tasks/wait_set_done.yml b/roles/uperf/tasks/wait_set_done.yml index 0db9c67ed..f82d0ccbf 100644 --- a/roles/uperf/tasks/wait_set_done.yml +++ b/roles/uperf/tasks/wait_set_done.yml @@ -15,7 +15,7 @@ status: state: Run Next Set - when: "num_completion.stdout|int == ((resource_state.resources[0].status.node_idx|int +1) * (resource_state.resources[0].status.pod_idx|int +1))" + when: "num_completion.stdout|int == ((benchmark_state.resources[0].status.node_idx|int +1) * (benchmark_state.resources[0].status.pod_idx|int +1))" when: resource_kind == "pod" diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index f314d622c..dd9a4cd9f 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -15,7 +15,7 @@ items: template: metadata: labels: - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} benchmark-operator-workload: uperf benchmark-operator-role: server app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'') }}-{{ item }}-{{ trunc_uuid }} diff --git a/roles/uperf/templates/server_vm.yml.j2 b/roles/uperf/templates/server_vm.yml.j2 index b9051a93c..d006baf5b 100644 --- a/roles/uperf/templates/server_vm.yml.j2 +++ b/roles/uperf/templates/server_vm.yml.j2 @@ -5,7 +5,7 @@ metadata: name: 'uperf-server-{{ item }}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' labels: - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} benchmark-operator-workload: uperf benchmark-operator-role: server app : uperf-bench-server-{{ item }}-{{ trunc_uuid }} diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index a35b3dee6..9d008e062 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -10,7 +10,7 @@ items: name: uperf-service-{{ item }}-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' labels: - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} benchmark-operator-workload: uperf app: uperf-bench-server-{{ worker_node_list[ node_idx_item ] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 5b239e108..837ba334b 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -17,7 +17,7 @@ items: template: metadata: labels: - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} benchmark-operator-workload: uperf benchmark-operator-role: client app: uperf-bench-client-{{ trunc_uuid }} diff --git a/roles/uperf/templates/workload_vm.yml.j2 b/roles/uperf/templates/workload_vm.yml.j2 index a7bb4aac9..0053db676 100644 --- a/roles/uperf/templates/workload_vm.yml.j2 +++ b/roles/uperf/templates/workload_vm.yml.j2 @@ -5,7 +5,7 @@ metadata: name: 'uperf-client-{{item.status.interfaces[0].ipAddress}}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' labels: - benchmark-operator-uuid: {{ uuid }} + benchmark-uuid: {{ uuid }} benchmark-operator-workload: uperf benchmark-operator-role: client app: uperf-bench-client-{{ trunc_uuid }} diff --git a/roles/vegeta/tasks/main.yml b/roles/vegeta/tasks/main.yml index ff648fce2..c5fb5e559 100644 --- a/roles/vegeta/tasks/main.yml +++ b/roles/vegeta/tasks/main.yml @@ -1,32 +1,4 @@ --- - -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - - block: - name: Create targets configmap k8s: @@ -37,38 +9,15 @@ definition: "{{ lookup('template', 'vegeta.yml.j2') | from_yaml }}" loop: "{{ range(0, workload_args.clients|default(1)|int)|list }}" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - - when: resource_state.resources[0].status.state == "Building" - -- block: - - - name: Waiting for pods to complete.... - k8s_info: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = vegeta-benchmark-{{ trunc_uuid }} - field_selectors: - - status.phase=Succeeded - register: client_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: workload_args.clients|default(1)|int == client_pods.resources|length + when: benchmark_state.resources[0].status.state == "Building" - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/vegeta/templates/vegeta.yml.j2 b/roles/vegeta/templates/vegeta.yml.j2 index 49f7130eb..b1e0ecf7f 100644 --- a/roles/vegeta/templates/vegeta.yml.j2 +++ b/roles/vegeta/templates/vegeta.yml.j2 @@ -5,10 +5,13 @@ metadata: name: vegeta-{{ item }}-{{ trunc_uuid }} namespace: {{ operator_namespace }} spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: vegeta-benchmark-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/ycsb/tasks/main.yml b/roles/ycsb/tasks/main.yml index 6bc2882cd..cbbe1ac17 100644 --- a/roles/ycsb/tasks/main.yml +++ b/roles/ycsb/tasks/main.yml @@ -1,30 +1,4 @@ --- -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - register: resource_state - - block: - name: setting ycsb_workload_load var @@ -40,27 +14,21 @@ definition: "{{ lookup('template', 'ycsb_load.yaml') | from_yaml }}" when: workload_args.loaded is undefined or not workload_args.loaded - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running Load" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Running Load" when: workload_args.loaded is undefined or not workload_args.loaded|default('false') - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting First Workload" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting First Workload" when: workload_args.loaded is defined and workload_args.loaded|default('false') - when: resource_state.resources[0].status.state == "Building" + when: benchmark_state.resources[0].status.state == "Building" - block: @@ -78,27 +46,14 @@ - name = 'ycsb-load-{{ trunc_uuid }}' register: ycsb_load_pod - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting First Workload" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting First Workload" when: "ycsb_load_pod | json_query('resources[].status.succeeded')" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Complete" - when: workload_args.workloads is undefined and ycsb_load_pod | json_query('resources[].status.succeeded') - - when: resource_state.resources[0].status.state == "Running Load" + when: benchmark_state.resources[0].status.state == "Running Load" - block: @@ -108,16 +63,13 @@ - name: Add the first workload index to redis command: "redis-cli set {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current 0" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Workload" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Workload" - when: resource_state.resources[0].status.state == "Starting First Workload" + when: benchmark_state.resources[0].status.state == "Starting First Workload" - block: @@ -137,16 +89,13 @@ k8s: definition: "{{ lookup('template', 'ycsb_run.yaml') | from_yaml }}" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running Workload" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Running Workload" - when: resource_state.resources[0].status.state == "Starting Workload" + when: benchmark_state.resources[0].status.state == "Starting Workload" - block: @@ -168,17 +117,14 @@ - name = 'ycsb-run-{{ trunc_uuid }}' register: ycsb_bench - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Workload Complete" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Workload Complete" when: "ycsb_bench | json_query('resources[].status.succeeded')" - when: resource_state.resources[0].status.state == "Running Workload" + when: benchmark_state.resources[0].status.state == "Running Workload" - block: @@ -197,14 +143,11 @@ - name: Update current workload item in redis command: "redis-cli set {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current {{ new_workload_index }}" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Workload" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Workload" when: workload_list.stdout != new_workload_index - name: Update resource state @@ -214,7 +157,8 @@ name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: - state: "Complete" + state: Complete + complete: true when: workload_list.stdout == new_workload_index - when: resource_state.resources[0].status.state == "Workload Complete" + when: benchmark_state.resources[0].status.state == "Workload Complete" From ec4635c82ffb320467b4862dc1fa1bcf947e44a6 Mon Sep 17 00:00:00 2001 From: Keith Whitley Date: Mon, 23 Aug 2021 11:37:16 -0400 Subject: [PATCH 123/249] Chart fix (#651) * Pushing chart fix * Updaing install docs --- README.md | 4 +++- charts/benchmark-operator/templates/operator.yaml | 2 ++ charts/benchmark-operator/values.yaml | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9d299d054..3271620eb 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ make image-build image-push deploy IMG=$YOUR_IMAGE ## Installation (Helm) -Installing the benchmark-operator is easiest by using the helm chart and can be done with the following commands. This requires +Installing the benchmark-operator via Helm can be done with the following commands. This requires your machine to have Helm installed. [Install Helm](https://helm.sh/docs/intro/install/) > Note: If running on openshift you'll need to run this command before installing the chart. `oc adm policy -n benchmark-operator add-scc-to-user privileged -z benchmark-operator` @@ -35,6 +35,8 @@ your machine to have Helm installed. [Install Helm](https://helm.sh/docs/intro/i ```bash git clone https://github.com/cloud-bulldozer/benchmark-operator cd benchmark-operator/charts/benchmark-operator +kubectl create namespace benchmark-operator +oc adm policy -n benchmark-operator add-scc-to-user privileged -z benchmark-operator # Openshift Only helm install benchmark-operator . -n benchmark-operator --create-namespace ``` diff --git a/charts/benchmark-operator/templates/operator.yaml b/charts/benchmark-operator/templates/operator.yaml index 92f1a6a7e..73f1b6641 100644 --- a/charts/benchmark-operator/templates/operator.yaml +++ b/charts/benchmark-operator/templates/operator.yaml @@ -41,6 +41,8 @@ spec: env: - name: MASTER value: "true" + - name: ALLOW_EMPTY_PASSWORD + value: "yes" ports: - containerPort: 6379 resources: diff --git a/charts/benchmark-operator/values.yaml b/charts/benchmark-operator/values.yaml index 7b2a24779..bd315e3dc 100644 --- a/charts/benchmark-operator/values.yaml +++ b/charts/benchmark-operator/values.yaml @@ -20,12 +20,12 @@ operator: image: repository: quay.io/cloud-bulldozer/benchmark-operator pullPolicy: Always - tag: master + tag: latest redisImage: - repository: k8s.gcr.io/redis + repository: bitnami/redis pullPolicy: Always - tag: v1 + tag: latest resources: limits: From 1cc1216837459095a69781262d178918aa7c32fa Mon Sep 17 00:00:00 2001 From: Ben England Date: Tue, 24 Aug 2021 16:31:56 -0400 Subject: [PATCH 124/249] update cache dropping doc (#650) - get rid of a now-unnecessary step - explain how to set labels for kernel cache dropper --- docs/cache_dropping.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/cache_dropping.md b/docs/cache_dropping.md index e95093552..32fd2b2b0 100644 --- a/docs/cache_dropping.md +++ b/docs/cache_dropping.md @@ -37,6 +37,13 @@ For this to work, you must **label** the nodes that you want to drop kernel cach ``` # kubectl label node minikube kernel-cache-dropper=yes ``` + +or to label all worker nodes: + +``` +# kubectl label node kernel-cache-dropper=yes -l node-role.kubernetes.io/worker='' +``` + If you do not do this, benchmark-operator will reject the benchmark with an error to the effect that none of the cluster nodes have this label. This label controls where kernel cache is dropped. @@ -44,13 +51,7 @@ There will be a short delay after kernel cache is dropped in order to allow the some key cache contents before stressing it with a workload. This is controllable via the CACHE_RELOAD_TIME env. var. and defaults to 10 sec. -You must also execute this command before running a benchmark. - -``` -oc create -f ripsaw.l/resources/kernel-cache-drop-clusterrole.yaml -``` - -Lastly, the specific benchmark must support this feature. +The specific benchmark must support this feature. Benchmarks supported for kernel cache dropping at present are: - fio From e991ac4270f78e8cb7e206d6fe2079fcbe7b6c13 Mon Sep 17 00:00:00 2001 From: ebattat <73884315+ebattat@users.noreply.github.com> Date: Mon, 30 Aug 2021 12:00:24 +0300 Subject: [PATCH 125/249] fix vm complete state (#663) * fix vm complete state for stressng, hammerdb and uperf workloads --- roles/hammerdb/tasks/main.yaml | 28 +++++++++++++++++-- .../templates/db_mariadb_workload_vm.sh.j2 | 2 +- .../templates/db_mssql_workload_vm.sh.j2 | 2 +- .../templates/db_postgres_workload_vm.sh.j2 | 2 +- .../templates/stressng_workload_vm.yml.j2 | 2 +- roles/uperf/templates/configmap_script.yml.j2 | 6 ++-- 6 files changed, 32 insertions(+), 10 deletions(-) diff --git a/roles/hammerdb/tasks/main.yaml b/roles/hammerdb/tasks/main.yaml index 1b631ea38..91ebf3ceb 100644 --- a/roles/hammerdb/tasks/main.yaml +++ b/roles/hammerdb/tasks/main.yaml @@ -269,7 +269,29 @@ when: benchmark_state.resources[0].status.state == "Starting DB Workload" -- include_role: - name: benchmark_state - tasks_from: completed.yml +- block: + + - include_role: + name: benchmark_state + tasks_from: completed.yml + when: resource_kind == "pod" + + + - block: + + - name: get complete + command: "redis-cli get complete-{{ trunc_uuid }}" + register: complete_status + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + when: complete_status.stdout == "true" + when: resource_kind == "vm" when: benchmark_state.resources[0].status.state == "Running" + diff --git a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 index 6a34b9aec..b68c124d1 100644 --- a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 @@ -59,4 +59,4 @@ cd /hammer; ./hammerdbcli auto /creator/createdb.tcl; redis-cli set db-creation-{{trunc_uuid}} true; run_snafu --tool hammerdb -u {{ uuid }}; - +redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true; diff --git a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 index 409a2b75f..6ad224f4e 100644 --- a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 @@ -63,4 +63,4 @@ cd /hammer; ./hammerdbcli auto /creator/createdb.tcl; redis-cli set db-creation-{{trunc_uuid}} true; run_snafu --tool hammerdb -u {{ uuid }}; - +redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true; diff --git a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 index e8544ef1e..096200192 100644 --- a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 @@ -68,4 +68,4 @@ cd /hammer; ./hammerdbcli auto /creator/createdb.tcl; redis-cli set db-creation-{{trunc_uuid}} true; run_snafu --tool hammerdb -u {{ uuid }}; - +redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true; diff --git a/roles/stressng/templates/stressng_workload_vm.yml.j2 b/roles/stressng/templates/stressng_workload_vm.yml.j2 index bc59dcad0..fa0a57d98 100644 --- a/roles/stressng/templates/stressng_workload_vm.yml.j2 +++ b/roles/stressng/templates/stressng_workload_vm.yml.j2 @@ -82,7 +82,7 @@ spec: - dnf install -y git redis stress-ng - pip install git+https://github.com/cloud-bulldozer/benchmark-wrapper - run_snafu --tool stressng -j /tmp/stressng-test/jobfile -u {{ uuid }} - - redis-cli -h {{bo.resources[0].status.podIP}} set complete true + - redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{ trunc_uuid }} true name: cloudinitdisk - configMap: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" diff --git a/roles/uperf/templates/configmap_script.yml.j2 b/roles/uperf/templates/configmap_script.yml.j2 index bb5e44749..5abc31d1b 100644 --- a/roles/uperf/templates/configmap_script.yml.j2 +++ b/roles/uperf/templates/configmap_script.yml.j2 @@ -26,7 +26,7 @@ data: redis-cli -h {{bo.resources[0].status.podIP}} setnx clients-{{trunc_uuid}} 0 redis-cli -h {{bo.resources[0].status.podIP}} incr clients-{{trunc_uuid}} while true; do - BO_START=$(redis-cli -h {{bo.resources[0].status.podIP}} get start) + BO_START=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{ trunc_uuid }}) CLIENTS_READY=$(redis-cli -h {{bo.resources[0].status.podIP}} get clients-{{ trunc_uuid }}) SERVERS_READY=$(redis-cli -h {{bo.resources[0].status.podIP}} get {{ trunc_uuid }}) if [[ ("${BO_START}" =~ 'true') && ("${CLIENTS_READY}" == "${SERVERS_READY}") ]]; then @@ -57,6 +57,6 @@ data: fi; break; done; - redis-cli -h {{bo.resources[0].status.podIP}} set start false + redis-cli -h {{bo.resources[0].status.podIP}} set start-{{trunc_uuid}} false redis-cli -h {{bo.resources[0].status.podIP}} del clients-{{trunc_uuid}} - redis-cli -h {{bo.resources[0].status.podIP}} set complete true + redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true From 24839e5f3470d7cd73b12e0be1f924c0ed624cea Mon Sep 17 00:00:00 2001 From: Eduardo Minguez Perez Date: Mon, 30 Aug 2021 16:53:33 +0200 Subject: [PATCH 126/249] Able to run the sysbench test on a specific node --- config/samples/sysbench/cr.yaml | 1 + docs/sysbench.md | 4 ++++ roles/sysbench/templates/workload.yml | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/config/samples/sysbench/cr.yaml b/config/samples/sysbench/cr.yaml index f414d504f..5674ab6f1 100644 --- a/config/samples/sysbench/cr.yaml +++ b/config/samples/sysbench/cr.yaml @@ -10,6 +10,7 @@ spec: enabled: true #kind: vm # If you want to run this as a VM uncomment the above + #pin_node: "worker-0.mylab.example.com" tests: - name: cpu parameters: diff --git a/docs/sysbench.md b/docs/sysbench.md index 3575a2383..88d80d951 100644 --- a/docs/sysbench.md +++ b/docs/sysbench.md @@ -11,6 +11,9 @@ The optional argument **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The `pin_node` parameter allows to place the sysbench pod +on a specific node, using the `hostname` label. + Note: please ensure you set 0 for other workloads if editing the [cr.yaml](../config/samples/sysbench/cr.yaml) file otherwise @@ -29,6 +32,7 @@ spec: enabled: true #kind: vm # If you want to run this as a VM uncomment the above + #pin_node: "worker-0.mylab.example.com" tests: - name: cpu parameters: diff --git a/roles/sysbench/templates/workload.yml b/roles/sysbench/templates/workload.yml index 10cd834a6..a1ec6cc1c 100644 --- a/roles/sysbench/templates/workload.yml +++ b/roles/sysbench/templates/workload.yml @@ -14,6 +14,10 @@ spec: app: sysbench-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} spec: +{% if workload_args.pin_node is defined %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_node }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} From 46ab53b5f9759864171186aaa46706af11e7924c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Tue, 31 Aug 2021 14:14:46 +0200 Subject: [PATCH 127/249] Set restartPolicy Never to jobs (#664) Signed-off-by: Raul Sevilla --- roles/byowl/templates/workload.yml | 2 +- roles/flent/templates/workload.yml.j2 | 2 +- roles/hammerdb/templates/db_creation.yml | 2 +- roles/hammerdb/templates/db_mariadb_workload.yml.j2 | 2 +- roles/hammerdb/templates/db_mssql_workload.yml.j2 | 2 +- roles/hammerdb/templates/db_postgres_workload.yml.j2 | 2 +- roles/iperf3/templates/client.yml.j2 | 2 +- roles/iperf3/templates/client_store.yml.j2 | 2 +- roles/iperf3/templates/server.yml.j2 | 2 +- roles/pgbench/templates/workload.yml.j2 | 2 +- roles/stressng/templates/stressng_workload.yml.j2 | 2 +- roles/sysbench/templates/workload.yml | 2 +- roles/testpmd/templates/testpmd.yml.j2 | 2 +- roles/testpmd/templates/trex.yml.j2 | 2 +- roles/uperf/templates/workload.yml.j2 | 2 +- roles/vegeta/templates/vegeta.yml.j2 | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/roles/byowl/templates/workload.yml b/roles/byowl/templates/workload.yml index 214403169..c8f353c2a 100755 --- a/roles/byowl/templates/workload.yml +++ b/roles/byowl/templates/workload.yml @@ -32,7 +32,7 @@ spec: {{ label }}: {{ value | to_json }} {% endfor %} {% endif %} - restartPolicy: OnFailure + restartPolicy: Never {% if workload_args.nodeselector is defined %} nodeSelector: {% for label, value in workload_args.nodeselector.items() %} diff --git a/roles/flent/templates/workload.yml.j2 b/roles/flent/templates/workload.yml.j2 index 1f9791d66..9f63326e7 100644 --- a/roles/flent/templates/workload.yml.j2 +++ b/roles/flent/templates/workload.yml.j2 @@ -89,7 +89,7 @@ spec: volumes: - name: flent-tmp emptyDir: {} - restartPolicy: OnFailure + restartPolicy: Never {% if workload_args.pin is sameas true %} nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_client }}' diff --git a/roles/hammerdb/templates/db_creation.yml b/roles/hammerdb/templates/db_creation.yml index d09731a50..c85670350 100644 --- a/roles/hammerdb/templates/db_creation.yml +++ b/roles/hammerdb/templates/db_creation.yml @@ -45,5 +45,5 @@ spec: configMap: name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" defaultMode: 0640 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 index 621f7118a..4afe3ede9 100644 --- a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 @@ -104,6 +104,6 @@ spec: configMap: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0640 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/hammerdb/templates/db_mssql_workload.yml.j2 b/roles/hammerdb/templates/db_mssql_workload.yml.j2 index ddaf94283..ac8794761 100644 --- a/roles/hammerdb/templates/db_mssql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mssql_workload.yml.j2 @@ -110,5 +110,5 @@ spec: configMap: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0640 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/hammerdb/templates/db_postgres_workload.yml.j2 b/roles/hammerdb/templates/db_postgres_workload.yml.j2 index 9e1ef570b..b3bd9b591 100644 --- a/roles/hammerdb/templates/db_postgres_workload.yml.j2 +++ b/roles/hammerdb/templates/db_postgres_workload.yml.j2 @@ -108,5 +108,5 @@ spec: configMap: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0640 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/iperf3/templates/client.yml.j2 b/roles/iperf3/templates/client.yml.j2 index 90e63dabc..02d083848 100644 --- a/roles/iperf3/templates/client.yml.j2 +++ b/roles/iperf3/templates/client.yml.j2 @@ -48,7 +48,7 @@ spec: {% if workload_args.ip_tos is defined %} -S {{ workload_args.ip_tos }} {% endif %} {% if workload_args.omit_start is defined %} -O {{ workload_args.omit_start }} {% endif %} {% if workload_args.extra_options_client is defined %} {{ workload_args.extra_options_client }} {% endif %}" - restartPolicy: OnFailure + restartPolicy: Never {% if workload_args.pin_client is defined %} nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_client }}' diff --git a/roles/iperf3/templates/client_store.yml.j2 b/roles/iperf3/templates/client_store.yml.j2 index 219df6c53..8c54a2f44 100644 --- a/roles/iperf3/templates/client_store.yml.j2 +++ b/roles/iperf3/templates/client_store.yml.j2 @@ -60,4 +60,4 @@ spec: nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_client }}' {% endif %} - restartPolicy: OnFailure + restartPolicy: Never diff --git a/roles/iperf3/templates/server.yml.j2 b/roles/iperf3/templates/server.yml.j2 index 95f6e5128..56761f358 100644 --- a/roles/iperf3/templates/server.yml.j2 +++ b/roles/iperf3/templates/server.yml.j2 @@ -36,7 +36,7 @@ spec: - "iperf3 -s {% if workload_args.port is defined %} -p {{ workload_args.port }} {% endif %} {% if workload_args.extra_options_server is defined %} {{ workload_args.extra_options_server }} {% endif %}" - restartPolicy: OnFailure + restartPolicy: Never {% if workload_args.pin_server is defined %} nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_server }}' diff --git a/roles/pgbench/templates/workload.yml.j2 b/roles/pgbench/templates/workload.yml.j2 index 540feb5f3..42cc85337 100644 --- a/roles/pgbench/templates/workload.yml.j2 +++ b/roles/pgbench/templates/workload.yml.j2 @@ -88,7 +88,7 @@ spec: break; done; fi" - restartPolicy: OnFailure + restartPolicy: Never {% if item.1.pin_node is defined and item.1.pin_node %} nodeSelector: kubernetes.io/hostname: '{{ item.1.pin_node }}' diff --git a/roles/stressng/templates/stressng_workload.yml.j2 b/roles/stressng/templates/stressng_workload.yml.j2 index c65b651af..609d9fbe5 100644 --- a/roles/stressng/templates/stressng_workload.yml.j2 +++ b/roles/stressng/templates/stressng_workload.yml.j2 @@ -106,5 +106,5 @@ spec: configMap: name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0660 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/sysbench/templates/workload.yml b/roles/sysbench/templates/workload.yml index a1ec6cc1c..a736f7c23 100644 --- a/roles/sysbench/templates/workload.yml +++ b/roles/sysbench/templates/workload.yml @@ -39,5 +39,5 @@ spec: configMap: name: "sysbench-config-{{ trunc_uuid }}" defaultMode: 0777 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/testpmd/templates/testpmd.yml.j2 b/roles/testpmd/templates/testpmd.yml.j2 index e3f7a2bd4..6f2c7afc8 100644 --- a/roles/testpmd/templates/testpmd.yml.j2 +++ b/roles/testpmd/templates/testpmd.yml.j2 @@ -80,7 +80,7 @@ spec: - name: hugepage emptyDir: medium: HugePages - restartPolicy: OnFailure + restartPolicy: Never serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} diff --git a/roles/testpmd/templates/trex.yml.j2 b/roles/testpmd/templates/trex.yml.j2 index 309896f42..7555fad34 100644 --- a/roles/testpmd/templates/trex.yml.j2 +++ b/roles/testpmd/templates/trex.yml.j2 @@ -138,7 +138,7 @@ spec: - name: modules hostPath: path: /lib/modules - restartPolicy: OnFailure + restartPolicy: Never serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 837ba334b..e99e8f9f7 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -194,7 +194,7 @@ items: - name: config-volume configMap: name: uperf-test-{{ trunc_uuid }} - restartPolicy: OnFailure + restartPolicy: Never {% if workload_args.pin is sameas false %} {% if workload_args.colocate is sameas true %} nodeSelector: diff --git a/roles/vegeta/templates/vegeta.yml.j2 b/roles/vegeta/templates/vegeta.yml.j2 index b1e0ecf7f..ad300cc3a 100644 --- a/roles/vegeta/templates/vegeta.yml.j2 +++ b/roles/vegeta/templates/vegeta.yml.j2 @@ -91,7 +91,7 @@ spec: - name: targets-volume configMap: name: vegeta-targets-{{ trunc_uuid }} - restartPolicy: OnFailure + restartPolicy: Never {% if workload_args.nodeselector is defined %} nodeSelector: {{ nodeselector|to_json }} {% endif %} From 65ba13c12d85a03da5d4bfe94e1788a6c55e2fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Wed, 1 Sep 2021 15:22:40 +0200 Subject: [PATCH 128/249] Elasticsearch URL (#665) Signed-off-by: Raul Sevilla --- config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml | 2 ++ roles/cyclictest/templates/cyclictestjob.yaml | 2 +- roles/fio_distributed/templates/client.yaml | 2 +- roles/flent/templates/workload.yml.j2 | 2 +- roles/fs-drift/templates/workload_job.yml.j2 | 2 +- roles/hammerdb/templates/db_mariadb_workload.yml.j2 | 2 +- roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 | 4 +--- roles/hammerdb/templates/db_mssql_workload.yml.j2 | 2 +- roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 | 4 +--- roles/hammerdb/templates/db_postgres_workload.yml.j2 | 2 +- roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 | 4 +--- roles/image_pull/templates/image_pull.yml | 2 +- roles/log_generator/templates/log_generator.yml | 2 +- roles/oslat/templates/oslatjob.yaml | 2 +- roles/pgbench/templates/workload.yml.j2 | 2 +- roles/scale_openshift/templates/scale.yml | 2 +- roles/smallfile/templates/workload_job.yml.j2 | 2 +- roles/stressng/templates/stressng_workload.yml.j2 | 2 +- roles/stressng/templates/stressng_workload_vm.yml.j2 | 4 +--- roles/testpmd/templates/trex.yml.j2 | 2 +- roles/uperf/templates/configmap_script.yml.j2 | 4 +--- roles/uperf/templates/workload.yml.j2 | 2 +- roles/vegeta/templates/vegeta.yml.j2 | 2 +- roles/ycsb/templates/ycsb_load.yaml | 2 +- roles/ycsb/templates/ycsb_run.yaml | 2 +- 25 files changed, 26 insertions(+), 34 deletions(-) diff --git a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml index 90d4a716d..bd8e86e15 100644 --- a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml +++ b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml @@ -29,8 +29,10 @@ spec: properties: url: type: string + default: "" index_name: type: string + default: "" parallel: type: boolean default: false diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index 38f0682e0..19e21aa20 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -43,7 +43,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url}}" - name: es_index diff --git a/roles/fio_distributed/templates/client.yaml b/roles/fio_distributed/templates/client.yaml index df4e1def9..3256e4bf7 100644 --- a/roles/fio_distributed/templates/client.yaml +++ b/roles/fio_distributed/templates/client.yaml @@ -39,7 +39,7 @@ spec: - name: CEPH_CACHE_DROP_PORT_NUM value: "{{ ceph_cache_drop_svc_port }}" {% endif %} -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/flent/templates/workload.yml.j2 b/roles/flent/templates/workload.yml.j2 index 9f63326e7..e9c0ed7e2 100644 --- a/roles/flent/templates/workload.yml.j2 +++ b/roles/flent/templates/workload.yml.j2 @@ -42,7 +42,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/fs-drift/templates/workload_job.yml.j2 b/roles/fs-drift/templates/workload_job.yml.j2 index 80d437098..ba2970a39 100644 --- a/roles/fs-drift/templates/workload_job.yml.j2 +++ b/roles/fs-drift/templates/workload_job.yml.j2 @@ -48,7 +48,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 index 4afe3ede9..434aabb58 100644 --- a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 @@ -41,7 +41,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 index b68c124d1..9e837ee79 100644 --- a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 @@ -1,12 +1,10 @@ -{% if elasticsearch is defined %} -{% if elasticsearch.url is defined %} +{% if elasticsearch.url %} export es={{ elasticsearch.url }}; export es_index={{ elasticsearch.index_name | default("ripsaw-hammerdb") }}; export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; export parallel={{ elasticsearch.parallel | default("false") }}; export uuid={{uuid}}; {% endif %} -{% endif %} {% if workload_args.client_vm.network.multiqueue.enabled %} dnf install -y ethtool; ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; diff --git a/roles/hammerdb/templates/db_mssql_workload.yml.j2 b/roles/hammerdb/templates/db_mssql_workload.yml.j2 index ac8794761..152353797 100644 --- a/roles/hammerdb/templates/db_mssql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mssql_workload.yml.j2 @@ -41,7 +41,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 index 6ad224f4e..938415535 100644 --- a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 @@ -1,12 +1,10 @@ -{% if elasticsearch is defined %} -{% if elasticsearch.url is defined %} +{% if elasticsearch.url %} export es={{ elasticsearch.url }}; export es_index={{ elasticsearch.index_name | default("ripsaw-hammerdb") }}; export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; export parallel={{ elasticsearch.parallel | default("false") }}; export uuid={{uuid}}; {% endif %} -{% endif %} {% if workload_args.client_vm.network.multiqueue.enabled %} dnf install -y ethtool; ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; diff --git a/roles/hammerdb/templates/db_postgres_workload.yml.j2 b/roles/hammerdb/templates/db_postgres_workload.yml.j2 index b3bd9b591..9c5baa8f6 100644 --- a/roles/hammerdb/templates/db_postgres_workload.yml.j2 +++ b/roles/hammerdb/templates/db_postgres_workload.yml.j2 @@ -41,7 +41,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 index 096200192..fa7aea661 100644 --- a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 @@ -1,12 +1,10 @@ -{% if elasticsearch is defined %} -{% if elasticsearch.url is defined %} +{% if elasticsearch.url %} export es={{ elasticsearch.url }}; export es_index={{ elasticsearch.index_name | default("ripsaw-hammerdb") }}; export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; export parallel={{ elasticsearch.parallel | default("false") }}; export uuid={{uuid}}; {% endif %} -{% endif %} {% if workload_args.client_vm.network.multiqueue.enabled %} dnf install -y ethtool; ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; diff --git a/roles/image_pull/templates/image_pull.yml b/roles/image_pull/templates/image_pull.yml index c0a100196..ad2434cda 100644 --- a/roles/image_pull/templates/image_pull.yml +++ b/roles/image_pull/templates/image_pull.yml @@ -52,7 +52,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/log_generator/templates/log_generator.yml b/roles/log_generator/templates/log_generator.yml index c180911e0..24cbb9899 100644 --- a/roles/log_generator/templates/log_generator.yml +++ b/roles/log_generator/templates/log_generator.yml @@ -54,7 +54,7 @@ spec: value: "{{ clustername }}" - name: snafu_disable_logs value: "{{ workload_args.snafu_disable_logs | default(false) }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index e98b31682..9e165b13c 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -44,7 +44,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url}}" - name: es_index diff --git a/roles/pgbench/templates/workload.yml.j2 b/roles/pgbench/templates/workload.yml.j2 index 42cc85337..7cdb51858 100644 --- a/roles/pgbench/templates/workload.yml.j2 +++ b/roles/pgbench/templates/workload.yml.j2 @@ -34,7 +34,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: '{{ clustername }}' -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/scale_openshift/templates/scale.yml b/roles/scale_openshift/templates/scale.yml index c9ea17575..092c384f1 100755 --- a/roles/scale_openshift/templates/scale.yml +++ b/roles/scale_openshift/templates/scale.yml @@ -44,7 +44,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/smallfile/templates/workload_job.yml.j2 b/roles/smallfile/templates/workload_job.yml.j2 index 4b983c6a4..322d0d7e5 100644 --- a/roles/smallfile/templates/workload_job.yml.j2 +++ b/roles/smallfile/templates/workload_job.yml.j2 @@ -60,7 +60,7 @@ spec: value: "{{ ceph_cache_drop_svc_port }}" {% endif %} {% endif %} -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/stressng/templates/stressng_workload.yml.j2 b/roles/stressng/templates/stressng_workload.yml.j2 index 609d9fbe5..917bac5bc 100644 --- a/roles/stressng/templates/stressng_workload.yml.j2 +++ b/roles/stressng/templates/stressng_workload.yml.j2 @@ -70,7 +70,7 @@ spec: - name: mem_stressors value: "{{workload_args.mem_stressors}}" {% endif %} -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/stressng/templates/stressng_workload_vm.yml.j2 b/roles/stressng/templates/stressng_workload_vm.yml.j2 index fa0a57d98..9a1ad6a2a 100644 --- a/roles/stressng/templates/stressng_workload_vm.yml.j2 +++ b/roles/stressng/templates/stressng_workload_vm.yml.j2 @@ -65,15 +65,13 @@ spec: - "mkdir /tmp/stressng-test/jobfile" - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240D | cut -f1 -d' ') /tmp/stressng-test" runcmd: -{% if elasticsearch is defined %} -{% if elasticsearch.url is defined %} +{% if elasticsearch.url %} - export es={{ elasticsearch.url }}; - export es_index={{ elasticsearch.index_name | default("ripsaw-stressng") }}; - export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; - export parallel={{ elasticsearch.parallel | default("false") }}; - export uuid={{uuid}}; {% endif %} -{% endif %} {% if workload_args.client_vm.network.multiqueue.enabled %} - dnf install -y ethtool - ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }} diff --git a/roles/testpmd/templates/trex.yml.j2 b/roles/testpmd/templates/trex.yml.j2 index 7555fad34..1e8eaa745 100644 --- a/roles/testpmd/templates/trex.yml.j2 +++ b/roles/testpmd/templates/trex.yml.j2 @@ -113,7 +113,7 @@ spec: value: "{{ value }}" {% endfor %} {% endif %} -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/uperf/templates/configmap_script.yml.j2 b/roles/uperf/templates/configmap_script.yml.j2 index 5abc31d1b..884bea473 100644 --- a/roles/uperf/templates/configmap_script.yml.j2 +++ b/roles/uperf/templates/configmap_script.yml.j2 @@ -7,15 +7,13 @@ metadata: data: run_script.sh : | export h={{item.status.interfaces[0].ipAddress}} -{% if elasticsearch is defined %} -{% if elasticsearch.url is defined %} +{% if elasticsearch.url %} export es={{ elasticsearch.url }} export es_index={{ elasticsearch.index_name | default("ripsaw-uperf") }} export es_verify_cert={{ elasticsearch.verify_cert | default(true) }} export parallel={{ elasticsearch.parallel | default(false) }} export uuid={{uuid}} {% endif %} -{% endif %} {% if test_user is defined %} export test_user={{test_user}} {% endif %} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index e99e8f9f7..990254949 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -57,7 +57,7 @@ items: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/vegeta/templates/vegeta.yml.j2 b/roles/vegeta/templates/vegeta.yml.j2 index ad300cc3a..b915b2299 100644 --- a/roles/vegeta/templates/vegeta.yml.j2 +++ b/roles/vegeta/templates/vegeta.yml.j2 @@ -42,7 +42,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/ycsb/templates/ycsb_load.yaml b/roles/ycsb/templates/ycsb_load.yaml index 556510d6a..4985315f6 100644 --- a/roles/ycsb/templates/ycsb_load.yaml +++ b/roles/ycsb/templates/ycsb_load.yaml @@ -32,7 +32,7 @@ spec: value: "{{workload_args.operationcount}}" - name: uuid value: "{{uuid}}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index diff --git a/roles/ycsb/templates/ycsb_run.yaml b/roles/ycsb/templates/ycsb_run.yaml index c7523cf18..5d5cfbbc1 100644 --- a/roles/ycsb/templates/ycsb_run.yaml +++ b/roles/ycsb/templates/ycsb_run.yaml @@ -32,7 +32,7 @@ spec: value: "{{workload_args.operationcount}}" - name: uuid value: "{{uuid}}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index From 0c7ffee173a077d6bab633e985ebf44a1e9e818f Mon Sep 17 00:00:00 2001 From: Ben England Date: Wed, 1 Sep 2021 09:27:33 -0400 Subject: [PATCH 129/249] Fio doc fixes (#660) * fix fio sample CRs to show correct cache dropping syntax * update cache dropping description * point to example CR instead of copy-and-pasting example CR * document run_timeout * fix relative URL, point to section of doc --- config/samples/fio/cr.yaml | 8 ++- config/samples/fio/vm-cr.yaml | 2 - docs/fio_distributed.md | 123 ++-------------------------------- 3 files changed, 12 insertions(+), 121 deletions(-) diff --git a/config/samples/fio/cr.yaml b/config/samples/fio/cr.yaml index ffd21cf29..c6e1c19c5 100644 --- a/config/samples/fio/cr.yaml +++ b/config/samples/fio/cr.yaml @@ -53,8 +53,12 @@ spec: log_sample_rate: 3000 #storageclass: rook-ceph-block #storagesize: 5Gi - #rook_ceph_drop_caches: True - #rook_ceph_drop_cache_pod_ip: 192.168.111.20 + # use drop_cache_kernel to have set of labeled nodes drop kernel buffer cache before each sample + #drop_cache_kernel: False + # use drop_cache_rook_ceph to have Ceph OSDs drop their cache before each sample + #drop_cache_rook_ceph: False + # increase this if you want fio to run for more than 1 hour without being terminated by K8S + #job_timeout: 3600 ####################################### # EXPERT AREA - MODIFY WITH CAUTION # ####################################### diff --git a/config/samples/fio/vm-cr.yaml b/config/samples/fio/vm-cr.yaml index 175bc492a..aef25617a 100644 --- a/config/samples/fio/vm-cr.yaml +++ b/config/samples/fio/vm-cr.yaml @@ -67,8 +67,6 @@ spec: # Can be one of Filesystem,Block Default: Filesystem pvcvolumemode: Block storagesize: 5Gi - #rook_ceph_drop_caches: True - #rook_ceph_drop_cache_pod_ip: 192.168.111.20 ####################################### # EXPERT AREA - MODIFY WITH CAUTION # ####################################### diff --git a/docs/fio_distributed.md b/docs/fio_distributed.md index d8728a629..5ec427e18 100644 --- a/docs/fio_distributed.md +++ b/docs/fio_distributed.md @@ -12,82 +12,8 @@ point for executing the workload on the server pods in parallel. ## Running Distributed FIO The Custom Resource (CR) file for fio includes a significant number of options to offer the user -flexibility. +flexibility. [Click here for an example of an fio CR](../config/samples/fio/cr.yaml). -```yaml -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: fio-benchmark - namespace: benchmark-operator -spec: - elasticsearch: - url: "http://my.es.server:9200" - clustername: myk8scluster - test_user: my_test_user_name - workload: - name: "fio_distributed" - args: - prefill: true - # for compressed volume uncomment the next 2 lines and make the cmp_bs same as bs - # prefill_bs: 8KiB - # cmp_ratio: 70 - samples: 3 - servers: 3 - # Chose to run servers in 'pod' or 'vm' - # 'vm' needs kubevirt to be available - # Default: pod - kind: pod - runtime_class: class_name - jobs: - - write - - read - bs: - - 4KiB - - 64KiB - numjobs: - - 1 - - 8 - iodepth: 4 - read_runtime: 60 - read_ramp_time: 5 - filesize: 2GiB - log_sample_rate: 1000 - storageclass: rook-ceph-block - storagesize: 5Gi - rook_ceph_drop_caches: True - rook_ceph_drop_cache_pod_ip: 192.168.111.20 -####################################### -# EXPERT AREA - MODIFY WITH CAUTION # -####################################### -# global_overrides: -# - key=value - job_params: - - jobname_match: w - params: - - fsync_on_close=1 - - create_on_open=1 - - jobname_match: read - params: - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: rw - params: - - rwmixread=50 - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: readwrite - params: - - rwmixread=50 - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} -# - jobname_match: -# params: -# - key=value -``` ### NodeSelector and Taint/Tolerations vs. pin server @@ -157,19 +83,6 @@ The workload loops are nested as such from the CR options: > Therefore, be aware in providing units to the CR values that fio options should use the `MiB` format > while the `storagesize` option used for the K8S persistent volume claim should use the `Mi` format -#### metadata - -*Values here will usually be left unchanged* - -- **name**: The name the Ripsaw operator will use for the benchmark resource -- **namespace**: The namespace in which the benchmark will run - -#### spec - -- **elasticsearch**: (optional) Values are used to enable indexing of fio data; [further details are below](#indexing-in-elasticsearch-and-visualization-through-grafana) -- **clustername**: (optional) An arbitrary name for your system under test (SUT) that can aid indexing -- **test_user**: (optional) An arbitrary name for the user performing the tests that can aid indexing - #### spec.workload - **name**: **DO NOT CHANGE** This value is used by the Ripsaw operator to trigger the correct Ansible role @@ -225,16 +138,16 @@ The workload loops are nested as such from the CR options: - **pvcvolumemode**: (optional) The volmeMode to request with the persistent volume claim (PVC) for the fio server. Can be one of Filesystem,Block Default: Filesystem > Note: It is recommended to change this to `Block` for VM tests - **storagesize**: (optional) The size of the PVCs to request from the StorageClass ([note units quirk per above](#understanding-the-cr-options)) -- **rook_ceph_drop_caches**: (optional) If set to `True`, the Rook-Ceph OSD caches will be dropped prior to each sample -- **rook_ceph_drop_cache_pod_ip**: (optional) The IP address of the pod hosting the Rook-Ceph cache drop URL -- See [cache drop pod instructions](#dropping-rook-ceph-osd-caches) below +- **drop_cache_kernel**: (optional, default false) If set to `True`, kernel cache will be dropped on the labeled hosts before each sample. See [here for how to set up kernel cache dropping](./cache_dropping.md#how-to-drop-kernel-cache) +- **drop_cache_rook_ceph**: (optional, default false) The IP address of the pod hosting the Rook-Ceph cache drop URL -- See [here for how to set up Ceph OSD cache dropping](./cache_dropping.md#how-to-drop-Ceph-OSD-cache) > Technical Note: If you are running kube/openshift on VMs make sure the diskimage or volume is preallocated. - **prefill**: (Optional) boolean to enable/disable prefill SDS - prefill requirement stems from Ceph RBD thin-provisioning - just creating the RBD volume doesn't mean that there is space allocated to read and write out there. For example, reads to an uninitialized volume don't even talk to the Ceph OSDs, they just return immediately with zeroes in the client. - **prefill_bs** (Optional) The Block size that need to used for the prefill. - When running against compressed volumes, the prefill operation need to be done with the same block size as using in the test, otherwise the compression ratio will not be as expected. - **cmp_ratio** (Optional) When running against compressed volumes, the expected compression ratio (0-100) -- **fio_json_to_log**: (Optional) boolean to enable/disable sending job results in json format to client pod log. - +- **fio_json_to_log**: (Optional, default false) boolean to enable/disable sending job results in json format to client pod log. +- **job_timeout**: (Optional, default 3600) set this parameter if you want the fio to run for more than 1 hour without automatic termination #### EXPERT: spec.global_overrides The `key=value` combinations provided in the list here will be appended to the `[global]` section of the fio @@ -245,30 +158,6 @@ jobfile configmap. These options will therefore override the global values for a Under most circumstances, the options provided in the EXPERT AREA here should not be modified. The `key=value` pairs under `params` here are used to append additional fio job options based on the job type. Each `jobname_match` in the list uses a "search string" to match a job name per `fio(1)`, and if a match is made, the `key=value` list items under `params` are appended to the `[job]` section of the fio jobfile configmap. -## Dropping Rook-Ceph OSD Caches - -Dropping the OSD caches before workloads is a normal and advised part of tests that involve storage I/O. -Doing this with Rook-Ceph requires a privileged pod running the same namespace as the Ceph pods and with -the Ceph command tools available. To facilitate this, we provide the -[config/samples/fio/cr.yaml](../config/samples/fio/cr.yaml) file, which will -deploy a pod with the correct permissions and tools, as well as running a simple HTTP listener to trigger -the cache drop by URL. -**You must deploy this privileged pod in order for the drop caches requests in the workload to function.** - -```bash -# kubectl apply -f config/samples/fio/cr.yaml # if edited the original one -# kubectl apply -f # if created a new cr file -``` - -*Note: If Ceph is in a namespace other than `rook-ceph` you will need to modify the provided YAML accordingly.* - -Since the cache drop pod is deployed with host networking, the pod will take on the IP address -of the node on which it is running. You will need to use this IP address in the CR file as described above. - -```bash -# kubectl get pod -n rook-ceph rook-ceph-osd-cache-drop --template={{.status.podIP}} -``` - ## Indexing in elasticsearch and visualization through Grafana ### Setup of Elasticsearch and Grafana @@ -304,4 +193,4 @@ The field for timestamp will always be `time_ms` . In order to index your fio results to elasticsearch, you will need to define the parameters appropriately in your workload CR file. The `spec.elasticsearch.url` parameter is required. -The `spec.clustername` and `spec.test_user` values are advised to allow for better indexing of your data. +The `spec.clustername` and `spec.test_user` values are advised to make it easier to find your set of test results in elasticsearch. From 9826c52bd02ec21302e656155fcb3d2369de00c4 Mon Sep 17 00:00:00 2001 From: hnhan Date: Wed, 1 Sep 2021 09:28:24 -0400 Subject: [PATCH 130/249] UPERF: Fix node name truncation to avoid ending with a dot '.' char (#655) * when truncating name, make sure last char is not a dot, '.' * remove cosmetic spaces --- roles/uperf/templates/server.yml.j2 | 13 +++++++++++-- roles/uperf/templates/service.yml.j2 | 12 ++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index dd9a4cd9f..c5285192e 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -7,7 +7,11 @@ items: - kind: Job apiVersion: batch/v1 metadata: - name: 'uperf-server-{{worker_node_list[ node_idx_item ] | truncate(27,true,'') }}-{{ item }}-{{ trunc_uuid }}' +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + name: 'uperf-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }}' +{% else %} + name: 'uperf-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }}' +{% endif %} namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -18,7 +22,12 @@ items: benchmark-uuid: {{ uuid }} benchmark-operator-workload: uperf benchmark-operator-role: server - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'') }}-{{ item }}-{{ trunc_uuid }} +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} annotations: {% if workload_args.multus.enabled is sameas true %} diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 9d008e062..2664a4872 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -12,14 +12,22 @@ items: labels: benchmark-uuid: {{ uuid }} benchmark-operator-workload: uperf - app: uperf-bench-server-{{ worker_node_list[ node_idx_item ] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} annotations: node_idx: '{{ node_idx_item }}' pod_idx: '{{ item }}' spec: selector: - app: uperf-bench-server-{{worker_node_list[ node_idx_item ] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} ports: - name: uperf port: 20000 From a67e8e5c61987236aa26e668180d6cf52ce84d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Wed, 1 Sep 2021 19:54:01 +0200 Subject: [PATCH 131/249] Fix not targeted metadata collection (#667) Signed-off-by: Raul Sevilla --- roles/backpack/tasks/main.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/roles/backpack/tasks/main.yml b/roles/backpack/tasks/main.yml index 8a1ff169e..40f2f1177 100644 --- a/roles/backpack/tasks/main.yml +++ b/roles/backpack/tasks/main.yml @@ -5,7 +5,6 @@ name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: - state: "Metadata Collecting" complete: false metadata: "Collecting" when: benchmark_state.resources[0].status.state is not defined @@ -71,6 +70,16 @@ state: Complete complete: true metadata: Complete + when: workload.name == "backpack" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + metadata: Complete + when: workload.name != "backpack" - name: Get benchmark state k8s_facts: From 3f134efab4339d02a5985a17bd06c1696359a70f Mon Sep 17 00:00:00 2001 From: dry923 Date: Wed, 1 Sep 2021 13:56:38 -0400 Subject: [PATCH 132/249] Removing default value for image_name as it was causing the real defaults defined in the yaml template not to be used --- config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml index bd8e86e15..473f720a8 100644 --- a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml +++ b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml @@ -32,7 +32,6 @@ spec: default: "" index_name: type: string - default: "" parallel: type: boolean default: false From 15bba07d7f3face5228f1f0f6e7fc890c226318c Mon Sep 17 00:00:00 2001 From: Ben England Date: Wed, 1 Sep 2021 18:20:35 -0400 Subject: [PATCH 133/249] Fio no log hist (#666) * disable histogram logging by default * test different cases for configmap.yml.j2 * shrink time needed to run test, less logging --- docs/fio_distributed.md | 3 ++- roles/fio_distributed/templates/configmap.yml.j2 | 8 ++++++-- tests/test_crs/valid_fiod_bsrange.yaml | 1 + tests/test_crs/valid_fiod_hostpath.yaml | 6 +----- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/fio_distributed.md b/docs/fio_distributed.md index 5ec427e18..cfbb96e3b 100644 --- a/docs/fio_distributed.md +++ b/docs/fio_distributed.md @@ -132,7 +132,8 @@ The workload loops are nested as such from the CR options: > using these parameters, but note this behavious is configured via the EXPERT AREA section of the CR as > [described below](#expert-specjob_params), and therefore this may be adjusted to user preferences. - **filesize**: The size of the file used for each job in the workload (per `numjobs * servers` as described above) -- **log_sample_rate**: Applied to fio options `log_avg_msec` and `log_hist_msec` in the jobfile configmap; see `fio(1)` +- **log_sample_rate**: (optional) Applied to fio options `log_avg_msec` (milliseconds) in the jobfile configmap; see `fio(1)` +- **log_hist_msec** (optional) if set, enables histogram logging at this specified interval in milliseconds - **storageclass**: (optional) The K8S StorageClass to use for persistent volume claims (PVC) per server pod - **pvcaccessmode**: (optional) The AccessMode to request with the persistent volume claim (PVC) for the fio server. Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce - **pvcvolumemode**: (optional) The volmeMode to request with the persistent volume claim (PVC) for the fio server. Can be one of Filesystem,Block Default: Filesystem diff --git a/roles/fio_distributed/templates/configmap.yml.j2 b/roles/fio_distributed/templates/configmap.yml.j2 index 06cf1463b..06dde3ab7 100644 --- a/roles/fio_distributed/templates/configmap.yml.j2 +++ b/roles/fio_distributed/templates/configmap.yml.j2 @@ -51,9 +51,13 @@ data: write_bw_log=fio write_iops_log=fio write_lat_log=fio +{% if workload_args.log_sample_rate is defined %} + log_avg_msec={{ workload_args.log_sample_rate }} +{% endif %} +{% if workload.log_hist_msec is defined %} write_hist_log=fio - log_avg_msec={{workload_args.log_sample_rate}} - log_hist_msec={{workload_args.log_sample_rate}} + log_hist_msec={{ workload_args.log_hist_msec }} +{% endif %} clocksource=clock_gettime kb_base=1000 unit_base=8 diff --git a/tests/test_crs/valid_fiod_bsrange.yaml b/tests/test_crs/valid_fiod_bsrange.yaml index f88ee8a40..2c5cdbb26 100644 --- a/tests/test_crs/valid_fiod_bsrange.yaml +++ b/tests/test_crs/valid_fiod_bsrange.yaml @@ -38,6 +38,7 @@ spec: write_ramp_time: 1 filesize: 10MiB log_sample_rate: 2000 + log_hist_msec: 3000 storagesize: 16Mi debug: true ####################################### diff --git a/tests/test_crs/valid_fiod_hostpath.yaml b/tests/test_crs/valid_fiod_hostpath.yaml index 79092c6db..cc56c4cbd 100644 --- a/tests/test_crs/valid_fiod_hostpath.yaml +++ b/tests/test_crs/valid_fiod_hostpath.yaml @@ -26,14 +26,11 @@ spec: prefill: true jobs: - write - - read - - randwrite - randread - randrw bs: - 4KiB numjobs: - - 1 - 2 iodepth: 2 read_runtime: 6 @@ -41,8 +38,7 @@ spec: read_ramp_time: 1 write_ramp_time: 1 filesize: 10MiB - log_sample_rate: 2000 - debug: true + debug: false ####################################### # EXPERT AREA - MODIFY WITH CAUTION # ####################################### From abc48ea9691313df5e2478e3afa6d2743789696f Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 2 Sep 2021 11:55:13 +0200 Subject: [PATCH 134/249] Fix oslat execution Signed-off-by: Raul Sevilla --- roles/oslat/templates/oslatjob.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index 9e165b13c..9c48a8ced 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -20,9 +20,9 @@ spec: image: {{ workload_args.image | default('quay.io/cloud-bulldozer/oslat:latest') }} command: ["/bin/sh", "-c"] args: - - run_snafu --tool oslat -p /tmp/oslat.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ + - run_snafu --tool oslat -p /tmp/oslat.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}} {% if workload_args.debug is defined and workload_args.debug %} - -v \ + -v {% endif %} ; imagePullPolicy: Always From befbd516e550e5ec1844dbc83f07d0512f4fd385 Mon Sep 17 00:00:00 2001 From: Eduardo Minguez Perez Date: Thu, 2 Sep 2021 12:10:54 +0200 Subject: [PATCH 135/249] Add a nodeSelector for vegeta pods --- config/samples/vegeta/cr.yaml | 1 + docs/vegeta.md | 4 ++++ roles/vegeta/templates/vegeta.yml.j2 | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/config/samples/vegeta/cr.yaml b/config/samples/vegeta/cr.yaml index dfa521393..ece98d77c 100644 --- a/config/samples/vegeta/cr.yaml +++ b/config/samples/vegeta/cr.yaml @@ -12,6 +12,7 @@ spec: workload: name: vegeta args: + # node_selector: "vegeta=true" hostnetwork: false clients: 2 targets: diff --git a/docs/vegeta.md b/docs/vegeta.md index a0623a42e..c769c908e 100644 --- a/docs/vegeta.md +++ b/docs/vegeta.md @@ -11,6 +11,9 @@ The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The **node_selector** option can be used to limit the nodes where +the vegeta pods are deployed. + ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -24,6 +27,7 @@ spec: workload: name: vegeta args: + # node_selector: "vegeta=true" clients: 2 image: quay.io/cloud-bulldozer/vegeta:latest hostnetwork: false diff --git a/roles/vegeta/templates/vegeta.yml.j2 b/roles/vegeta/templates/vegeta.yml.j2 index b915b2299..e9cea4300 100644 --- a/roles/vegeta/templates/vegeta.yml.j2 +++ b/roles/vegeta/templates/vegeta.yml.j2 @@ -13,6 +13,10 @@ spec: app: vegeta-benchmark-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} spec: +{% if workload_args.node_selector is defined %} + nodeSelector: + '{{ workload_args.node_selector.split("=")[0] }}': '{{ workload_args.node_selector.split("=")[1] }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} From 8594fb8de2e6ab25879f468fe9e99961726d2bbd Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Fri, 3 Sep 2021 09:32:03 +0200 Subject: [PATCH 136/249] Convert testpmd into pod Signed-off-by: Raul Sevilla --- roles/testpmd/tasks/main.yml | 4 +- roles/testpmd/templates/testpmd.yml.j2 | 117 ++++++++++++------------- 2 files changed, 59 insertions(+), 62 deletions(-) diff --git a/roles/testpmd/tasks/main.yml b/roles/testpmd/tasks/main.yml index e2d38b8aa..ba190e76c 100644 --- a/roles/testpmd/tasks/main.yml +++ b/roles/testpmd/tasks/main.yml @@ -7,7 +7,7 @@ include_tasks: 20_mac_parser.yml loop: "{{ workload_args.networks.testpmd }}" loop_control: - loop_var: network + loop_var: network vars: net_type: testpmd @@ -15,7 +15,7 @@ include_tasks: 20_mac_parser.yml loop: "{{ workload_args.networks.trex }}" loop_control: - loop_var: network + loop_var: network vars: net_type: trex diff --git a/roles/testpmd/templates/testpmd.yml.j2 b/roles/testpmd/templates/testpmd.yml.j2 index 6f2c7afc8..6823cf66f 100644 --- a/roles/testpmd/templates/testpmd.yml.j2 +++ b/roles/testpmd/templates/testpmd.yml.j2 @@ -1,86 +1,83 @@ # https://github.com/rh-nfv-int/testpmd-operator -kind: Job -apiVersion: batch/v1 +kind: Pod +apiVersion: v1 metadata: name: "testpmd-application-pod-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" -spec: - template: - metadata: - labels: - app: "testpmd-application-pod-{{ trunc_uuid }}" - benchmark-uuid: {{ uuid }} - annotations: - k8s.v1.cni.cncf.io/networks: '[ + labels: + app: "testpmd-application-pod-{{ trunc_uuid }}" + benchmark-uuid: {{ uuid }} + annotations: + maclist_trex: "{{ maclist_trex|list|join(',') }}" + maclist_testpmd: "{{ maclist_testpmd|list|join(',') }}" + k8s.v1.cni.cncf.io/networks: '[ {% set i = namespace(value=0) %} {% for network in workload_args.networks.testpmd %} {% set outer_loop = loop %} {% for count in range(network.count) %} - { - "name": "{{ network.name }}", - "mac": "{{ maclist_testpmd[i.value] }}", - "namespace": "{{ operator_namespace }}" - }{% if loop.index < network.count or outer_loop.index < network_name_list|length %},{% endif %} + { + "name": "{{ network.name }}", + "mac": "{{ maclist_testpmd[i.value] }}", + "namespace": "{{ operator_namespace }}" + }{% if loop.index < network.count or outer_loop.index < network_name_list|length %},{% endif %} {% set i.value = i.value + 1 %} {% endfor %} {% endfor %} - ]' - maclist_trex: "{{ maclist_trex|list|join(',') }}" - maclist_testpmd: "{{ maclist_testpmd|list|join(',') }}" - spec: + ]' +spec: {% if workload_args.pin is sameas true %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_testpmd }}' + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_testpmd }}' {% endif %} - terminationGracePeriodSeconds: 10 - containers: - - name: testpmd - command: ["/bin/sh"] - args: - - "-c" - - echo "set promisc all off" >> /tmp/testpmd-cmdline.txt && testpmd -l $(cat /sys/fs/cgroup/cpuset/cpuset.cpus) $(PCI=""; IFS=',' read -r -a NETWORK_ARRAY <<< "$NETWORK_NAME_LIST"; for item in "${NETWORK_ARRAY[@]}"; do IFS='/' read -r -a RES_ARRAY <<< "$item"; NAME="PCIDEVICE_OPENSHIFT_IO_${RES_ARRAY[1]^^}";IFS=',' read -r -a PCI_ARRAY <<< "${!NAME}";for pci_item in "${PCI_ARRAY[@]}"; do PCI+=" -w ${pci_item} ";done;done;echo $PCI) --socket-mem {{ workload_args.socket_memory | default('1024,0') }} -n {{ workload_args.memory_channels | default(4) }} --proc-type auto --file-prefix pg -- {% if workload_args.disable_rss | default(true) %} --disable-rss {% endif %} --nb-cores={{ workload_args.forwarding_cores | default(4) }} --rxq={{ workload_args.rx_queues | default(1) }} --txq={{ workload_args.tx_queues | default(1) }} --rxd={{ workload_args.rx_descriptors | default(1024) }} --txd={{ workload_args.tx_descriptors | default(1024) }} --auto-start {% for mac in maclist_trex %} --eth-peer={{ loop.index -1 }},{{ mac }} {% endfor %} --forward-mode={{ workload_args.forward_mode | default('mac') }} --stats-period {{ workload_args.stats_period | default(1) }} --cmdline-file /tmp/testpmd-cmdline.txt - image: "{{ workload_args.image_testpmd | default('registry.redhat.io/openshift4/dpdk-base-rhel8:v4.6') }}" - imagePullPolicy: "{{ workload_args.image_pull_policy | default('Always') }}" - securityContext: + terminationGracePeriodSeconds: 10 + containers: + - name: testpmd + command: ["/bin/sh"] + args: + - "-c" + - echo "set promisc all off" >> /tmp/testpmd-cmdline.txt && testpmd -l $(cat /sys/fs/cgroup/cpuset/cpuset.cpus) $(PCI=""; IFS=',' read -r -a NETWORK_ARRAY <<< "$NETWORK_NAME_LIST"; for item in "${NETWORK_ARRAY[@]}"; do IFS='/' read -r -a RES_ARRAY <<< "$item"; NAME="PCIDEVICE_OPENSHIFT_IO_${RES_ARRAY[1]^^}";IFS=',' read -r -a PCI_ARRAY <<< "${!NAME}";for pci_item in "${PCI_ARRAY[@]}"; do PCI+=" -w ${pci_item} ";done;done;echo $PCI) --socket-mem {{ workload_args.socket_memory | default('1024,0') }} -n {{ workload_args.memory_channels | default(4) }} --proc-type auto --file-prefix pg -- {% if workload_args.disable_rss | default(true) %} --disable-rss {% endif %} --nb-cores={{ workload_args.forwarding_cores | default(4) }} --rxq={{ workload_args.rx_queues | default(1) }} --txq={{ workload_args.tx_queues | default(1) }} --rxd={{ workload_args.rx_descriptors | default(1024) }} --txd={{ workload_args.tx_descriptors | default(1024) }} --auto-start {% for mac in maclist_trex %} --eth-peer={{ loop.index -1 }},{{ mac }} {% endfor %} --forward-mode={{ workload_args.forward_mode | default('mac') }} --stats-period {{ workload_args.stats_period | default(1) }} --cmdline-file /tmp/testpmd-cmdline.txt + image: "{{ workload_args.image_testpmd | default('registry.redhat.io/openshift4/dpdk-base-rhel8:v4.6') }}" + imagePullPolicy: "{{ workload_args.image_pull_policy | default('Always') }}" + securityContext: {% if workload_args.privileged %} - privileged: true + privileged: true {% else %} - capabilities: - add: ["IPC_LOCK", "NET_ADMIN"] + capabilities: + add: ["IPC_LOCK", "NET_ADMIN"] {% endif %} - resources: - limits: - hugepages-1Gi: {{ workload_args.pod_hugepage_1gb_count | default('4Gi') }} - memory: {{ workload_args.pod_memory | default('1000Mi') }} - cpu: {{ workload_args.pod_cpu | default(6) }} + resources: + limits: + hugepages-1Gi: {{ workload_args.pod_hugepage_1gb_count | default('4Gi') }} + memory: {{ workload_args.pod_memory | default('1000Mi') }} + cpu: {{ workload_args.pod_cpu | default(6) }} {% for key, value in network_resources.items() %} - {{ key }}: {{ value }} + {{ key }}: {{ value }} {% endfor %} - requests: - hugepages-1Gi: {{ workload_args.pod_hugepage_1gb_count | default('4Gi') }} - memory: {{ workload_args.pod_memory | default('1000Mi') }} - cpu: {{ workload_args.pod_cpu | default(6) }} + requests: + hugepages-1Gi: {{ workload_args.pod_hugepage_1gb_count | default('4Gi') }} + memory: {{ workload_args.pod_memory | default('1000Mi') }} + cpu: {{ workload_args.pod_cpu | default(6) }} {% for key, value in network_resources.items() %} - {{ key }}: {{ value }} + {{ key }}: {{ value }} {% endfor %} - volumeMounts: - - name: hugepage - mountPath: /dev/hugepages - env: - - name: NETWORK_NAME_LIST - value: "{{ network_resources.keys()|list|join(',') }}" + volumeMounts: + - name: hugepage + mountPath: /dev/hugepages + env: + - name: NETWORK_NAME_LIST + value: "{{ network_resources.keys()|list|join(',') }}" {% if workload_args.environments is defined %} {% for key, value in workload_args.environments.items() %} - - name: {{ key }} - value: "{{ value }}" + - name: {{ key }} + value: "{{ value }}" {% endfor %} {% endif %} - volumes: - - name: hugepage - emptyDir: - medium: HugePages - restartPolicy: Never - serviceAccountName: benchmark-operator -{% include "metadata.yml.j2" %} + volumes: + - name: hugepage + emptyDir: + medium: HugePages + restartPolicy: Never + serviceAccountName: benchmark-operator +{% include "metadata_pod.yml.j2" %} From 8a755148ebdd202b87f3edb34e87f20f81265e1d Mon Sep 17 00:00:00 2001 From: ebattat Date: Tue, 7 Sep 2021 22:51:30 +0300 Subject: [PATCH 137/249] add uuid to start --- roles/uperf/tasks/send_client_run_signal.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/uperf/tasks/send_client_run_signal.yml b/roles/uperf/tasks/send_client_run_signal.yml index ce833492a..756274cf7 100644 --- a/roles/uperf/tasks/send_client_run_signal.yml +++ b/roles/uperf/tasks/send_client_run_signal.yml @@ -3,7 +3,7 @@ - block: - name: Signal workload - command: "redis-cli set start true" + command: "redis-cli set start-{{trunc_uuid}} true" - name: A26 Update resource state operator_sdk.util.k8s_status: From 2f834bd9b3b144583b7fdc612a591ccaf3ddb93a Mon Sep 17 00:00:00 2001 From: Ben England Date: Wed, 8 Sep 2021 04:09:54 -0400 Subject: [PATCH 138/249] Smallfile cr params (#652) * add auto-pause related params, allow all params from CR * output smallfile per-thread logs in debug mode * remove unusable smallfile params, add missing * clarify smallfile example, point to documentation * add spaces to improve consistency, readability remove unsupported parameter * improve consistency, readability, and robustness do not assume clustername is defined in CR use redis_host with publisher to clarify meaning add spaces inside double squiggly brackets for readability * test auto-pause param in CI * redis_host not defined to jinja2 * update and link to github smallfile repo doc do not exhaustively track all parameters here, link to where they are defined, and indicate which ones NOT to specify for benchmark-operator also give a more realistic example log * oops on doc * more concise defaulting, use single-quote inside double-quote * pause and auto-pause are mutually exclusive --- config/samples/smallfile/cr.yaml | 12 +- docs/smallfile.md | 198 ++++++------------ .../smallfilejob.other_parameters.yaml.j2 | 62 +++--- roles/smallfile/templates/workload_job.yml.j2 | 25 ++- tests/test_crs/valid_smallfile.yaml | 3 +- 5 files changed, 119 insertions(+), 181 deletions(-) diff --git a/config/samples/smallfile/cr.yaml b/config/samples/smallfile/cr.yaml index 427f4536b..90ad63f6b 100644 --- a/config/samples/smallfile/cr.yaml +++ b/config/samples/smallfile/cr.yaml @@ -5,17 +5,23 @@ metadata: namespace: benchmark-operator spec: test_user: homer_simpson - clustername: aws-2009-09-10 + clustername: nuclear-power-plant elasticsearch: url: http://my.elasticsearch.server:9200 workload: name: smallfile args: + #drop_cache_kernel: true + #drop_cache_rook_ceph: true + #storageclass: my-kubernetes-storageclass + storagesize: 2Gi clients: 1 samples: 1 operation: ["create", "read"] threads: 5 file_size: 64 files: 100 - storageclass: rook-ceph-block - storagesize: 2Gi + # more parameters available, see smallfile documentation + # at https://github.com/distributed-system-analysis/smallfile/blob/master/README.md + # subset of parameters supported for benchmark-operator are in + # roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 diff --git a/docs/smallfile.md b/docs/smallfile.md index b4ca9c627..de8cdb795 100644 --- a/docs/smallfile.md +++ b/docs/smallfile.md @@ -5,104 +5,28 @@ ## Running Smallfile Benchmark using Ripsaw Once the operator has been installed following the instructions, one needs to modify the clients parameter(which is currently set to 0), to value greater than 0 in [cr.yaml](../config/samples/smallfile/cr.yaml) to run default "create" the test. Also, in addtion to that, smallfile operator is completely dependent on storageclass and storagesize. Please make sure to double check the parameters in CRD file. -```yaml -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: example-benchmark - namespace: benchmark-operator -spec: - workload: - name: smallfile - args: - clients: 1 - operation: ["create"] - threads: 5 - file_size: 64 - files: 100 - storageclass: rook-ceph-block # Provide if PV is needed - storagesize: 5Gi # Provide if PV is needed -``` - -Smallfile operator also gives the leverage to run multiple test operations in a user-defined sequence. Like in the [Custom Resource Definition file](../config/samples/smallfile/cr.yaml), the series of operation can be mentioned as: +Smallfile operator also gives the leverage to run multiple test operations in a user-defined sequence. Like in the [Custom Resource Definition file](../config/samples/smallfile/cr.yaml), the series of operations can be specified as a list. -```yaml -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: example-benchmark - namespace: benchmark-operator -spec: - workload: - name: smallfile - args: - clients: 1 - operation: ["create","read","append", "delete"] - threads: 5 - file_size: 64 - files: 100 - storageclass: rook-ceph-block # Provide if PV is needed - storagesize: 5Gi # Provide if PV is needed -``` +NOTE: While running the sequence of tests using smallfile workload, please make sure that the initial operation must be create, and the "cleanup" operation should come in termination, else smallfile might produce error due to meaningless sequence of tests. For example: -NOTE: While running the sequence of tests using smallfile workload, please make sure that the initial operation must be create, and the "delete" operation should come in termination, else smallfile might produce error due to meaningless sequence of tests. For example: ```bash -operation: ["read","append","create", "delete"] + +operation: ["read","append","create", "delete","cleanup"] #This will be meaningless sequence of test as trying to read something, which has not been created yet.The same logic applies for the append test as well. Hence, smallfile will produce error. ``` ## Adding More options in smallfile tests -Smallfile also comes with a variety of configurable options for running tests, following are the parameters that can be added to CR file: - - * **response_times** – if Y then save response time for each file operation in a - rsptimes\*csv file in the shared network directory. Record format is - operation-type, start-time, response-time. The operation type is included so - that you can run different workloads at the same time and easily merge the - data from these runs. The start-time field is the time that the file - operation started, down to microsecond resolution. The response time field is - the file operation duration down to microsecond resolution. - * **file_size_distribution** – only supported value today is exponential. - * **record_size** -- record size in KB, how much data is transferred in a single - read or write system call. If 0 then it is set to the minimum of the file - size and 1-MiB record size limit. - * **files_per_dir** -- maximum number of files contained in any one directory. - * **dirs_per_dir** -- maximum number of subdirectories contained in any one - directory. - * **hash_into_dirs** – if Y then assign next file to a directory using a hash - function, otherwise assign next –files-per-dir files to next directory. - * **same_dir** -- if Y then threads will share a single directory. - * **network_sync_dir** – don't need to specify unless you run a multi-host test - and the –top parameter points to a non-shared directory (see discussion - below). Default: network_shared subdirectory under –top dir. - * **permute_host_dirs** – if Y then have each host process a different - subdirectory tree than it otherwise would (see below for directory tree - structure). - * **xattr_size** -- size of extended attribute value in bytes (names begin with - 'user.smallfile-') - * **xattr_count** -- number of extended attributes per file - * **prefix** -- a string prefix to prepend to files (so they don't collide with -previous runs for example) - * **suffix** -- a string suffix to append to files (so they don't collide with - previous runs for example) - * **incompressible** – if Y then generate a pure-random file that - will not be compressible (useful for tests where intermediate network or file - copy utility attempts to compress data - * **record_ctime_size** -- if Y then label each created file with an - xattr containing a time of creation and a file size. This will be used by - –await-create operation to compute performance of asynchonous file - replication/copy. - * **finish** -- if Y, thread will complete all requested file operations even if - measurement has finished. - * **stonewall** -- if Y then thread will measure throughput as soon as it detects - that another thread has finished. - * **verify_read** – if Y then smallfile will verify read data is correct. - * **remote_pgm_dir** – don't need to specify this unless the smallfile software - lives in a different directory on the target hosts and the test-driver host. - * **pause** -- integer (microseconds) each thread will wait before starting next - file. - * **runtime_class** - If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. +Smallfile also comes with a variety of configurable options that can be added by the user to the CR for running tests, documented [at its github site here](https://github.com/distributed-system-analysis/smallfile#readme) . To obtain the YAML parameter name, remove the 2 preceding dashes and convert remaining dashes to underscores and append a colon character. For example, **--file-size** becomes **file_size:** . Parameters that **are not** usable in the smallfile CR include: +* --yaml-input-file - used by the benchmark +* --operation - you specify this in the CR in the operation list +* --top - you do not specify this, Kubernetes points smallfile to the PV mountpoint +* --response-times - used by the benchmark +* --output-json - used by the benchmark +* --network-sync-dir - not applicable for pods +* --permute-host-dirs - not applicable for pods +* --remote-pgm-dir - not applicable for pods Once done creating/editing the resource file, one can run it by: @@ -117,57 +41,55 @@ $ kubectl get pods NAME READY STATUS RESTARTS AGE benchmark-operator-7c6bc98b8c-2j5x5 2/2 Running 0 47s example-benchmark-smallfile-client-1-benchmark-hwj4b 1/1 Running 0 33s +example-benchmark-smallfile-publisher... ``` -To see the output of the run one has to run `kubectl logs `. This is shown below: +To see the output of the run one has to run `kubectl logs `. This looks *approximately* like: ```bash -$ kubectl logs example-benchmark-smallfile-client-1-benchmark-hwj4b -create -RUnning_OPERATION_create -top: /mnt/pvc/smallfile_test_data -threads: 5 -file-size: 64 -files: 100 - version : 3.1 - hosts in test : None - launch by daemon : False - top test directory(s) : ['/mnt/pvc/smallfile_test_data'] - operation : create - files/thread : 100 - threads : 5 - record size (KB, 0 = maximum) : 0 - file size (KB) : 64 - file size distribution : fixed - files per dir : 100 - dirs per dir : 10 - threads share directories? : N - filename prefix : - filename suffix : - hash file number into dir.? : N - fsync after modify? : N - pause between files (microsec) : 0 - minimum directories per sec : 50 - finish all requests? : Y - stonewall? : Y - measure response times? : N - verify read? : Y - verbose? : False - log to stderr? : False - ext.attr.size : 0 - ext.attr.count : 0 -host = example-benchmark-smallfile-client-1-benchmark-hwj4b,thr = 00,elapsed = 0.015719,files = 100,records = 100,status = ok -host = example-benchmark-smallfile-client-1-benchmark-hwj4b,thr = 01,elapsed = 0.020679,files = 100,records = 100,status = ok -host = example-benchmark-smallfile-client-1-benchmark-hwj4b,thr = 02,elapsed = 0.028791,files = 100,records = 100,status = ok -host = example-benchmark-smallfile-client-1-benchmark-hwj4b,thr = 03,elapsed = 0.029367,files = 100,records = 100,status = ok -host = example-benchmark-smallfile-client-1-benchmark-hwj4b,thr = 04,elapsed = 0.029750,files = 100,records = 100,status = ok -total threads = 5 -total files = 500 -total IOPS = 500 -total data = 0.031 GiB -100.00% of requested files processed, minimum is 70.00 -elapsed time = 0.030 -files/sec = 16806.661271 -IOPS = 16806.661271 -MiB/sec = 1050.41633 +$ kubectl logs example-benchmark-smallfile-client-1-benchmark-hwj4b +Waiting For all Smallfile Pods to get ready ... +Executing Smallfile... +2021-08-24T20:26:32Z - INFO - MainProcess - trigger_smallfile: running:smallfile_cli.py --operation create --top /mnt/pvc/smallfile_test_data --output-json /var/tmp/RESULTS/1/create.json --response-times Y --yaml-input-file /tmp/smallfile/smallfilejob +2021-08-24T20:26:32Z - INFO - MainProcess - trigger_smallfile: from current directory /opt/snafu + version : 3.2 + hosts in test : None + launch by daemon : False + top test directory(s) : ['/mnt/pvc/smallfile_test_data'] + operation : create + files/thread : 10000 + threads : 1 + record size (KB, 0 = maximum) : 0 + file size (KB) : 4 + file size distribution : fixed + files per dir : 100 + dirs per dir : 10 + threads share directories? : N + filename prefix : + filename suffix : + hash file number into dir.? : N + fsync after modify? : N + pause between files (microsec) : 0 + auto-pause? : N + delay after cleanup per file (microsec) : 0 + minimum directories per sec : 50 + total hosts : 30 + finish all requests? : Y + stonewall? : Y + measure response times? : Y + verify read? : Y + verbose? : False + log to stderr? : False +host = smallfile-client-1-benchmark-84ad212e-9h454,thr = 00,elapsed = 0.592771,files = 10000,records = 10000,status = ok +total threads = 1 +total files = 10000 +total IOPS = 16869 +total data = 0.038 GiB +100.00% of requested files processed, warning threshold is 70.00 +elapsed time = 0.593 +files/sec = 16869.919582 +IOPS = 16869.919582 +MiB/sec = 65.898123 +2021-08-24T20:26:37Z - INFO - MainProcess - trigger_smallfile: completed sample 1 for operation create , results in /var/tmp/RESULTS/1/create.json +... ``` diff --git a/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 b/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 index 5540a5fcb..05fda394c 100644 --- a/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 +++ b/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 @@ -1,6 +1,12 @@ - - top: {{ smallfile_path }}/smallfile_test_data + +{% if workload_args.auto_pause is defined %} +auto-pause: {{ workload_args.auto_pause }} +{% endif %} + +{% if workload_args.clients is defined %} +total-hosts: {{ workload_args.clients }} +{% endif %} {% if workload_args.threads is defined %} threads: {{ workload_args.threads }} @@ -15,81 +21,81 @@ files: {{ workload_args.files }} {% endif %} {% if workload_args.fsync is defined %} -fsync: y +fsync: {{ workload_args.fsync }} {% endif %} -{% if workload_args.response_times is defined %} -response-times: true -{% endif %} +# response-times is always set to Y so that +# benchmark-wrapper can collect response time data -{% if workload_args.network_sync_dir is defined %} -network-sync-dir: {{workload_args.network_sync_dir}} -{% endif %} +# network-sync-dir is never used with benchmark-operator +# since it uses redis to synchronize pods and elastic to return data {% if workload_args.files_per_dir is defined %} -files-per-dir: {{workload_args.files_per_dir}} +files-per-dir: {{ workload_args.files_per_dir }} {% endif %} {% if workload_args.dirs_per_dir is defined %} -dirs-per-dir: {{workload_args.dirs_per_dir}} +dirs-per-dir: {{ workload_args.dirs_per_dir }} {% endif %} {% if workload_args.record_size is defined %} -record-size: {{workload_args.record_size}} +record-size: {{ workload_args.record_size }} {% endif %} {% if workload_args.file_size_distribution is defined %} -file-size-distribution: {{workload_args.file_size_distribution}} +file-size-distribution: {{ workload_args.file_size_distribution }} {% endif %} {% if workload_args.xattr_size is defined %} -xattr-size: {{workload_args.xattr_size}} +xattr-size: {{ workload_args.xattr_size }} {% endif %} {% if workload_args.xattr_count is defined %} -xattr-count: {{workload_args.xattr_count}} +xattr-count: {{ workload_args.xattr_count }} {% endif %} {% if workload_args.pause is defined %} -pause: {{workload_args.pause}} +pause: {{ workload_args.pause }} {% endif %} {% if workload_args.stonewall is defined %} -stonewall: {{workload_args.stonewall}} +stonewall: {{ workload_args.stonewall }} {% endif %} {% if workload_args.finish is defined %} -finish: {{workload_args.finish}} +finish: {{ workload_args.finish }} {% endif %} {% if workload_args.prefix is defined %} -prefix: {{workload_args.prefix}} +prefix: {{ workload_args.prefix }} +{% endif %} + +{% if workload_args.suffix is defined %} +suffix: {{ workload_args.suffix }} {% endif %} {% if workload_args.hash_into_dirs is defined %} -hash-into-dirs: {{workload_args.hash_into_dirs}} +hash-into-dirs: {{ workload_args.hash_into_dirs }} {% endif %} {% if workload_args.same_dir is defined %} -same-dir: {{workload_args.same_dir}} +same-dir: {{ workload_args.same_dir }} {% endif %} {% if workload_args.verbose is defined %} -verbose: {{workload_args.verbose}} +verbose: {{ workload_args.verbose }} {% endif %} -{% if workload_args.permute_host_dirs is defined %} -permute-host-dirs: true -{% endif %} +# permute-host-dirs is not applicable to K8S pods {% if workload_args.record_ctime_size is defined %} -record-ctime-size: true +record-ctime-size: {{ workload_args.record_ctime_size }} {% endif %} {% if workload_args.verify_read is defined %} -verify-read: true +verify-read: {{ workload_args.verify_read }} {% endif %} {% if workload_args.incompressible is defined %} -incompressible: true +incompressible: {{ workload_args.incompressible }} {% endif %} diff --git a/roles/smallfile/templates/workload_job.yml.j2 b/roles/smallfile/templates/workload_job.yml.j2 index 322d0d7e5..fb773dd35 100644 --- a/roles/smallfile/templates/workload_job.yml.j2 +++ b/roles/smallfile/templates/workload_job.yml.j2 @@ -43,9 +43,9 @@ spec: - name: uuid value: "{{ uuid }}" - name: test_user - value: "{{ test_user | default("ripsaw") }}" + value: "{{ test_user | default('ripsaw') }}" - name: clustername - value: "{{ clustername }}" + value: "{{ clustername | default('mycluster') }}" {% if item | int == 1 %} {% if kcache_drop_pod_ips is defined %} - name: kcache_drop_pod_ips @@ -64,7 +64,7 @@ spec: - name: es value: "{{ elasticsearch.url }}" - name: es_index - value: "{{ elasticsearch.index_name | default("ripsaw-smallfile") }}" + value: "{{ elasticsearch.index_name | default('ripsaw-smallfile') }}" - name: parallel value: "{{ elasticsearch.parallel | default(false) }}" - name: es_verify_cert @@ -87,25 +87,28 @@ spec: - name: clients value: "{{ workload_args.clients|default('1')|int }}" args: - - python /tmp/smallfile/subscriber {{bo.resources[0].status.podIP}} {{trunc_uuid}}; + - python /tmp/smallfile/subscriber {{ bo.resources[0].status.podIP }} {{ trunc_uuid }}; export TMPDIR=/tmp - arr=$(python <<< "print(','.join({{workload_args.operation}}))"); - rm -rf {{smallfile_path}}/RESULTS ; - mkdir -p {{smallfile_path}}/RESULTS; - mkdir -p {{smallfile_path}}/smallfile_test_data; + arr=$(python <<< "print(','.join({{ workload_args.operation }}))"); + rm -rf {{ smallfile_path }}/RESULTS ; + mkdir -p {{ smallfile_path }}/RESULTS; + mkdir -p {{ smallfile_path }}/smallfile_test_data; run_snafu {% if workload_args.samples is defined %} - --samples {{workload_args.samples}} + --samples {{ workload_args.samples }} {% endif %} --tool smallfile --operations $arr - --top {{smallfile_path}}/smallfile_test_data + --top {{ smallfile_path }}/smallfile_test_data --dir /var/tmp/RESULTS --yaml-input-file /tmp/smallfile/smallfilejob {% if workload_args.debug is defined and workload_args.debug %} -v {% endif %} ; +{% if workload_args.debug is defined and workload_args.debug %} + ls /tmp ; cat /tmp/invoke*log ; +{% endif %} if [ $? = 0 ] ; then echo RUN STATUS DONE ; else exit 1 ; fi volumeMounts: - name: config-volume @@ -123,7 +126,7 @@ spec: {% if workload_args.storageclass is defined %} - name: data-volume persistentVolumeClaim: - claimName: "claim{{item}}-{{ trunc_uuid }}" + claimName: "claim{{ item }}-{{ trunc_uuid }}" {% elif hostpath is defined %} - name: data-volume hostPath: diff --git a/tests/test_crs/valid_smallfile.yaml b/tests/test_crs/valid_smallfile.yaml index 0541aa915..afe49c9bd 100644 --- a/tests/test_crs/valid_smallfile.yaml +++ b/tests/test_crs/valid_smallfile.yaml @@ -28,7 +28,8 @@ spec: file_size: 2 record_size: 1 xattr_size: 50 - pause: 2 + pause: 0 + auto_pause: y stonewall: n finish: y prefix: abc From a5fdb1b625eacae2d4c7e78b3469408d4b2bdba2 Mon Sep 17 00:00:00 2001 From: jeniferh Date: Wed, 8 Sep 2021 10:49:48 -0400 Subject: [PATCH 139/249] Update CONTRIBUTING.md Add changes since v0.1 to update the operator image --- CONTRIBUTING.md | 36 +++++------------------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3dae03a37..650b4254d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -121,41 +121,15 @@ case of failure or when disabled. This ensures no interference with subsequent w ### The operator container image Any changes to the [roles](roles/) tree or to the [playbook](playbook.yml) file will necessitate a new build of the operator container image. -The benchmark-operator container can be built using [podman](https://podman.io/) and pushed to a public repository. -The public repository could be [quay](https://quay.io) in which case you'll need to: +The benchmark-operator container can be built and pushed to a public repository, which could be [quay](https://quay.io), using the provided Makefile. -```bash -$ podman build -f build/Dockerfile -t quay.io//benchmark-operator:testing . - -$ podman push quay.io//benchmark-operator:testing -``` - -Note: you can also use docker, and no, we'll not judge ;) - -`:testing` is simply a tag. You can define different tags to use with your image, like `:latest` or `:master` - -To test with your own operator image, you will need the [operator](resources/operator.yml) file to point the container image to your testing version. -Be sure to do this outside of your git tree to avoid mangling the official file that points to our stable image. - -This can be done as follows: +To test with your own operator image, you will need to delete the current deployment and then build, push, and redeploy using your operator container image as follows: ```bash -$ sed 's/image:.*/image: quay.io\/\/benchmark-operator:testing/' resources/operator.yaml > /my/testing/operator.yaml -``` - -You can then redeploy operator -```bash -# kubectl delete -f resources/operator.yaml -# kubectl apply -f /my/testing/operator.yaml -``` -Redefine CRD -```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -``` -Apply a new CR -```bash -# kubectl apply -f config/samples/uperf/cr.yaml +# kubectl delete deployment -n benchmark-operator benchmark-controller-manager +# make image-build image-push deploy IMG=quay.io//benchmark-operator:testing ``` +`:testing` is simply a tag. You can define different tags to use with your image, like `:latest` or `:master` ## CI Currently we have a CI that runs against PRs. From 2da6f53788ccafc9b9e73d1b6171ca06fc4328f7 Mon Sep 17 00:00:00 2001 From: Marko Karg Date: Mon, 13 Sep 2021 11:45:49 +0200 Subject: [PATCH 140/249] fixed cyclictest execution --- roles/cyclictest/templates/cyclictestjob.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index 19e21aa20..55d08865c 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -19,9 +19,9 @@ spec: image: {{ workload_args.image | default('quay.io/cloud-bulldozer/cyclictest:latest') }} command: ["/bin/sh", "-c"] args: - - run_snafu --tool cyclictest -p /tmp/cyclictest.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ + - run_snafu --tool cyclictest -p /tmp/cyclictest.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}} {% if workload_args.debug is defined and workload_args.debug %} - -v \ + -v {% endif %} ; imagePullPolicy: Always From eb8be371d4285289f966a1b503c8e5584d9d8954 Mon Sep 17 00:00:00 2001 From: morenod Date: Fri, 10 Sep 2021 11:20:04 +0200 Subject: [PATCH 141/249] add rosa parameters to scale template --- docs/scale_openshift.md | 24 +++++++++++++++++++++++ roles/scale_openshift/templates/scale.yml | 11 +++++++++++ 2 files changed, 35 insertions(+) diff --git a/docs/scale_openshift.md b/docs/scale_openshift.md index a393b7e7d..ab2306c7b 100644 --- a/docs/scale_openshift.md +++ b/docs/scale_openshift.md @@ -28,6 +28,22 @@ Optional variables: `runtime_class` If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. +`rosa` For clusters installed using ROSA. Following parameters will be required: + +`rosa.cluster_name` Name of cluster as it is shown on `rosa list clusters` command + +`rosa.env` ROSA environment where cluster is installed. + +`rosa.token` ROSA token required to execute commands + +`aws` AWS credentials required by ROSA cli to execute scalation commands + +`aws.access_key_id` Exported as AWS_ACCESS_KEY_ID + +`aws.secret_access_key` Exported as AWS_SECRET_ACCESS_KEY + +`aws.default_region` Exported as AWS_DEFAULT_REGION + Your resource file may look like this when using all avaible options: ```yaml @@ -54,6 +70,14 @@ spec: key: role value: workload effect: NoSchedule + rosa: + cluster_name: rosa-test-01 + env: staging + token: "xxxx" + aws: + access_key_id: XXXXX + secret_access_key: XXXXXX + default_region: us-west-2 ``` *NOTE:* If the cluster is already at the desired scale the timings will still be captured and uploaded to diff --git a/roles/scale_openshift/templates/scale.yml b/roles/scale_openshift/templates/scale.yml index 092c384f1..73661ec99 100755 --- a/roles/scale_openshift/templates/scale.yml +++ b/roles/scale_openshift/templates/scale.yml @@ -63,12 +63,23 @@ spec: value: "{{ prometheus.prom_token | default() }}" - name: prom_url value: "{{ prometheus.prom_url | default() }}" +{% endif %} +{% if workload_args.aws is defined %} + - name: AWS_ACCESS_KEY_ID + value: "{{ workload_args.aws.access_key_id | default() }}" + - name: AWS_SECRET_ACCESS_KEY + value: "{{ workload_args.aws.secret_access_key | default() }}" + - name: AWS_DEFAULT_REGION + value: "{{ workload_args.aws.default_region | default() }}" {% endif %} command: ["/bin/sh", "-c"] args: - "run_snafu --tool scale --scale {{workload_args.scale}} -u {{uuid}} --user {{test_user|default("ripsaw")}} --incluster true --poll_interval {{workload_args.poll_interval|default(5)}} \ {% if workload_args.debug is defined and workload_args.debug %} -v \ +{% endif %} +{% if workload_args.rosa is defined %} + --rosa-cluster {{workload_args.rosa.cluster_name}} --rosa-env {{workload_args.rosa.env}} --rosa-token {{workload_args.rosa.token}} \ {% endif %} ; sleep {{post_sleep|default(0)}}" From 237863eae96c00a36965df2fc77172823f31eb23 Mon Sep 17 00:00:00 2001 From: Ben England Date: Tue, 14 Sep 2021 04:09:39 -0400 Subject: [PATCH 142/249] Cleanup delay pass to smf (#680) Make CI test cleanup_delay_usec_per_file --- .../templates/smallfilejob.other_parameters.yaml.j2 | 5 +++++ tests/test_crs/valid_smallfile.yaml | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 b/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 index 05fda394c..666e8153b 100644 --- a/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 +++ b/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 @@ -99,3 +99,8 @@ verify-read: {{ workload_args.verify_read }} {% if workload_args.incompressible is defined %} incompressible: {{ workload_args.incompressible }} {% endif %} + +{% if workload_args.cleanup_delay_usec_per_file is defined %} +cleanup-delay-usec-per-file: {{ workload_args.cleanup_delay_usec_per_file }} +{% endif %} + diff --git a/tests/test_crs/valid_smallfile.yaml b/tests/test_crs/valid_smallfile.yaml index afe49c9bd..f37b85731 100644 --- a/tests/test_crs/valid_smallfile.yaml +++ b/tests/test_crs/valid_smallfile.yaml @@ -23,7 +23,7 @@ spec: name: smallfile args: clients: 2 - operation: ["create", "read", "append", "delete"] + operation: ["create", "read", "delete", "cleanup"] threads: 1 file_size: 2 record_size: 1 @@ -42,5 +42,5 @@ spec: files_per_dir: 2000 dirs_per_dir: 4 files: 100000 - cleanup_delay_usec_per_file: 200 + cleanup_delay_usec_per_file: 20 debug: true From 58da6234cbd15118721c4d451c6415599b75b263 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 15 Sep 2021 13:27:07 +0200 Subject: [PATCH 143/249] Improve vegeta synchronization Signed-off-by: Raul Sevilla --- roles/vegeta/templates/vegeta.yml.j2 | 14 +++++--------- tests/test_crs/valid_vegeta.yaml | 4 ++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/roles/vegeta/templates/vegeta.yml.j2 b/roles/vegeta/templates/vegeta.yml.j2 index e9cea4300..d0ed029b5 100644 --- a/roles/vegeta/templates/vegeta.yml.j2 +++ b/roles/vegeta/templates/vegeta.yml.j2 @@ -72,21 +72,17 @@ spec: args: - | {% for t in workload_args.targets %} - redis-cli -h {{ bo.resources[0].status.podIP }} set vegeta-{{ item }}-{{ trunc_uuid }} ready - ready=0 - while [[ ${ready} == 0 ]]; do - for client in $(seq 0 {{ workload_args.clients-1 }}); do - [[ `redis-cli -h {{ bo.resources[0].status.podIP }} get vegeta-${client}-{{ trunc_uuid }}` != "ready" ]] && ready=0 && break - ready=1 - done + redis-cli -h {{ bo.resources[0].status.podIP }} incr vegeta-{{ trunc_uuid }} + echo "Waiting for {{ workload_args.clients-1 }} clients to be ready" + while [[ $(redis-cli -h {{ bo.resources[0].status.podIP }} get vegeta-{{ trunc_uuid }}) != {{ workload_args.clients }} ]]; do + sleep 0.1 done - sleep 0.1 - redis-cli -h {{ bo.resources[0].status.podIP }} set vegeta-{{ item }}-{{ trunc_uuid }} notready run_snafu --tool vegeta --targets /tmp/vegeta/{{ t.name|replace(" ","") }} -u ${uuid} -d {{ t.duration }} --user ${test_user} -w {{ t.workers|default(1) }} -s {{ t.samples|default(1) }} {{ "--keepalive" if t.keepalive|default(false) }} \ {% if workload_args.debug is defined and workload_args.debug %} -v \ {% endif %} ; + redis-cli -h {{ bo.resources[0].status.podIP }} decr vegeta-{{ trunc_uuid }} {% endfor %} volumeMounts: - name: targets-volume diff --git a/tests/test_crs/valid_vegeta.yaml b/tests/test_crs/valid_vegeta.yaml index 14a1d6fdc..4d9e79f5e 100644 --- a/tests/test_crs/valid_vegeta.yaml +++ b/tests/test_crs/valid_vegeta.yaml @@ -19,13 +19,13 @@ spec: workload: name: vegeta args: - clients: 2 + clients: 3 targets: - name: 2w-ka urls: - GET https://1.1.1.1 - GET http://1.1.1.1 - samples: 1 + samples: 3 workers: 2 duration: 5 keepalive: true From 6f4c1431450959c4484641dd6b4bf68bd3d1e3be Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Wed, 1 Sep 2021 17:52:01 +0200 Subject: [PATCH 144/249] Adds initial config for api-load tests Signed-off-by: Vicente Zepeda Mas --- roles/api_load/tasks/main.yml | 20 +++++++ roles/api_load/templates/ocm_load.yml | 78 +++++++++++++++++++++++++++ tests/test_crs/valid_api_load.yaml | 21 ++++++++ 3 files changed, 119 insertions(+) create mode 100644 roles/api_load/tasks/main.yml create mode 100644 roles/api_load/templates/ocm_load.yml create mode 100644 tests/test_crs/valid_api_load.yaml diff --git a/roles/api_load/tasks/main.yml b/roles/api_load/tasks/main.yml new file mode 100644 index 000000000..87f7bd9d5 --- /dev/null +++ b/roles/api_load/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- block: + + - name: Image pull pods + k8s: + state: present + definition: "{{ lookup('template', 'api_load.yml') }}" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running + + when: benchmark_state.resources[0].status.state == "Building" + +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/api_load/templates/ocm_load.yml b/roles/api_load/templates/ocm_load.yml new file mode 100644 index 000000000..4c8ff63eb --- /dev/null +++ b/roles/api_load/templates/ocm_load.yml @@ -0,0 +1,78 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: 'api-load-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' +spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} + parallelism: {{ workload_args.pod_count | default(1) | int }} + template: + metadata: + labels: + app: api-load-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.tolerations is defined %} + tolerations: + - key: {{ workload_args.tolerations.key }} + value: {{ workload_args.tolerations.value }} + effect: {{ workload_args.tolerations.effect }} +{% endif %} +{% if workload_args.label is defined %} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ workload_args.label.key }} + operator: In + values: + - {{ workload_args.label.value }} +{% endif %} + containers: + - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/ocm-api-load:latest') }} + name: pull-image + env: + - name: my_node_name + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: my_pod_name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: uuid + value: "{{ uuid }}" +{% if elasticsearch is defined %} + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("image-pull") }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(true) }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" +{% endif %} + command: ["/bin/sh", "-c"] + args: + - > + ocm-load-test + --test-id={{ uuid }} + --gateway-url {{ workload_args.gateway_url }} + --ocm-token={{ workload_args.ocm_token }} + --duration={{ workload_args.duration}} + --rate={{ workload_args.rate}} + --output-path={{ workload_args.output_path}} + --test-names {{ workload_args.test_name }} + --aws-access-key {{ workload_args.aws_access_key }} + --aws-access-secret {{ workload_args.aws_access_secret }} + --aws-account-id {{ workload_args.aws_account_id }} + + imagePullPolicy: Always + restartPolicy: Never +{% include "metadata.yml.j2" %} diff --git a/tests/test_crs/valid_api_load.yaml b/tests/test_crs/valid_api_load.yaml new file mode 100644 index 000000000..a5221da4a --- /dev/null +++ b/tests/test_crs/valid_api_load.yaml @@ -0,0 +1,21 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: api-load + namespace: benchmark-operator +spec: + elasticsearch: + url: ES_SERVER + index_name: "api-load" + workload: + name: api_load + args: + gateway_url: https://api.integration.openshift.com + ocm_token: definitelyArealToken-Here + duration: 1 + rate: 5/s + output_path: /results + test_names: list-clusters + aws_access_key: empty + aws_access_secret: empty + aws_account_id: empty From 1f791db0f3d51aa4e89db164f527688f44bc5d78 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Wed, 8 Sep 2021 15:40:09 +0200 Subject: [PATCH 145/249] API load test execution Signed-off-by: Vicente Zepeda Mas --- config/manager/kustomization.yaml | 2 +- roles/api_load/tasks/main.yml | 2 +- .../templates/{ocm_load.yml => api_load.yml} | 0 tests/test_api_load.sh | 43 +++++++++++++++++++ 4 files changed, 45 insertions(+), 2 deletions(-) rename roles/api_load/templates/{ocm_load.yml => api_load.yml} (100%) create mode 100755 tests/test_api_load.sh diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index f6241927a..6a0c6fd37 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: quay.io/cloud-bulldozer/benchmark-operator + newName: quay.io/chentex/benchmark-operator newTag: latest diff --git a/roles/api_load/tasks/main.yml b/roles/api_load/tasks/main.yml index 87f7bd9d5..33f6c6e4f 100644 --- a/roles/api_load/tasks/main.yml +++ b/roles/api_load/tasks/main.yml @@ -1,7 +1,7 @@ --- - block: - - name: Image pull pods + - name: Api load pods k8s: state: present definition: "{{ lookup('template', 'api_load.yml') }}" diff --git a/roles/api_load/templates/ocm_load.yml b/roles/api_load/templates/api_load.yml similarity index 100% rename from roles/api_load/templates/ocm_load.yml rename to roles/api_load/templates/api_load.yml diff --git a/tests/test_api_load.sh b/tests/test_api_load.sh new file mode 100755 index 000000000..1c28b0e03 --- /dev/null +++ b/tests/test_api_load.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up API load Test" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_api_load { + test_name=$1 + cr=$2 + delete_benchmark $cr + echo "Performing: ${test_name}" + kubectl apply -f ${cr} + benchmark_name=$(get_benchmark_name $cr) + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + check_benchmark_for_desired_state $benchmark_name Complete 500s + + index="ocm-requests" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_api_load "API load" tests/test_crs/valid_api_load.yaml From 58b89c28af2a2df6cd638e704bdb085cd1d27da2 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Tue, 14 Sep 2021 15:21:45 +0200 Subject: [PATCH 146/249] Adds volume for test results Signed-off-by: Vicente Zepeda Mas --- config/manager/kustomization.yaml | 2 +- roles/api_load/tasks/main.yml | 2 +- roles/api_load/templates/api_load.yml | 14 +++++++++++--- tests/test_crs/valid_api_load.yaml | 4 ++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 6a0c6fd37..f6241927a 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: quay.io/chentex/benchmark-operator + newName: quay.io/cloud-bulldozer/benchmark-operator newTag: latest diff --git a/roles/api_load/tasks/main.yml b/roles/api_load/tasks/main.yml index 33f6c6e4f..a5f90c980 100644 --- a/roles/api_load/tasks/main.yml +++ b/roles/api_load/tasks/main.yml @@ -4,7 +4,7 @@ - name: Api load pods k8s: state: present - definition: "{{ lookup('template', 'api_load.yml') }}" + definition: "{{ lookup('template', 'api_load.yml') | from_yaml }}" - include_role: name: benchmark_state diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index 4c8ff63eb..9acc43668 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -52,7 +52,7 @@ spec: - name: es value: "{{ elasticsearch.url }}" - name: es_index - value: "{{ elasticsearch.index_name | default("image-pull") }}" + value: "{{ elasticsearch.index_name | default("api-load") }}" - name: es_verify_cert value: "{{ elasticsearch.verify_cert | default(true) }}" - name: parallel @@ -71,8 +71,16 @@ spec: --test-names {{ workload_args.test_name }} --aws-access-key {{ workload_args.aws_access_key }} --aws-access-secret {{ workload_args.aws_access_secret }} - --aws-account-id {{ workload_args.aws_account_id }} - + --aws-account-id {{ workload_args.aws_account_id }}; + volumeMounts: + - mountPath: /tmp/results + name: results + volumes: + - name: results + hostDisk: + path: /tmp/results + capacity: 10Gi + type: DiskOrCreate imagePullPolicy: Always restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/tests/test_crs/valid_api_load.yaml b/tests/test_crs/valid_api_load.yaml index a5221da4a..642120447 100644 --- a/tests/test_crs/valid_api_load.yaml +++ b/tests/test_crs/valid_api_load.yaml @@ -14,8 +14,8 @@ spec: ocm_token: definitelyArealToken-Here duration: 1 rate: 5/s - output_path: /results - test_names: list-clusters + output_path: /tmp/results + test_name: list-clusters aws_access_key: empty aws_access_secret: empty aws_account_id: empty From d3046b691971ae8b3d70fa13222ff62d667d52c2 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Tue, 14 Sep 2021 16:43:08 +0200 Subject: [PATCH 147/249] Fixing tests for api-load benchmark Signed-off-by: Vicente Zepeda Mas --- roles/api_load/templates/api_load.yml | 4 ++++ tests/test_api_load.sh | 8 -------- tests/test_crs/valid_api_load.yaml | 10 +--------- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index 9acc43668..1cb6e94d1 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -62,6 +62,9 @@ spec: args: - > ocm-load-test +{% if workload_args.override is defined %} + {{ workload_args.override }} +{% else %} --test-id={{ uuid }} --gateway-url {{ workload_args.gateway_url }} --ocm-token={{ workload_args.ocm_token }} @@ -72,6 +75,7 @@ spec: --aws-access-key {{ workload_args.aws_access_key }} --aws-access-secret {{ workload_args.aws_access_secret }} --aws-account-id {{ workload_args.aws_account_id }}; +{% endif %} volumeMounts: - mountPath: /tmp/results name: results diff --git a/tests/test_api_load.sh b/tests/test_api_load.sh index 1c28b0e03..2e0fceb89 100755 --- a/tests/test_api_load.sh +++ b/tests/test_api_load.sh @@ -28,14 +28,6 @@ function functional_test_api_load { check_benchmark_for_desired_state $benchmark_name Complete 500s - index="ocm-requests" - if check_es "${long_uuid}" "${index}" - then - echo "${test_name} test: Success" - else - echo "Failed to find data for ${test_name} in ES" - exit 1 - fi delete_benchmark $cr } diff --git a/tests/test_crs/valid_api_load.yaml b/tests/test_crs/valid_api_load.yaml index 642120447..d49a497a1 100644 --- a/tests/test_crs/valid_api_load.yaml +++ b/tests/test_crs/valid_api_load.yaml @@ -10,12 +10,4 @@ spec: workload: name: api_load args: - gateway_url: https://api.integration.openshift.com - ocm_token: definitelyArealToken-Here - duration: 1 - rate: 5/s - output_path: /tmp/results - test_name: list-clusters - aws_access_key: empty - aws_access_secret: empty - aws_account_id: empty + override: version From 4c6424d138c6c206ef33506a9f5befd8c245ecaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Fri, 17 Sep 2021 13:25:36 +0200 Subject: [PATCH 148/249] E2E testing framework (#625) E2e testing framework --- .github/workflows/e2e.yml | 84 +++++ .gitignore | 7 +- Makefile | 18 ++ README.md | 3 + config/manager/manager.yaml | 2 +- docs/e2e-ci.md | 70 +++++ e2e/001-scale_openshift.bats | 47 +++ e2e/002-sysbench.bats | 24 ++ e2e/003-byowl.bats | 32 ++ e2e/004-backpack.bats | 40 +++ e2e/005-stressng.bats | 25 ++ e2e/006-vegeta.bats | 34 ++ e2e/007-smallfile.bats | 34 ++ e2e/008-kube-burner.bats | 76 +++++ e2e/009-iperf3.bats | 23 ++ e2e/010-fio.bats | 64 ++++ e2e/011-ycsb.bats | 33 ++ e2e/012-flent.bats | 35 +++ e2e/013-uperf.bats | 61 ++++ e2e/014-pgbench.bats | 35 +++ e2e/015-image_pull.bats | 25 ++ e2e/016-hammerdb.bats | 34 ++ e2e/017-fs_drift.bats | 35 +++ e2e/018-log-generator.bats | 25 ++ e2e/019-api_load.bats | 22 ++ .../api_load/api_load.yaml | 5 +- .../backpack/backpack-daemonset.yaml | 4 +- .../backpack/backpack-init.yaml | 4 +- .../byowl/byowl-not-targeted.yaml | 11 +- e2e/byowl/byowl-targeted.yaml | 23 ++ .../valid_fiod.yaml => e2e/fio/fio.yaml | 12 +- .../fio/fio_bsrange.yaml | 14 +- .../fio/fio_hostpath.yaml | 13 +- .../fio/fio_ocs_cache_drop.yaml | 14 +- .../valid_flent.yaml => e2e/flent/flent.yaml | 9 +- .../flent/flent_resources.yaml | 11 +- .../fs_drift/fs_drift.yaml | 8 +- .../fs_drift/fs_drift_hostpath.yaml | 8 +- .../hammerdb/hammerdb.yaml | 8 +- .../hammerdb/sql-server.yaml | 0 e2e/helpers.bash | 91 ++++++ .../image_pull/image_pull.yaml | 8 +- .../iperf3/iperf3.yaml | 6 +- e2e/kube-burner/cluster-density.yaml | 39 +++ .../kube-burner/concurrent-builds.yaml | 13 +- e2e/kube-burner/max-namespaces.yaml | 39 +++ e2e/kube-burner/max-services.yaml | 39 +++ e2e/kube-burner/node-density-heavy.yaml | 39 +++ e2e/kube-burner/node-density.yaml | 39 +++ .../log-generator/log_generator.yaml | 4 +- .../pgbench/pgbench.yaml | 8 +- e2e/pgbench/postgres.yaml | 41 +++ .../scale-openshift/scale_down.yaml | 6 +- .../scale-openshift/scale_up.yaml | 8 +- .../smallfile/smallfile.yaml | 8 +- .../smallfile/smallfile_hostpath.yaml | 10 +- .../stressng/stressng.yaml | 8 +- .../sysbench/sysbench.yaml | 6 +- .../valid_uperf.yaml => e2e/uperf/uperf.yaml | 10 +- e2e/uperf/uperf_hostnetwork.yaml | 38 +++ .../uperf/uperf_networkpolicy.yaml | 15 +- .../uperf/uperf_resources.yaml | 17 +- .../uperf/uperf_serviceip.yaml | 14 +- .../vegeta/vegeta.yaml | 10 +- .../vegeta/vegeta_hostnetwork.yaml | 7 +- e2e/ycsb/mongo.yaml | 42 +++ .../ycsb/ycsb-mongo.yaml | 6 +- hack/install-bats.sh | 20 ++ roles/fs-drift/templates/workload_job.yml.j2 | 2 +- roles/smallfile/templates/workload_job.yml.j2 | 2 +- run_test.sh | 58 ---- test.sh | 140 --------- tests/check_es.py | 61 ---- tests/common.sh | 297 ------------------ tests/full_test_file_trigger | 7 - tests/test_api_load.sh | 35 --- tests/test_backpack.sh | 51 --- tests/test_byowl.sh | 32 -- tests/test_crs/valid_servicemesh.yaml | 16 - tests/test_fiod.sh | 67 ---- tests/test_flent.sh | 46 --- tests/test_fs_drift.sh | 46 --- tests/test_hammerdb.sh | 48 --- tests/test_image_pull.sh | 43 --- tests/test_iperf3.sh | 42 --- tests/test_kubeburner.sh | 57 ---- tests/test_list | 17 - tests/test_log_generator.sh | 45 --- tests/test_pgbench.sh | 98 ------ tests/test_scale_openshift.sh | 49 --- tests/test_servicemesh.sh | 57 ---- tests/test_smallfile.sh | 45 --- tests/test_stressng.sh | 46 --- tests/test_sysbench.sh | 37 --- tests/test_uperf.sh | 48 --- tests/test_vegeta.sh | 49 --- tests/test_ycsb.sh | 91 ------ 97 files changed, 1466 insertions(+), 1789 deletions(-) create mode 100644 .github/workflows/e2e.yml create mode 100644 docs/e2e-ci.md create mode 100755 e2e/001-scale_openshift.bats create mode 100755 e2e/002-sysbench.bats create mode 100755 e2e/003-byowl.bats create mode 100755 e2e/004-backpack.bats create mode 100755 e2e/005-stressng.bats create mode 100755 e2e/006-vegeta.bats create mode 100755 e2e/007-smallfile.bats create mode 100755 e2e/008-kube-burner.bats create mode 100755 e2e/009-iperf3.bats create mode 100755 e2e/010-fio.bats create mode 100755 e2e/011-ycsb.bats create mode 100755 e2e/012-flent.bats create mode 100755 e2e/013-uperf.bats create mode 100755 e2e/014-pgbench.bats create mode 100755 e2e/015-image_pull.bats create mode 100755 e2e/016-hammerdb.bats create mode 100755 e2e/017-fs_drift.bats create mode 100755 e2e/018-log-generator.bats create mode 100755 e2e/019-api_load.bats rename tests/test_crs/valid_api_load.yaml => e2e/api_load/api_load.yaml (80%) rename tests/test_crs/valid_backpack_daemonset.yaml => e2e/backpack/backpack-daemonset.yaml (84%) rename tests/test_crs/valid_backpack_init.yaml => e2e/backpack/backpack-init.yaml (88%) rename tests/test_crs/valid_byowl.yaml => e2e/byowl/byowl-not-targeted.yaml (69%) create mode 100644 e2e/byowl/byowl-targeted.yaml rename tests/test_crs/valid_fiod.yaml => e2e/fio/fio.yaml (93%) rename tests/test_crs/valid_fiod_bsrange.yaml => e2e/fio/fio_bsrange.yaml (92%) rename tests/test_crs/valid_fiod_hostpath.yaml => e2e/fio/fio_hostpath.yaml (92%) rename tests/test_crs/valid_fiod_ocs_cache_drop.yaml => e2e/fio/fio_ocs_cache_drop.yaml (92%) rename tests/test_crs/valid_flent.yaml => e2e/flent/flent.yaml (83%) rename tests/test_crs/valid_flent_resources.yaml => e2e/flent/flent_resources.yaml (78%) rename tests/test_crs/valid_fs_drift.yaml => e2e/fs_drift/fs_drift.yaml (86%) rename tests/test_crs/valid_fs_drift_hostpath.yaml => e2e/fs_drift/fs_drift_hostpath.yaml (87%) rename tests/test_crs/valid_hammerdb.yaml => e2e/hammerdb/hammerdb.yaml (94%) rename tests/mssql.yaml => e2e/hammerdb/sql-server.yaml (100%) mode change 100755 => 100644 create mode 100644 e2e/helpers.bash rename tests/test_crs/valid_image_pull.yaml => e2e/image_pull/image_pull.yaml (70%) rename tests/test_crs/valid_iperf3.yaml => e2e/iperf3/iperf3.yaml (89%) create mode 100644 e2e/kube-burner/cluster-density.yaml rename tests/test_crs/valid_kube-burner.yaml => e2e/kube-burner/concurrent-builds.yaml (85%) create mode 100644 e2e/kube-burner/max-namespaces.yaml create mode 100644 e2e/kube-burner/max-services.yaml create mode 100644 e2e/kube-burner/node-density-heavy.yaml create mode 100644 e2e/kube-burner/node-density.yaml rename tests/test_crs/valid_log_generator.yaml => e2e/log-generator/log_generator.yaml (86%) rename tests/test_crs/valid_pgbench.yaml => e2e/pgbench/pgbench.yaml (86%) create mode 100644 e2e/pgbench/postgres.yaml rename tests/test_crs/valid_scale_down.yaml => e2e/scale-openshift/scale_down.yaml (86%) rename tests/test_crs/valid_scale_up.yaml => e2e/scale-openshift/scale_up.yaml (82%) rename tests/test_crs/valid_smallfile.yaml => e2e/smallfile/smallfile.yaml (90%) rename tests/test_crs/valid_smallfile_hostpath.yaml => e2e/smallfile/smallfile_hostpath.yaml (85%) rename tests/test_crs/valid_stressng.yaml => e2e/stressng/stressng.yaml (87%) rename tests/test_crs/valid_sysbench.yaml => e2e/sysbench/sysbench.yaml (84%) rename tests/test_crs/valid_uperf.yaml => e2e/uperf/uperf.yaml (84%) create mode 100644 e2e/uperf/uperf_hostnetwork.yaml rename tests/test_crs/valid_uperf_networkpolicy.yaml => e2e/uperf/uperf_networkpolicy.yaml (76%) rename tests/test_crs/valid_uperf_resources.yaml => e2e/uperf/uperf_resources.yaml (77%) rename tests/test_crs/valid_uperf_serviceip.yaml => e2e/uperf/uperf_serviceip.yaml (77%) rename tests/test_crs/valid_vegeta.yaml => e2e/vegeta/vegeta.yaml (85%) rename tests/test_crs/valid_vegeta_hostnetwork.yaml => e2e/vegeta/vegeta_hostnetwork.yaml (85%) create mode 100644 e2e/ycsb/mongo.yaml rename tests/test_crs/valid_ycsb-mongo.yaml => e2e/ycsb/ycsb-mongo.yaml (90%) create mode 100755 hack/install-bats.sh delete mode 100755 run_test.sh delete mode 100755 test.sh delete mode 100755 tests/check_es.py delete mode 100755 tests/common.sh delete mode 100755 tests/full_test_file_trigger delete mode 100755 tests/test_api_load.sh delete mode 100755 tests/test_backpack.sh delete mode 100755 tests/test_byowl.sh delete mode 100644 tests/test_crs/valid_servicemesh.yaml delete mode 100755 tests/test_fiod.sh delete mode 100755 tests/test_flent.sh delete mode 100755 tests/test_fs_drift.sh delete mode 100755 tests/test_hammerdb.sh delete mode 100755 tests/test_image_pull.sh delete mode 100755 tests/test_iperf3.sh delete mode 100755 tests/test_kubeburner.sh delete mode 100755 tests/test_list delete mode 100755 tests/test_log_generator.sh delete mode 100755 tests/test_pgbench.sh delete mode 100755 tests/test_scale_openshift.sh delete mode 100755 tests/test_servicemesh.sh delete mode 100755 tests/test_smallfile.sh delete mode 100755 tests/test_stressng.sh delete mode 100755 tests/test_sysbench.sh delete mode 100755 tests/test_uperf.sh delete mode 100755 tests/test_vegeta.sh delete mode 100755 tests/test_ycsb.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..e67062530 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,84 @@ +name: Run e2e tests +on: + pull_request_target: + types: [labeled, synchronize] + push: + branches: + - master + workflow_dispatch: + inputs: + test_list: + description: Only run tests that match the regular expression + default: "" + img: + description: Benchmark-operator image location + default: quay.io/rht_perf_ci/benchmark-operator:e2e +jobs: + e2e: + name: E2E testing + runs-on: ubuntu-latest + timeout-minutes: 120 + if: "github.event_name == 'push' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ok to test'))" + env: + BATS_TESTS: ${{ github.event.inputs.test_list }} + IMG: ${{ github.event.inputs.img }} + steps: + + - name: Set env vars # Workaround: When the workflow is triggered by events different from workflow_dispatch the IMG env var is not set + run: echo "IMG=quay.io/rht_perf_ci/benchmark-operator:e2e" >> $GITHUB_ENV + if: github.event_name != 'workflow_dispatch' + + - name: Check out code + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + if: github.event_name == 'pull_request_target' + + - name: Check out code + uses: actions/checkout@v2 + if: "github.event_name == 'workflow_dispatch' || github.event_name == 'push'" + + - name: Install bats + run: make install-bats + + - name: Authenticate against OCP cluster + uses: redhat-actions/oc-login@v1 + with: + openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }} + openshift_username: ${{ secrets.OPENSHIFT_USER }} + openshift_password: ${{ secrets.OPENSHIFT_PASSWORD }} + insecure_skip_tls_verify: true + + - name: Login in quay + run: podman login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN} + env: + QUAY_USER: ${{ secrets.QUAY_CI_USER }} + QUAY_TOKEN: ${{ secrets.QUAY_CI_TOKEN }} + + - name: Build, push and deploy benchmark-operator + run: make image-build image-push deploy + + - name: Ensure benchmark-operator is running + run: kubectl rollout status -n benchmark-operator --timeout=5m deploy/benchmark-controller-manager + + - name: Execute e2e tests + run: make e2e-tests + env: + TERM: linux + + - name: Undeploy benchmark-operator + run: make undeploy + if: always() # always run even if the previous step fails + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + if: failure() + with: + name: benchmark-operator-debug-${{ github.run_id }} + path: e2e/artifacts + + - name: Publish Test Report + uses: mikepenz/action-junit-report@v2.4.3 + if: failure() # always run even if any of the previous steps fails + with: + report_paths: e2e/report.xml diff --git a/.gitignore b/.gitignore index ef9123ae7..fa0908090 100755 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ - # Binaries for programs and plugins *.dll *.so @@ -155,5 +154,7 @@ dmypy.json # pytype static type analyzer .pytype/ -# Cython debug symbols -cython_debug/ \ No newline at end of file +cython_debug/ +.cache +e2e/report.xml +e2e/artifacts diff --git a/Makefile b/Makefile index bb4701448..af0d3b64f 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,14 @@ else ENGINE := podman endif +# E2E testing +E2E_DIR = e2e +E2E_RESULTS = test-results.xml +BATS_FLAGS = -j 5 -F pretty --report-formatter junit -T +ifdef BATS_TESTS + FILTERED_TESTS := -f "$(BATS_TESTS)" +endif + all: image-build ##@ General @@ -198,3 +206,13 @@ catalog-build: opm ## Build a catalog image. .PHONY: catalog-push catalog-push: ## Push a catalog image. $(MAKE) image-push IMG=$(CATALOG_IMG) + +##@ Test + +.PHONY: e2e-tests +e2e-tests: ## Triggers e2e testing, by default all e2e/*.bats tests are executed. You can execute specific tests by setting a regix in the vabiable BATS_TESTS like: BATS_TESTS="fio|uperf|ycsb" make e2e-tests + cd $(E2E_DIR); bats $(BATS_FLAGS) $(FILTERED_TESTS) . + +.PHONY: install-bats +install-bats: + VERSION=v1.4.1 ./hack/install-bats.sh diff --git a/README.md b/README.md index 3271620eb..959f12668 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,9 @@ Why did we decide to switch to this? Our operator would implement long running t However, long running tasks blocks the Operator, causing us to delete the Operator and re-create the operator to un-block it. The benchmarks mentioned above that state `Used` for Reconciliation, no longer have this issue. +# E2E tests +Benchmark-operator includes a series of end 2 end tests that can be triggered in local. More info in the [documentation.](docs/e2e-ci.md#running-in-local) + ## Optional workload images Optional locations for workload images can now be added easily without the need to rebuild the operator. To do so in the workload args section of the CR add image: [location] diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d719ee940..e87d75501 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -45,7 +45,7 @@ spec: - name: OPERATOR_NAME value: "benchmark-operator" - name: MAX_CONCURRENT_RECONCILES_BENCHMARK_RIPSAW_CLOUDBULLDOZER_IO - value: "1" + value: "4" - name: ANSIBLE_VERBOSITY_BENCHMARK_RIPSAW_CLOUDBULLDOZER_IO value: "4" securityContext: diff --git a/docs/e2e-ci.md b/docs/e2e-ci.md new file mode 100644 index 000000000..d4193eabe --- /dev/null +++ b/docs/e2e-ci.md @@ -0,0 +1,70 @@ +# E2E CI + +## Introduction + +Benchmark-operator ships an end to end CI mechanism that runs by default in the events described in the [e2e workflow](../.github/workflows/e2e.yml). +The main goal of this workflow is to test breaking/dangerous changes pushed into benchmark-operator, this means that it's not needed to run all tests before merging every pull request. + +## Workflow + +### Bats + +[Bats](https://github.com/bats-core/bats-core) is a TAP-compliant testing framework for Bash. It provides a simple way to verify that the UNIX programs you write behave as expected. + +A Bats test file is a Bash script with special syntax for defining test cases. Under the hood, each test case is just a function with a description. + +```bash +#!/usr/bin/env bats + +@test "addition using bc" { + result="$(echo 2+2 | bc)" + [ "$result" -eq 4 ] +} + +@test "addition using dc" { + result="$(echo 2 2+p | dc)" + [ "$result" -eq 4 ] +} +``` + +If every command in the test case exits with a 0 status code (success), the test passes. This workflow uses bats framework to execute end to end tests, these tests are defined in files with the termination *.bats under the [e2e directory](./e2e) + +### Events + +This workflow is at the moment of writing this document triggered by the following events: + +- A commit is pushed to the master branch: This allows to detect issues when we want a commit is merged to this branch + +- Workflow dispatch: This event allows a repository administrator to **manually trigger the worklflow**. The interesting part of this event is that allows to run specific tests using an input text dialog, this input dialog expectes a regular expression which is passed to the `-f` flag of the bats command. (`-f, --filter Only run tests that match the regular expression`). For example if you fill out this this dialog with uperf|fio, the workflow will trigger only the tests for FIO and smallfile, test names are defined with the special @test suffix in each bats file. `grep -h @test e2e/*.bats` + +- pull_request_target: This event is generated by pull requests, **the PRs must be labeled with "ok to test" to trigger this workflow.** + +## Running in local + +It's possible to run e2e tests in local, you can trigger them with executing the command: `make e2e-tests` +> Benchmark-operator pod must be running before triggering the tests. + +In case you want to test your own code you can use the following command to build, push and deploy your benchmark-operator bits, and eventually trigger e2e test suite. + +```shell +$ IMG= make image-build image-push deploy e2e-tests +``` + +It's also possible to trigger specific tests by setting the BATS_TESTS env var with a regex that will be passed to the `-f` flag of bats. + +```shell +$ IMG=container-registry.io/org/benchmark-operator:mybranch BATS_TESTS="uperf|fio" make image-build image-push deploy e2e-tests +``` + +A preconfigured Elasticsearch endpoint is set by default in the file [helpers.bash](../e2e/helpers.bash), however it's possible to set a custom one by exporting the variable `ES_SERVER`. + +### Software requirements + +Running tests in local has several requirements: + +- podman +- parallel +- bats +- make +- kubectl +- oc diff --git a/e2e/001-scale_openshift.bats b/e2e/001-scale_openshift.bats new file mode 100755 index 000000000..bc3eaf1b8 --- /dev/null +++ b/e2e/001-scale_openshift.bats @@ -0,0 +1,47 @@ +#!/usr/bin/env bats + +# vi: ft=bash + + +load helpers.bash + +ES_INDEX=openshift-cluster-timings + + +@test "scale-up" { + CR=scale-openshift/scale_up.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + # Make node no schedulable as soon as benchmark finishes + kubectl cordon $(kubectl get node --sort-by='{.metadata.creationTimestamp}' -o name | tail -1) + # Annotate machine to ensure it's the one deleted in the scale-down test + # Reference: https://github.com/openshift/machine-api-operator/blob/master/FAQ.md#what-decides-which-machines-to-destroy-when-a-machineset-is-scaled-down + kubectl -n openshift-machine-api annotate $(kubectl get machine -n openshift-machine-api --sort-by='{.metadata.creationTimestamp}' -o name | tail -1) machine.openshift.io/cluster-api-delete-machine=true --overwrite + check_es +} + +@test "scale-down" { + CR=scale-openshift/scale_down.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 || die "Timeout waiting for benchmark/${CR_NAME} to complete" + check_es +} + +setup_file() { + # Prevent running scale down/up simultaneously + export BATS_NO_PARALLELIZE_WITHIN_FILE=true + kubectl_exec apply -f ../resources/scale_role.yaml + basic_setup +} + +teardown_file() { + kubectl_exec delete -f ../resources/scale_role.yaml +} + +teardown() { + basic_teardown +} diff --git a/e2e/002-sysbench.bats b/e2e/002-sysbench.bats new file mode 100755 index 000000000..d8688f655 --- /dev/null +++ b/e2e/002-sysbench.bats @@ -0,0 +1,24 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-sysbench-results + + +@test "sysbench-standard" { + CR=sysbench/sysbench.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/003-byowl.bats b/e2e/003-byowl.bats new file mode 100755 index 000000000..b92b0b9b2 --- /dev/null +++ b/e2e/003-byowl.bats @@ -0,0 +1,32 @@ +#!/usr/bin/env bats + +# vi: ft=bash + + +load helpers.bash + + + +@test "byowl-targeted" { + CR=byowl/byowl-targeted.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 +} + +@test "byowl-not-targeted" { + CR=byowl/byowl-not-targeted.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/004-backpack.bats b/e2e/004-backpack.bats new file mode 100755 index 000000000..68c519d6b --- /dev/null +++ b/e2e/004-backpack.bats @@ -0,0 +1,40 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(cpu_vulnerabilities-metadata cpuinfo-metadata dmidecode-metadata k8s_nodes-metadata lspci-metadata meminfo-metadata sysctl-metadata ocp_network_operator-metadata ocp_install_config-metadata ocp_kube_apiserver-metadata ocp_dns-metadata ocp_kube_controllermanager-metadata) + + +@test "backpack-init" { + CR=backpack/backpack-init.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "backpack-daemonset" { + CR=backpack/backpack-daemonset.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup_file() { + basic_setup + kubectl apply -f ../resources/backpack_role.yaml --overwrite +} + + +teardown_file() { + kubectl delete -f ../resources/backpack_role.yaml +} + +teardown() { + basic_teardown +} diff --git a/e2e/005-stressng.bats b/e2e/005-stressng.bats new file mode 100755 index 000000000..cc1a85dd9 --- /dev/null +++ b/e2e/005-stressng.bats @@ -0,0 +1,25 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-stressng-results + + +@test "stressng-standard" { + CR=stressng/stressng.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/006-vegeta.bats b/e2e/006-vegeta.bats new file mode 100755 index 000000000..85cd1c560 --- /dev/null +++ b/e2e/006-vegeta.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-vegeta-results + + +@test "vegeta-standard" { + CR=vegeta/vegeta.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "vegeta-hostpath" { + CR=vegeta/vegeta_hostnetwork.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/007-smallfile.bats b/e2e/007-smallfile.bats new file mode 100755 index 000000000..00fc84e83 --- /dev/null +++ b/e2e/007-smallfile.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(ripsaw-smallfile-results ripsaw-smallfile-rsptimes) + + +@test "smallfile-standard" { + CR=smallfile/smallfile.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "smallfile-hostpath" { + CR=smallfile/smallfile_hostpath.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/008-kube-burner.bats b/e2e/008-kube-burner.bats new file mode 100755 index 000000000..c13f587b3 --- /dev/null +++ b/e2e/008-kube-burner.bats @@ -0,0 +1,76 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-kube-burner + + +@test "kube-burner-cluster-density" { + CR=kube-burner/cluster-density.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "kube-burner-node-density" { + CR=kube-burner/node-density.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "kube-burner-node-density-heavy" { + CR=kube-burner/node-density-heavy.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "kube-burner-max-services" { + CR=kube-burner/max-services.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "kube-burner-max-namespaces" { + CR=kube-burner/max-namespaces.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "kube-burner-concurrent-builds" { + CR=kube-burner/concurrent-builds.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup_file() { + basic_setup + kubectl apply -f ../resources/kube-burner-role.yml +} + +teardown_file() { + kubectl delete -f ../resources/kube-burner-role.yml +} + +teardown() { + basic_teardown + kubectl delete ns -l kube-burner-uuid=${uuid} --ignore-not-found +} diff --git a/e2e/009-iperf3.bats b/e2e/009-iperf3.bats new file mode 100755 index 000000000..23efd9a4e --- /dev/null +++ b/e2e/009-iperf3.bats @@ -0,0 +1,23 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + + + +@test "iperf3-standard" { + CR=iperf3/iperf3.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/010-fio.bats b/e2e/010-fio.bats new file mode 100755 index 000000000..cae59ab0d --- /dev/null +++ b/e2e/010-fio.bats @@ -0,0 +1,64 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(ripsaw-fio-results ripsaw-fio-log ripsaw-fio-analyzed-result) + + +@test "fio-standard" { + CR=fio/fio.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "fio-bsrange" { + CR=fio/fio_bsrange.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "fio-hostpath" { + CR=fio/fio_hostpath.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "fio-ocs-cachedrop" { + CR=fio/fio_ocs_cache_drop.yaml + CR_NAME=$(get_benchmark_name ${CR}) + openshift_storage_present=$(kubectl get ns | awk '/openshift-storage/' | wc -l) + if [ $openshift_storage_present -gt 0 ]; then + kubectl label nodes -l node-role.kubernetes.io/worker= kernel-cache-dropper=yes --overwrite + kubectl patch OCSInitialization ocsinit -n openshift-storage --type json --patch \ + '[{ "op": "replace", "path": "/spec/enableCephTools", "value": true }]' + drop_cache_pods=$(kubectl -n openshift-storage get pod | awk '/drop/' | awk '/unning/' | wc -l) + if [ $drop_cache_pods -eq 0 ] ; then + kubectl create -f ../resources/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml + kubectl wait --for=condition=Initialized pods/rook-ceph-osd-cache-drop -n openshift-storage --timeout=100s + fi + sleep 5 + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es + fi +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/011-ycsb.bats b/e2e/011-ycsb.bats new file mode 100755 index 000000000..d720de652 --- /dev/null +++ b/e2e/011-ycsb.bats @@ -0,0 +1,33 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(ripsaw-ycsb-summary ripsaw-ycsb-results) + + +@test "ycsb-mongo" { + CR=ycsb/ycsb-mongo.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup() { + kubectl_exec apply -f ycsb/mongo.yaml +} + +setup_file() { + basic_setup +} + +teardown_file() { + kubectl_exec delete -f ycsb/mongo.yaml --ignore-not-found +} + +teardown() { + basic_teardown +} diff --git a/e2e/012-flent.bats b/e2e/012-flent.bats new file mode 100755 index 000000000..8f6920dff --- /dev/null +++ b/e2e/012-flent.bats @@ -0,0 +1,35 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-flent-results + + +@test "flent-standard" { + CR=flent/flent.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + + +@test "flent-resources" { + CR=flent/flent_resources.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/013-uperf.bats b/e2e/013-uperf.bats new file mode 100755 index 000000000..b219ad6f1 --- /dev/null +++ b/e2e/013-uperf.bats @@ -0,0 +1,61 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-uperf-results + + +@test "uperf-standard" { + CR=uperf/uperf.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "uperf-resources" { + CR=uperf/uperf_resources.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "uperf-network policy" { + CR=uperf/uperf_networkpolicy.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "uperf-serviceip" { + CR=uperf/uperf_serviceip.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "uperf-hostnetwork" { + CR=uperf/uperf_hostnetwork.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/014-pgbench.bats b/e2e/014-pgbench.bats new file mode 100755 index 000000000..a78260184 --- /dev/null +++ b/e2e/014-pgbench.bats @@ -0,0 +1,35 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(ripsaw-pgbench-summary ripsaw-pgbench-raw) + + +@test "pgbench-standard" { + CR=pgbench/pgbench.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup() { + kubectl_exec apply -f pgbench/postgres.yaml + kubectl_exec rollout status deploy/postgres --timeout=60s + export POSTGRES_IP=$(kubectl_exec get pod -l app=postgres -o jsonpath="{.items[*].status.podIP}") +} + +setup_file() { + basic_setup +} + +teardown_file() { + kubectl_exec delete -f pgbench/postgres.yaml --ignore-not-found +} + +teardown() { + basic_teardown +} diff --git a/e2e/015-image_pull.bats b/e2e/015-image_pull.bats new file mode 100755 index 000000000..696229e25 --- /dev/null +++ b/e2e/015-image_pull.bats @@ -0,0 +1,25 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=image-pull-results + + +@test "image_pull-standard" { + CR=image_pull/image_pull.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/016-hammerdb.bats b/e2e/016-hammerdb.bats new file mode 100755 index 000000000..7b46b0e91 --- /dev/null +++ b/e2e/016-hammerdb.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-hammerdb-results + + +@test "hammerdb-standard" { + CR=hammerdb/hammerdb.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup() { + kubectl apply -f hammerdb/sql-server.yaml + kubectl rollout status -n sql-server deploy/mssql-deployment --timeout=60s +} + +setup_file() { + basic_setup +} + +teardown_file() { + kubectl delete ns sql-server --wait=false --ignore-not-found +} + +teardown() { + basic_teardown +} diff --git a/e2e/017-fs_drift.bats b/e2e/017-fs_drift.bats new file mode 100755 index 000000000..443047629 --- /dev/null +++ b/e2e/017-fs_drift.bats @@ -0,0 +1,35 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(ripsaw-fs-drift-results ripsaw-fs-drift-rsptimes ripsaw-fs-drift-rates-over-time) + + +@test "fs_drift-standard" { + CR=fs_drift/fs_drift.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + + +@test "fs_drift-hostpath" { + CR=fs_drift/fs_drift_hostpath.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/018-log-generator.bats b/e2e/018-log-generator.bats new file mode 100755 index 000000000..1a4059407 --- /dev/null +++ b/e2e/018-log-generator.bats @@ -0,0 +1,25 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=log-generator-results + + +@test "log-generator-standard" { + CR=log-generator/log_generator.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/019-api_load.bats b/e2e/019-api_load.bats new file mode 100755 index 000000000..0c9790a57 --- /dev/null +++ b/e2e/019-api_load.bats @@ -0,0 +1,22 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + + +@test "api-load-standard" { + CR=api_load/api_load.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/tests/test_crs/valid_api_load.yaml b/e2e/api_load/api_load.yaml similarity index 80% rename from tests/test_crs/valid_api_load.yaml rename to e2e/api_load/api_load.yaml index d49a497a1..626508f59 100644 --- a/tests/test_crs/valid_api_load.yaml +++ b/e2e/api_load/api_load.yaml @@ -1,3 +1,4 @@ +--- apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: @@ -5,8 +6,10 @@ metadata: namespace: benchmark-operator spec: elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: "api-load" + metadata: + collection: true workload: name: api_load args: diff --git a/tests/test_crs/valid_backpack_daemonset.yaml b/e2e/backpack/backpack-daemonset.yaml similarity index 84% rename from tests/test_crs/valid_backpack_daemonset.yaml rename to e2e/backpack/backpack-daemonset.yaml index 97fb74314..57fe3d3df 100644 --- a/tests/test_crs/valid_backpack_daemonset.yaml +++ b/e2e/backpack/backpack-daemonset.yaml @@ -1,11 +1,11 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: backpack + name: backpack-daemonset namespace: benchmark-operator spec: elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} metadata: collection: true targeted: false diff --git a/tests/test_crs/valid_backpack_init.yaml b/e2e/backpack/backpack-init.yaml similarity index 88% rename from tests/test_crs/valid_backpack_init.yaml rename to e2e/backpack/backpack-init.yaml index 6886d3faa..3b8cfbcd2 100644 --- a/tests/test_crs/valid_backpack_init.yaml +++ b/e2e/backpack/backpack-init.yaml @@ -1,11 +1,11 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: backpack + name: backpack-init namespace: benchmark-operator spec: elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} metadata: collection: true privileged: true diff --git a/tests/test_crs/valid_byowl.yaml b/e2e/byowl/byowl-not-targeted.yaml similarity index 69% rename from tests/test_crs/valid_byowl.yaml rename to e2e/byowl/byowl-not-targeted.yaml index 621d08dc8..f2f32fea9 100644 --- a/tests/test_crs/valid_byowl.yaml +++ b/e2e/byowl/byowl-not-targeted.yaml @@ -1,22 +1,23 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: byowl-benchmark + name: byowl-standard namespace: benchmark-operator spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} metadata: collection: true + targeted: false workload: name: byowl args: - image: "quay.io/jtaleric/uperf:testing" + image: "quay.io/cloud-bulldozer/uperf:latest" clients: 1 commands: "echo Test" diff --git a/e2e/byowl/byowl-targeted.yaml b/e2e/byowl/byowl-targeted.yaml new file mode 100644 index 000000000..f17654a93 --- /dev/null +++ b/e2e/byowl/byowl-targeted.yaml @@ -0,0 +1,23 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: byowl-targeted-metadta + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + targeted: true + workload: + name: byowl + args: + image: "quay.io/cloud-bulldozer/uperf:latest" + clients: 1 + commands: "echo Test" diff --git a/tests/test_crs/valid_fiod.yaml b/e2e/fio/fio.yaml similarity index 93% rename from tests/test_crs/valid_fiod.yaml rename to e2e/fio/fio.yaml index 9df7dfe1c..6c9321ae8 100644 --- a/tests/test_crs/valid_fiod.yaml +++ b/e2e/fio/fio.yaml @@ -1,17 +1,17 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: fiod-benchmark + name: fio-benchmark namespace: benchmark-operator spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-fio metadata: collection: true @@ -32,8 +32,8 @@ spec: numjobs: - 2 iodepth: 1 - read_runtime: 10 - write_runtime: 10 + read_runtime: 5 + write_runtime: 5 read_ramp_time: 1 write_ramp_time: 1 filesize: 10MiB diff --git a/tests/test_crs/valid_fiod_bsrange.yaml b/e2e/fio/fio_bsrange.yaml similarity index 92% rename from tests/test_crs/valid_fiod_bsrange.yaml rename to e2e/fio/fio_bsrange.yaml index 2c5cdbb26..e77fd32d4 100644 --- a/tests/test_crs/valid_fiod_bsrange.yaml +++ b/e2e/fio/fio_bsrange.yaml @@ -1,17 +1,17 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: fiod-bsrange-benchmark + name: fio-bsrange namespace: benchmark-operator spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-fio metadata: collection: true @@ -19,7 +19,7 @@ spec: workload: name: "fio_distributed" args: - samples: 2 + samples: 1 servers: 2 jobs: - write @@ -32,8 +32,8 @@ spec: numjobs: - 2 iodepth: 1 - read_runtime: 10 - write_runtime: 10 + read_runtime: 5 + write_runtime: 5 read_ramp_time: 1 write_ramp_time: 1 filesize: 10MiB diff --git a/tests/test_crs/valid_fiod_hostpath.yaml b/e2e/fio/fio_hostpath.yaml similarity index 92% rename from tests/test_crs/valid_fiod_hostpath.yaml rename to e2e/fio/fio_hostpath.yaml index cc56c4cbd..9fb5a2885 100644 --- a/tests/test_crs/valid_fiod_hostpath.yaml +++ b/e2e/fio/fio_hostpath.yaml @@ -1,17 +1,17 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: fiod-hostpath-benchmark + name: fio-hostpath namespace: benchmark-operator spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-fio metadata: collection: true @@ -20,7 +20,6 @@ spec: workload: name: "fio_distributed" args: - drop_cache_kernel: true samples: 1 servers: 2 prefill: true @@ -33,8 +32,8 @@ spec: numjobs: - 2 iodepth: 2 - read_runtime: 6 - write_runtime: 6 + read_runtime: 5 + write_runtime: 5 read_ramp_time: 1 write_ramp_time: 1 filesize: 10MiB diff --git a/tests/test_crs/valid_fiod_ocs_cache_drop.yaml b/e2e/fio/fio_ocs_cache_drop.yaml similarity index 92% rename from tests/test_crs/valid_fiod_ocs_cache_drop.yaml rename to e2e/fio/fio_ocs_cache_drop.yaml index 2ebba6a0e..7e3746a7a 100644 --- a/tests/test_crs/valid_fiod_ocs_cache_drop.yaml +++ b/e2e/fio/fio_ocs_cache_drop.yaml @@ -1,17 +1,17 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: fiod-ocs-benchmark + name: fio-ocs-cachedrop namespace: benchmark-operator spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-fio metadata: collection: true @@ -21,7 +21,7 @@ spec: args: drop_cache_rook_ceph: true drop_cache_kernel: true - samples: 2 + samples: 1 servers: 2 jobs: - randwrite @@ -32,8 +32,8 @@ spec: numjobs: - 2 iodepth: 1 - read_runtime: 10 - write_runtime: 10 + read_runtime: 5 + write_runtime: 5 read_ramp_time: 1 write_ramp_time: 1 filesize: 10MiB diff --git a/tests/test_crs/valid_flent.yaml b/e2e/flent/flent.yaml similarity index 83% rename from tests/test_crs/valid_flent.yaml rename to e2e/flent/flent.yaml index 4d17d4ce4..a733ab7f1 100644 --- a/tests/test_crs/valid_flent.yaml +++ b/e2e/flent/flent.yaml @@ -1,17 +1,17 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: flent-benchmark + name: flent namespace: benchmark-operator spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-flent metadata: collection: true @@ -23,7 +23,6 @@ spec: hostnetwork: false multus: enabled: false - pin: false pair: 1 test_types: - tcp_download diff --git a/tests/test_crs/valid_flent_resources.yaml b/e2e/flent/flent_resources.yaml similarity index 78% rename from tests/test_crs/valid_flent_resources.yaml rename to e2e/flent/flent_resources.yaml index bef6ba7ab..8fc7253a5 100644 --- a/tests/test_crs/valid_flent_resources.yaml +++ b/e2e/flent/flent_resources.yaml @@ -1,18 +1,17 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: flent-resources-benchmark + name: flent-resources namespace: benchmark-operator spec: - clustername: myk8scluster system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-flent workload: # cleanup: true @@ -27,8 +26,6 @@ spec: cpu: 100m memory: 100Mi pin: false - pin_server: "node-0" - pin_client: "node-1" hostnetwork: false multus: enabled: false diff --git a/tests/test_crs/valid_fs_drift.yaml b/e2e/fs_drift/fs_drift.yaml similarity index 86% rename from tests/test_crs/valid_fs_drift.yaml rename to e2e/fs_drift/fs_drift.yaml index 9ccd180ce..42e750698 100644 --- a/tests/test_crs/valid_fs_drift.yaml +++ b/e2e/fs_drift/fs_drift.yaml @@ -10,12 +10,12 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml # where elastic search is running elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-fs-drift metadata: collection: true @@ -26,5 +26,5 @@ spec: threads: 5 max_file_size_kb: 4 max_files: 1000 - duration: 15 + duration: 10 debug: true diff --git a/tests/test_crs/valid_fs_drift_hostpath.yaml b/e2e/fs_drift/fs_drift_hostpath.yaml similarity index 87% rename from tests/test_crs/valid_fs_drift_hostpath.yaml rename to e2e/fs_drift/fs_drift_hostpath.yaml index 91c8f65bb..77596f9b9 100644 --- a/tests/test_crs/valid_fs_drift_hostpath.yaml +++ b/e2e/fs_drift/fs_drift_hostpath.yaml @@ -10,12 +10,12 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml # where elastic search is running elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-fs-drift metadata: collection: true @@ -27,5 +27,5 @@ spec: threads: 5 max_file_size_kb: 4 max_files: 1000 - duration: 15 + duration: 10 debug: true diff --git a/tests/test_crs/valid_hammerdb.yaml b/e2e/hammerdb/hammerdb.yaml similarity index 94% rename from tests/test_crs/valid_hammerdb.yaml rename to e2e/hammerdb/hammerdb.yaml index 6c3d1b7f3..f72f833db 100644 --- a/tests/test_crs/valid_hammerdb.yaml +++ b/e2e/hammerdb/hammerdb.yaml @@ -7,11 +7,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-hammerdb metadata: collection: true @@ -31,7 +31,7 @@ spec: db_user: "SA" db_pass: "s3curePasswordString" db_name: "tpcc" - transactions: 10000 + transactions: 1000 raiseerror: "false" keyandthink: "false" driver: "timed" diff --git a/tests/mssql.yaml b/e2e/hammerdb/sql-server.yaml old mode 100755 new mode 100644 similarity index 100% rename from tests/mssql.yaml rename to e2e/hammerdb/sql-server.yaml diff --git a/e2e/helpers.bash b/e2e/helpers.bash new file mode 100644 index 000000000..3c2e4b2f8 --- /dev/null +++ b/e2e/helpers.bash @@ -0,0 +1,91 @@ +# vi: ft=bash + +uuid="" +suuid="" +ARTIFACTS_DIR=artifacts +NAMESPACE=benchmark-operator + + +basic_setup() { + export PROMETHEUS_TOKEN=$(oc sa get-token -n openshift-monitoring prometheus-k8s) + export ES_SERVER=${ES_SERVER:-https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com} +} + +basic_teardown() { + kubectl_exec delete benchmark ${CR_NAME} --ignore-not-found +} + +check_es() { + if [[ -n ${indexes} ]]; then + for index in "${indexes[@]}"; do + echo "Looking for documents with uuid: ${uuid} in index ${index}" + documents=$(curl -sS ${ES_SERVER}/${index}/_search?q=uuid.keyword:${uuid} | jq .hits.total.value) + if [[ ${documents} -le 0 ]]; then + die "${documents} documents found in index ${index}" + fi + done + else + echo "Looking for documents with uuid: ${uuid} in index ${ES_INDEX}" + documents=$(curl -sS ${ES_SERVER}/${ES_INDEX}/_search?q=uuid.keyword:${uuid} | jq .hits.total.value) + if [[ ${documents} -le 0 ]]; then + die "${documents} documents found in index ${ES_INDEX}" + fi + fi +} + +get_uuid() { + echo "Waiting for UUID from ${1}" + local timeout=300 + while [[ ${timeout} -gt 0 ]]; do + uuid=$(kubectl_exec get benchmarks ${1} -o jsonpath="{.status.uuid}") + if [[ -n ${uuid} ]]; then + suuid=$(kubectl_exec get benchmarks ${1} -o jsonpath="{.status.suuid}") + return + fi + sleep 1 + timeout=$((timeout - 1)) + done + die "Timeout waiting for uuid from benchmark ${1}" +} + +check_benchmark() { + local timeout=${1} + while [[ ${timeout} -gt 0 ]]; do + if [[ $(kubectl_exec get benchmark/${CR_NAME} -o jsonpath={.status.complete}) == 'true' ]]; then + break + fi + if [[ ${timeout} -lt 0 ]]; then + die "Timeout waiting for benchmark/${CR_NAME} to complete" + fi + sleep 10 + timeout=$((timeout - 10)) + done + local state=$(kubectl_exec get benchmark/${CR_NAME} -o jsonpath={.status.state}) + if [[ ${state} != "Complete" ]]; then + die "Benchmark state: ${state}" + fi +} + +die() { + printf "\nError message: ${1}\n" + local TEST_ARTIFACTS=${ARTIFACTS_DIR}/${CR_NAME} + mkdir -p ${TEST_ARTIFACTS} + echo "Dumping logs at ${TEST_ARTIFACTS}" + kubectl_exec get benchmark ${CR_NAME} -o yaml --ignore-not-found > ${TEST_ARTIFACTS}/${CR_NAME}.yaml + kubectl_exec logs deployment/benchmark-controller-manager --tail=-1 -c manager > ${ARTIFACTS_DIR}/benchmark-controller-manager.log + for pod in $(kubectl_exec get pod -l benchmark-uuid=${uuid} -o custom-columns="name:.metadata.name" --no-headers); do + log_file=${TEST_ARTIFACTS}/${pod}.log + echo "Saving log from pod ${pod} in ${log_file}" + kubectl_exec logs --tail=-1 ${pod} > ${log_file} + done + false +} + +get_benchmark_name() { + benchmark_file=$1 + yq e '.metadata.name' $benchmark_file +} + +kubectl_exec() { + kubectl -n ${NAMESPACE} $@ +} diff --git a/tests/test_crs/valid_image_pull.yaml b/e2e/image_pull/image_pull.yaml similarity index 70% rename from tests/test_crs/valid_image_pull.yaml rename to e2e/image_pull/image_pull.yaml index c39cf5e63..add809f5d 100644 --- a/tests/test_crs/valid_image_pull.yaml +++ b/e2e/image_pull/image_pull.yaml @@ -5,14 +5,14 @@ metadata: namespace: benchmark-operator spec: elasticsearch: - url: ES_SERVER - index_name: "image-pull" + url: ${ES_SERVER} + index_name: image-pull workload: name: image_pull args: pod_count: 2 - timeout: 100 + timeout: 60 retries: 1 image_list: - docker://quay.io/cloud-bulldozer/backpack - - docker://quay.io/cloud-bulldozer/fio + - docker://quay.io/cloud-bulldozer/hello-openshift diff --git a/tests/test_crs/valid_iperf3.yaml b/e2e/iperf3/iperf3.yaml similarity index 89% rename from tests/test_crs/valid_iperf3.yaml rename to e2e/iperf3/iperf3.yaml index d0ebe5b91..d0bae2507 100644 --- a/tests/test_crs/valid_iperf3.yaml +++ b/e2e/iperf3/iperf3.yaml @@ -7,11 +7,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} metadata: collection: true workload: diff --git a/e2e/kube-burner/cluster-density.yaml b/e2e/kube-burner/cluster-density.yaml new file mode 100644 index 000000000..d0c37f0b0 --- /dev/null +++ b/e2e/kube-burner/cluster-density.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-cluster-density + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + prometheus: + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + workload: cluster-density + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 1 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics-aggregated.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/tests/test_crs/valid_kube-burner.yaml b/e2e/kube-burner/concurrent-builds.yaml similarity index 85% rename from tests/test_crs/valid_kube-burner.yaml rename to e2e/kube-burner/concurrent-builds.yaml index 7efa673d8..76e1a9813 100644 --- a/tests/test_crs/valid_kube-burner.yaml +++ b/e2e/kube-burner/concurrent-builds.yaml @@ -2,25 +2,24 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: kube-burner-benchmark + name: kube-burner-concurrent-builds namespace: benchmark-operator spec: elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} metadata: collection: true prometheus: - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 workload: name: kube-burner args: - workload: WORKLOAD + workload: concurrent-builds cleanup: true default_index: ripsaw-kube-burner job_iterations: 1 - job_timeout: 300 pin_server: {"node-role.kubernetes.io/worker": ""} wait_when_finished: true pod_wait: false @@ -31,7 +30,7 @@ spec: verify_objects: true error_on_verify: true step: 30s - metrics_profile: METRICS_PROFILE + metrics_profile: metrics-aggregated.yaml app: django source_strat_env: PIP_INDEX_URL source_strat_from: python diff --git a/e2e/kube-burner/max-namespaces.yaml b/e2e/kube-burner/max-namespaces.yaml new file mode 100644 index 000000000..daaf3fe6e --- /dev/null +++ b/e2e/kube-burner/max-namespaces.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-max-namespaces + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + prometheus: + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + workload: max-namespaces + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 1 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics-aggregated.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/max-services.yaml b/e2e/kube-burner/max-services.yaml new file mode 100644 index 000000000..dfdc0e6d2 --- /dev/null +++ b/e2e/kube-burner/max-services.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-max-services + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + prometheus: + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + workload: max-services + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 1 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics-aggregated.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/node-density-heavy.yaml b/e2e/kube-burner/node-density-heavy.yaml new file mode 100644 index 000000000..543b2fd0d --- /dev/null +++ b/e2e/kube-burner/node-density-heavy.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-node-density-heavy + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + prometheus: + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + workload: node-density-heavy + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 2 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/node-density.yaml b/e2e/kube-burner/node-density.yaml new file mode 100644 index 000000000..76775d6a4 --- /dev/null +++ b/e2e/kube-burner/node-density.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-node-density + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + prometheus: + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + workload: node-density + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 10 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/tests/test_crs/valid_log_generator.yaml b/e2e/log-generator/log_generator.yaml similarity index 86% rename from tests/test_crs/valid_log_generator.yaml rename to e2e/log-generator/log_generator.yaml index 72b926db2..146496d78 100644 --- a/tests/test_crs/valid_log_generator.yaml +++ b/e2e/log-generator/log_generator.yaml @@ -5,7 +5,7 @@ metadata: namespace: benchmark-operator spec: elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: log-generator workload: name: log_generator @@ -14,5 +14,5 @@ spec: size: 512 messages_per_second: 1 duration: 1 - es_url: ES_SERVER + es_url: ${ES_SERVER} timeout: 600 diff --git a/tests/test_crs/valid_pgbench.yaml b/e2e/pgbench/pgbench.yaml similarity index 86% rename from tests/test_crs/valid_pgbench.yaml rename to e2e/pgbench/pgbench.yaml index 908a4e58b..63be13ad3 100644 --- a/tests/test_crs/valid_pgbench.yaml +++ b/e2e/pgbench/pgbench.yaml @@ -7,11 +7,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-pgbench metadata: collection: true @@ -30,7 +30,7 @@ spec: samples: 2 num_databases_pattern: 'all' databases: - - host: + - host: ${POSTGRES_IP} user: ci password: ci db_name: cidb diff --git a/e2e/pgbench/postgres.yaml b/e2e/pgbench/postgres.yaml new file mode 100644 index 000000000..6c739e8c0 --- /dev/null +++ b/e2e/pgbench/postgres.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-config + namespace: benchmark-operator + labels: + app: postgres +data: + POSTGRES_DB: cidb + POSTGRES_USER: ci + POSTGRES_PASSWORD: ci + PGDATA: /var/lib/postgresql/data/pgdata +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: benchmark-operator +spec: + selector: + matchLabels: + app: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:10.4 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5432 + envFrom: + - configMapRef: + name: postgres-config diff --git a/tests/test_crs/valid_scale_down.yaml b/e2e/scale-openshift/scale_down.yaml similarity index 86% rename from tests/test_crs/valid_scale_down.yaml rename to e2e/scale-openshift/scale_down.yaml index 4b1d03b04..8dfebbece 100644 --- a/tests/test_crs/valid_scale_down.yaml +++ b/e2e/scale-openshift/scale_down.yaml @@ -9,11 +9,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: openshift-cluster-timings workload: name: scale_openshift diff --git a/tests/test_crs/valid_scale_up.yaml b/e2e/scale-openshift/scale_up.yaml similarity index 82% rename from tests/test_crs/valid_scale_up.yaml rename to e2e/scale-openshift/scale_up.yaml index e1f1aceff..cb45b85fe 100644 --- a/tests/test_crs/valid_scale_up.yaml +++ b/e2e/scale-openshift/scale_up.yaml @@ -9,16 +9,16 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: openshift-cluster-timings workload: name: scale_openshift args: scale: 1 serviceaccount: scaler - poll_interval: 2 + poll_interval: 1 debug: true diff --git a/tests/test_crs/valid_smallfile.yaml b/e2e/smallfile/smallfile.yaml similarity index 90% rename from tests/test_crs/valid_smallfile.yaml rename to e2e/smallfile/smallfile.yaml index f37b85731..9a4752de1 100644 --- a/tests/test_crs/valid_smallfile.yaml +++ b/e2e/smallfile/smallfile.yaml @@ -1,3 +1,4 @@ +--- apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: @@ -10,12 +11,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml - # where elastic search is running elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-smallfile metadata: collection: true diff --git a/tests/test_crs/valid_smallfile_hostpath.yaml b/e2e/smallfile/smallfile_hostpath.yaml similarity index 85% rename from tests/test_crs/valid_smallfile_hostpath.yaml rename to e2e/smallfile/smallfile_hostpath.yaml index daf1c273b..573714502 100644 --- a/tests/test_crs/valid_smallfile_hostpath.yaml +++ b/e2e/smallfile/smallfile_hostpath.yaml @@ -1,8 +1,8 @@ - +--- apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: smallfile-hostpath-benchmark + name: smallfile-hostpath namespace: benchmark-operator spec: test_user: homer_simpson @@ -11,11 +11,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-smallfile metadata: collection: true diff --git a/tests/test_crs/valid_stressng.yaml b/e2e/stressng/stressng.yaml similarity index 87% rename from tests/test_crs/valid_stressng.yaml rename to e2e/stressng/stressng.yaml index 220004b18..608b5c2c7 100644 --- a/tests/test_crs/valid_stressng.yaml +++ b/e2e/stressng/stressng.yaml @@ -7,11 +7,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-stressng metadata: collection: true @@ -20,7 +20,7 @@ spec: args: # general options runtype: "parallel" - timeout: "30" + timeout: "10" instances: 1 # nodeselector: # cpu stressor options diff --git a/tests/test_crs/valid_sysbench.yaml b/e2e/sysbench/sysbench.yaml similarity index 84% rename from tests/test_crs/valid_sysbench.yaml rename to e2e/sysbench/sysbench.yaml index 9bd1aca1d..e05ae5638 100644 --- a/tests/test_crs/valid_sysbench.yaml +++ b/e2e/sysbench/sysbench.yaml @@ -7,11 +7,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} metadata: collection: true workload: diff --git a/tests/test_crs/valid_uperf.yaml b/e2e/uperf/uperf.yaml similarity index 84% rename from tests/test_crs/valid_uperf.yaml rename to e2e/uperf/uperf.yaml index a0ff00c5e..a9d910c7a 100644 --- a/tests/test_crs/valid_uperf.yaml +++ b/e2e/uperf/uperf.yaml @@ -1,17 +1,17 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: uperf + name: uperf-burstable namespace: benchmark-operator spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-uperf metadata: collection: true @@ -22,8 +22,6 @@ spec: hostnetwork: false serviceip: false pin: false - pin_server: "node-0" - pin_client: "node-1" multus: enabled: false samples: 2 diff --git a/e2e/uperf/uperf_hostnetwork.yaml b/e2e/uperf/uperf_hostnetwork.yaml new file mode 100644 index 000000000..2e0c1a35a --- /dev/null +++ b/e2e/uperf/uperf_hostnetwork.yaml @@ -0,0 +1,38 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-hostnetwork + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf + args: + hostnetwork: true + serviceip: false + pin: false + multus: + enabled: false + samples: 1 + pair: 1 + test_types: + - stream + protos: + - tcp + sizes: + - 1024 + nthrs: + - 1 + runtime: 2 + debug: true diff --git a/tests/test_crs/valid_uperf_networkpolicy.yaml b/e2e/uperf/uperf_networkpolicy.yaml similarity index 76% rename from tests/test_crs/valid_uperf_networkpolicy.yaml rename to e2e/uperf/uperf_networkpolicy.yaml index 54cb87e1f..3470bc5c0 100644 --- a/tests/test_crs/valid_uperf_networkpolicy.yaml +++ b/e2e/uperf/uperf_networkpolicy.yaml @@ -7,11 +7,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-uperf metadata: collection: true @@ -23,24 +23,17 @@ spec: serviceip: false networkpolicy: true pin: false - pin_server: "node-0" - pin_client: "node-1" multus: enabled: false - samples: 2 + samples: 1 pair: 1 test_types: - stream - - rr - - bidirec protos: - tcp - - udp sizes: - 1024 - - 512 nthrs: - 1 - - 2 runtime: 2 debug: true diff --git a/tests/test_crs/valid_uperf_resources.yaml b/e2e/uperf/uperf_resources.yaml similarity index 77% rename from tests/test_crs/valid_uperf_resources.yaml rename to e2e/uperf/uperf_resources.yaml index da3f60392..46c7d43eb 100644 --- a/tests/test_crs/valid_uperf_resources.yaml +++ b/e2e/uperf/uperf_resources.yaml @@ -7,11 +7,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-uperf metadata: collection: true @@ -27,26 +27,19 @@ spec: requests: cpu: 100m memory: 100Mi - hostnetwork: true + hostnetwork: false serviceip: false - pin: false - pin_server: "node-0" - pin_client: "node-1" multus: enabled: false - samples: 2 + samples: 1 pair: 1 test_types: - stream - - rr protos: - tcp - - udp sizes: - 1024 - - 512 nthrs: - 1 - - 2 runtime: 2 debug: true diff --git a/tests/test_crs/valid_uperf_serviceip.yaml b/e2e/uperf/uperf_serviceip.yaml similarity index 77% rename from tests/test_crs/valid_uperf_serviceip.yaml rename to e2e/uperf/uperf_serviceip.yaml index 9837bbe7c..5a1fda9af 100644 --- a/tests/test_crs/valid_uperf_serviceip.yaml +++ b/e2e/uperf/uperf_serviceip.yaml @@ -7,11 +7,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-uperf metadata: collection: true @@ -21,24 +21,18 @@ spec: args: hostnetwork: false serviceip: true - pin: false - pin_server: "node-0" - pin_client: "node-1" multus: enabled: false - samples: 2 + samples: 1 pair: 1 test_types: - stream - - rr protos: - tcp - udp sizes: - - 1024 - 512 nthrs: - 1 - - 2 runtime: 2 debug: true diff --git a/tests/test_crs/valid_vegeta.yaml b/e2e/vegeta/vegeta.yaml similarity index 85% rename from tests/test_crs/valid_vegeta.yaml rename to e2e/vegeta/vegeta.yaml index 4d9e79f5e..93cb362c8 100644 --- a/tests/test_crs/valid_vegeta.yaml +++ b/e2e/vegeta/vegeta.yaml @@ -8,24 +8,24 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-vegeta metadata: collection: true workload: name: vegeta args: - clients: 3 + clients: 4 targets: - name: 2w-ka urls: - GET https://1.1.1.1 - GET http://1.1.1.1 - samples: 3 + samples: 1 workers: 2 duration: 5 keepalive: true diff --git a/tests/test_crs/valid_vegeta_hostnetwork.yaml b/e2e/vegeta/vegeta_hostnetwork.yaml similarity index 85% rename from tests/test_crs/valid_vegeta_hostnetwork.yaml rename to e2e/vegeta/vegeta_hostnetwork.yaml index 6d3f1ecb8..3eecec5c5 100644 --- a/tests/test_crs/valid_vegeta_hostnetwork.yaml +++ b/e2e/vegeta/vegeta_hostnetwork.yaml @@ -8,15 +8,14 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-vegeta metadata: collection: true - serviceaccount: benchmark-operator workload: name: vegeta args: diff --git a/e2e/ycsb/mongo.yaml b/e2e/ycsb/mongo.yaml new file mode 100644 index 000000000..622911bef --- /dev/null +++ b/e2e/ycsb/mongo.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: mongo + labels: + name: mongo +spec: + ports: + - port: 27017 + targetPort: 27017 + clusterIP: None + selector: + role: mongo +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mongo +spec: + selector: + matchLabels: + role: mongo + serviceName: "mongo" + replicas: 1 + selector: + matchLabels: + role: mongo + template: + metadata: + labels: + role: mongo + environment: test + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mongo + image: mongo + command: ["/bin/sh"] + args: ["-c", "mkdir -p /tmp/data/db; mongod --bind_ip 0.0.0.0 --dbpath /tmp/data/db"] + ports: + - containerPort: 27017 diff --git a/tests/test_crs/valid_ycsb-mongo.yaml b/e2e/ycsb/ycsb-mongo.yaml similarity index 90% rename from tests/test_crs/valid_ycsb-mongo.yaml rename to e2e/ycsb/ycsb-mongo.yaml index 3e8a56cda..5f03033e6 100644 --- a/tests/test_crs/valid_ycsb-mongo.yaml +++ b/e2e/ycsb/ycsb-mongo.yaml @@ -7,11 +7,11 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER - prom_token: PROMETHEUS_TOKEN + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} metrics_profile: node-metrics.yml elasticsearch: - url: ES_SERVER + url: ${ES_SERVER} index_name: ripsaw-ycsb metadata: collection: true diff --git a/hack/install-bats.sh b/hack/install-bats.sh new file mode 100755 index 000000000..9f9219571 --- /dev/null +++ b/hack/install-bats.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e + +if [[ "$(type -t bats)" != "" ]]; then + # bats is already installed. + exit 0 +fi + +buildDir=$(mktemp -d) +git clone https://github.com/bats-core/bats-core $buildDir + +pushd $buildDir +pwd +git reset --hard ${VERSION} +echo "Installing bats to /usr/local (requires root)" +sudo ./install.sh /usr/local +popd + +rm -rf $buildDir diff --git a/roles/fs-drift/templates/workload_job.yml.j2 b/roles/fs-drift/templates/workload_job.yml.j2 index ba2970a39..699b38af6 100644 --- a/roles/fs-drift/templates/workload_job.yml.j2 +++ b/roles/fs-drift/templates/workload_job.yml.j2 @@ -30,7 +30,7 @@ spec: topologyKey: "kubernetes.io/hostname" restartPolicy: Never containers: - - name: benchmark-server + - name: fs-drift {% if hostpath is defined %} securityContext: privileged: true diff --git a/roles/smallfile/templates/workload_job.yml.j2 b/roles/smallfile/templates/workload_job.yml.j2 index fb773dd35..c8a36c769 100644 --- a/roles/smallfile/templates/workload_job.yml.j2 +++ b/roles/smallfile/templates/workload_job.yml.j2 @@ -30,7 +30,7 @@ spec: topologyKey: "kubernetes.io/hostname" restartPolicy: Never containers: - - name: benchmark-server + - name: smallfile {% if hostpath is defined %} securityContext: privileged: true diff --git a/run_test.sh b/run_test.sh deleted file mode 100755 index 5bc2b7e06..000000000 --- a/run_test.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -set -x - -ci_dir=$1 -ci_test=`echo $1 | sed 's/-/_/g'` -retries=3 - -figlet $ci_test - -# Random short sleep to avoid command collision -sleep $[ ( $RANDOM % 10 )]s - -cd $ci_dir -source tests/common.sh - -count=0 -start_time=`date` - -# Run the test up to a max of $retries -while [ $count -le $retries ] -do - wait_clean - - # Test ci - if /bin/bash tests/$ci_test.sh >> $ci_test.out 2>&1 - then - # if the test passes update the results and complete - end_time=`date` - duration=`date -ud@$(($(date -ud"$end_time" +%s)-$(date -ud"$start_time" +%s))) +%T` - echo "$ci_dir: Successful" - echo "$ci_dir: Successful" > ci_results - echo " " > results.xml - echo "$ci_test | Pass | $count | $duration" > results.markdown - count=$retries - else - # if the test failed check if we have done the max retries - if [ $count -lt $retries ] - then - echo "$ci_dir: Failed. Retrying" - echo "$ci_dir: Failed. Retrying" >> $ci_test.out - else - end_time=`date` - duration=`date -ud@$(($(date -ud"$end_time" +%s)-$(date -ud"$start_time" +%s))) +%T` - echo "$ci_dir: Failed retry" - echo "$ci_dir: Failed" > ci_results - echo " " > results.xml - echo " - " >> results.xml - echo "$ci_test | Fail | $count | $duration" > results.markdown - echo "Logs for "$ci_dir - - # Display the error log since we have failed to pass - cat $ci_test.out - fi - fi - ((count++)) -done -wait_clean diff --git a/test.sh b/test.sh deleted file mode 100755 index d64d95fa3..000000000 --- a/test.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bash -set -x - -# The maximum number of concurrent tests to run at one time (0 for unlimited). This can be overriden with -p -max_concurrent=3 - -# Presetting test_choice to be blank. -test_choice='' - -while getopts p:t:s: flag -do - case "${flag}" in - p) max_concurrent=${OPTARG};; - t) test_choice=${OPTARG};; - s) ES_SERVER=${OPTARG};; - esac -done - -if ! command -v yq &> /dev/null -then - echo "yq not installed, installing" - wget https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_linux_amd64 -O /usr/local/bin/yq && chmod +x /usr/local/bin/yq -fi - - -source tests/common.sh - -eStatus=0 - -git_diff_files="$(git diff remotes/origin/master --name-only)" - -if [[ $test_choice != '' ]]; then - echo "Running for requested tests" - populate_test_list "${test_choice}" -elif [[ `echo "${git_diff_files}" | grep -cv /` -gt 0 || `echo "${git_diff_files}" | grep -E "^(templates|build|deploy|group_vars|resources|tests/common.sh|roles/uuid)"` ]]; then - echo "Running full test" - cp tests/test_list tests/iterate_tests -else - echo "Running specific tests" - populate_test_list "${git_diff_files}" -fi - -test_list="$(cat tests/iterate_tests)" -echo "running test suit consisting of ${test_list}" - -if [[ ${test_list} == "" ]]; then - echo "No tests to run" - echo "Results for "$JOB_NAME > results.markdown - echo "No tests to run" >> results.markdown - exit 0 -fi - -# Massage the names into something that is acceptable for a namespace -sed 's/\.sh//g' tests/iterate_tests | sort | uniq > tests/my_tests -sed -i 's/_/-/g' tests/my_tests - -# Prep the results.xml file -echo ' - - ' > results.xml - -# Prep the results.markdown file -echo "Results for "$JOB_NAME > results.markdown -echo "" >> results.markdown -echo 'Test | Result | Retries| Duration (HH:MM:SS)' >> results.markdown -echo '-----|--------|--------|---------' >> results.markdown - -# Create a "gold" directory based off the current branch -mkdir gold - -# Generate uuid -NEW_UUID=$(uuidgen) -UUID=${NEW_UUID%-*} - -#Tag name -tag_name="${NODE_NAME:-master}" - -sed -i "s#ES_SERVER#$ES_SERVER#g" tests/test_crs/* -cp -pr * gold/ - -# Create individual directories for each test -for ci_dir in `cat tests/my_tests` -do - mkdir $ci_dir - cp -pr gold/* $ci_dir/ -done - -oc delete namespace benchmark-operator --ignore-not-found -kubectl delete benchmarks -n benchmark-operator --ignore-not-found --all -apply_operator - - -# Run scale test first if it is in the test list -scale_test="false" -if [[ `grep test-scale-openshift tests/my_tests` ]] -then - scale_test="true" - sed -i '/test-scale-openshift/d' tests/my_tests - ./run_test.sh test-scale-openshift -fi - -# Run tests in parallel up to $max_concurrent at a time. -parallel -n 1 -a tests/my_tests -P $max_concurrent ./run_test.sh -if [[ $scale_test == "true" ]] -then - echo "test-scale-openshift" >> tests/my_tests -fi - -# Update and close JUnit test results.xml and markdown file -for test_dir in `cat tests/my_tests` -do - cat $test_dir/results.xml >> results.xml - cat $test_dir/results.markdown >> results.markdown - cat $test_dir/ci_results >> ci_results -done - -# Get number of successes/failures -testcount=`wc -l ci_results` -success=`grep Successful ci_results | awk -F ":" '{print $1}'` -failed=`grep Failed ci_results | awk -F ":" '{print $1}'` -failcount=`grep -c Failed ci_results` -echo "CI tests that passed: "$success -echo "CI tests that failed: "$failed -echo "Smoke test: Complete" - -echo " -" >> results.xml - -sed -i "s/NUMTESTS/$testcount/g" results.xml -sed -i "s/NUMFAILURES/$failcount/g" results.xml - -if [ `grep -c Failed ci_results` -gt 0 ] -then - eStatus=1 -fi - -# Clean up our created directories -rm -rf gold test-* ci_results - -exit $eStatus diff --git a/tests/check_es.py b/tests/check_es.py deleted file mode 100755 index efd203629..000000000 --- a/tests/check_es.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import elasticsearch -import sys -import ssl -import urllib3 - - -def _check_index(server, uuid, index, es_ssl): - - if es_ssl == "true": - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - ssl_ctx = ssl.create_default_context() - ssl_ctx.check_hostname = False - ssl_ctx.verify_mode = ssl.CERT_NONE - es = elasticsearch.Elasticsearch([server], send_get_body_as='POST', ssl_context=ssl_ctx, use_ssl=True) - else: - es = elasticsearch.Elasticsearch([server], send_get_body_as='POST') - es.indices.refresh(index=index) - results = es.search(index=index, body={'query': {'term': {'uuid.keyword': uuid}}}, size=1) - if results['hits']['total']['value'] > 0: - return 0 - else: - print("No result found in ES") - return 1 - - -def main(): - parser = argparse.ArgumentParser(description="Script to verify uploads to ES") - parser.add_argument( - '-s', '--server', - help='Provide elastic server information') - parser.add_argument( - '-u', '--uuid', - help='UUID to provide to search') - parser.add_argument( - '-i', '--index', - help='Index to provide to search') - parser.add_argument( - '--sslskipverify', - help='if es is setup with ssl, but can disable tls cert verification', - default=False) - args = parser.parse_args() - - sys.exit(_check_index(args.server, args.uuid, args.index, args.sslskipverify)) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tests/common.sh b/tests/common.sh deleted file mode 100755 index 302e3d144..000000000 --- a/tests/common.sh +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env bash - -ERRORED=false -image_location=${RIPSAW_CI_IMAGE_LOCATION:-quay.io} -image_account=${RIPSAW_CI_IMAGE_ACCOUNT:-rht_perf_ci} -es_url=${ES_SERVER:-http://foo.esserver.com:9200} -echo "using container image location $image_location and account $image_account" - -function populate_test_list { - rm -f tests/iterate_tests - - for item in $@ - do - # Check for changes in roles - if [[ $(echo ${item} | grep 'roles/fs-drift') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/uperf') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/fio_distributed') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/iperf3') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/byowl') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/sysbench') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/pgbench') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/ycsb') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/backpack') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/hammerdb') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/smallfile') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/vegeta') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/stressng') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/scale_openshift') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/kube-burner') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/flent') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/log_generator') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'roles/image_pull') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi - - # Check for changes in cr files - if [[ $(echo ${item} | grep 'valid_backpack*') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_byowl*') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_fiod*') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_fs_drift*') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_hammerdb*') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_iperf3*') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_pgbench*') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_smallfile*') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_sysbench*') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_uperf*') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_ycsb*') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_vegeta*') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_stressng*') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_scale*') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_kube-burner*') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_flent*') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_log_generator*') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi - if [[ $(echo ${item} | grep 'valid_image_pull*') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi - - # Check for changes in test scripts - test_check=`echo $item | awk -F / '{print $2}'` - - if [[ $(echo ${test_check} | grep 'test_.*.sh') ]]; then echo ${test_check} >> tests/iterate_tests; fi - done -} - -function delete_benchmark { - kubectl delete -f $1 --ignore-not-found=true -} - -function wait_clean { - if [[ `kubectl get benchmarks.ripsaw.cloudbulldozer.io --all-namespaces` ]] - then - echo "skipping cleanup" - fi -} - -# The argument is 'timeout in seconds' -function get_uuid () { - sleep_time=20 - benchmark_name=$1 - sleep $sleep_time - counter=0 - counter_max=6 - uuid="False" - until [ $uuid != "False" ] ; do - uuid=$(kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath='{.status.uuid}') - if [ -z $uuid ]; then - sleep $sleep_time - uuid="False" - fi - counter=$(( counter+1 )) - if [ $counter -eq $counter_max ]; then - return 1 - fi - done - echo $uuid - return 0 -} - - -# Two arguments are 'pod label' and 'timeout in seconds' -function get_pod () { - counter=0 - sleep_time=5 - counter_max=$(( $2 / sleep_time )) - pod_name="False" - until [ $pod_name != "False" ] ; do - sleep $sleep_time - pod_name=$(kubectl get pods -l $1 --namespace ${3:-benchmark-operator} -o name | cut -d/ -f2) - if [ -z $pod_name ]; then - pod_name="False" - fi - counter=$(( counter+1 )) - if [ $counter -eq $counter_max ]; then - return 1 - fi - done - echo $pod_name - return 0 -} - -# Three arguments are 'pod label', 'expected count', and 'timeout in seconds' -function pod_count () { - counter=0 - sleep_time=5 - counter_max=$(( $3 / sleep_time )) - pod_count=0 - export $1 - until [ $pod_count == $2 ] ; do - sleep $sleep_time - pod_count=$(kubectl get pods -n benchmark-operator -l $1 -o name | wc -l) - if [ -z $pod_count ]; then - pod_count=0 - fi - counter=$(( counter+1 )) - if [ $counter -eq $counter_max ]; then - return 1 - fi - done - echo $pod_count - return 0 -} - -function apply_operator { - tag_name="${NODE_NAME:-master}" - make image-build image-push deploy IMG=$image_location/$image_account/benchmark-operator:$tag_name - kubectl wait --for=condition=available "deployment/benchmark-controller-manager" -n benchmark-operator --timeout=300s -} - -function delete_operator { - make undeploy -} - - -function backpack_requirements { - kubectl apply -f resources/backpack_role.yaml - if [[ `command -v oc` ]] - then - if [[ `oc get securitycontextconstraints.security.openshift.io` ]] - then - oc adm policy -n benchmark-operator add-scc-to-user privileged -z benchmark-operator - oc adm policy -n benchmark-operator add-scc-to-user privileged -z backpack-view - fi - fi -} - - -function cleanup_operator_resources { - delete_operator - wait_clean -} - -function check_log(){ - for i in {1..10}; do - if kubectl logs -f $1 --namespace benchmark-operator | grep -q $2 ; then - break; - else - sleep 10 - fi - done -} - -function get_benchmark_name() { - benchmark_file=$1 - echo $(yq e '.metadata.name' $benchmark_file) -} - -function get_benchmark_state() { - benchmark_name=$1 - kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath={.status.state} -} - -function wait_for_benchmark() { - benchmark_name=$1 - desired_state=$2 - until [[ $(get_benchmark_state $benchmark_name) == "$desired_state" ]]; do - if [[ $(get_benchmark_state $benchmark_name) == "Failed" ]]; then - echo "Benchmark $benchmark_name failed" - return 1 - fi - sleep 5 - done -} - -function check_benchmark_for_desired_state(){ - benchmark_name=$1 - desired_state=$2 - timeout=${3:-500s} - export -f wait_for_benchmark - export -f get_benchmark_state - if ! timeout -k $timeout $timeout bash -c "wait_for_benchmark $benchmark_name $desired_state" - then - echo "Timeout exceeded for: "$benchmark_name - - counter=3 - until [ $counter -gt $# ] - do - echo "Logs from "${@:$counter} - kubectl -n benchmark-operator logs --tail=40 ${@:$counter} - counter=$(( counter+1 )) - done - return 1 - fi - echo $(kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath={.status.state}) - -} - -# Takes 2 or more arguments: 'command to run', 'time to wait until true' -# Any additional arguments will be passed to kubectl -n benchmark-operator logs to provide logging if a timeout occurs -function wait_for() { - if ! timeout -k $2 $2 $1 - then - echo "Timeout exceeded for: "$1 - - counter=3 - until [ $counter -gt $# ] - do - echo "Logs from "${@:$counter} - kubectl -n benchmark-operator logs --tail=40 ${@:$counter} - counter=$(( counter+1 )) - done - return 1 - fi - return 0 -} - -function error { - if [[ $ERRORED == "true" ]] - then - exit - fi - - ERRORED=true - - echo "Error caught. Dumping logs before exiting" - echo "Benchmark operator Logs" - kubectl -n benchmark-operator logs --tail=200 -l control-plane=controller-manager -c manager -} - - - -function wait_for_backpack() { - echo "Waiting for backpack to complete before starting benchmark test" - - uuid=$1 - count=0 - max_count=60 - while [[ $count -lt $max_count ]] - do - if [[ `kubectl -n benchmark-operator get daemonsets backpack-$uuid` ]] - then - desired=`kubectl -n benchmark-operator get daemonsets backpack-$uuid | grep -v NAME | awk '{print $2}'` - ready=`kubectl -n benchmark-operator get daemonsets backpack-$uuid | grep -v NAME | awk '{print $4}'` - if [[ $desired -eq $ready ]] - then - echo "Backpack complete. Starting benchmark" - break - fi - fi - count=$((count + 1)) - if [[ $count -ne $max_count ]] - then - sleep 6 - else - echo "Backpack failed to complete. Exiting" - exit 1 - fi - done -} - -function check_es() { - if [[ ${#} != 2 ]]; then - echo "Wrong number of arguments: ${#}" - return 1 - fi - local uuid=$1 - local index=${@:2} - for my_index in $index; do - python3 tests/check_es.py -s $es_url -u $uuid -i $my_index \ - || return 1 - done -} diff --git a/tests/full_test_file_trigger b/tests/full_test_file_trigger deleted file mode 100755 index 7859a425c..000000000 --- a/tests/full_test_file_trigger +++ /dev/null @@ -1,7 +0,0 @@ -build/Dockerfile -group_vars/all.yml -resources/operator.yaml -tests/common.sh -playbook.yml -test.sh -watches.yaml diff --git a/tests/test_api_load.sh b/tests/test_api_load.sh deleted file mode 100755 index 2e0fceb89..000000000 --- a/tests/test_api_load.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up API load Test" - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_api_load { - test_name=$1 - cr=$2 - delete_benchmark $cr - echo "Performing: ${test_name}" - kubectl apply -f ${cr} - benchmark_name=$(get_benchmark_name $cr) - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - check_benchmark_for_desired_state $benchmark_name Complete 500s - - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_api_load "API load" tests/test_crs/valid_api_load.yaml diff --git a/tests/test_backpack.sh b/tests/test_backpack.sh deleted file mode 100755 index 28c1d6b8e..000000000 --- a/tests/test_backpack.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up backpack" - kubectl delete -f resources/backpack_role.yaml - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_backpack { - backpack_requirements - cr=$1 - delete_benchmark $cr - kubectl apply -f $cr - benchmark_name=$(get_benchmark_name $cr) - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - if [[ $1 == "tests/test_crs/valid_backpack_daemonset.yaml" ]] - then - wait_for_backpack $uuid - else - byowl_pod=$(get_pod "app=byowl-$uuid" 300) - wait_for "kubectl -n benchmark-operator wait --for=condition=Initialized pods/$byowl_pod --timeout=500s" "500s" $byowl_pod - wait_for "kubectl -n benchmark-operator wait --for=condition=complete -l app=byowl-$uuid jobs --timeout=500s" "500s" $byowl_pod - fi - - indexes="cpu_vulnerabilities-metadata cpuinfo-metadata dmidecode-metadata k8s_nodes-metadata lspci-metadata meminfo-metadata sysctl-metadata ocp_network_operator-metadata ocp_install_config-metadata ocp_kube_apiserver-metadata ocp_dns-metadata ocp_kube_controllermanager-metadata" - if check_es "${long_uuid}" "${indexes}" - then - echo "Backpack test: Success" - else - echo "Failed to find data in ES" - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_backpack tests/test_crs/valid_backpack_daemonset.yaml -functional_test_backpack tests/test_crs/valid_backpack_init.yaml diff --git a/tests/test_byowl.sh b/tests/test_byowl.sh deleted file mode 100755 index b210c9f44..000000000 --- a/tests/test_byowl.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up byowl" - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_byowl { - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - cr=tests/test_crs/valid_byowl.yaml - - delete_benchmark $cr - benchmark_name=$(get_benchmark_name $cr) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - - check_benchmark_for_desired_state $benchmark_name Complete 500s - echo "BYOWL test: Success" - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_byowl diff --git a/tests/test_crs/valid_servicemesh.yaml b/tests/test_crs/valid_servicemesh.yaml deleted file mode 100644 index c41fab396..000000000 --- a/tests/test_crs/valid_servicemesh.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: servicemesh-benchmark - namespace: benchmark-operator -spec: - workload: - name: servicemesh - args: - deployments: 2 - pods_per_deployment: 1 - workload_cpu: 1 - hyperfoil_threads: 1 - test_name: closedmodel - closedmodel: - steadystate_duration: 10s diff --git a/tests/test_fiod.sh b/tests/test_fiod.sh deleted file mode 100755 index 78fb7d8af..000000000 --- a/tests/test_fiod.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up fio" - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_fio { - wait_clean - test_name=$1 - cr=$2 - benchmark_name=$(get_benchmark_name $cr) - delete_benchmark $cr - echo "Performing: ${test_name}" - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - pod_count "app=fio-benchmark-$uuid" 2 300 - fio_pod=$(get_pod "app=fiod-client-$uuid" 300) - check_benchmark_for_desired_state $benchmark_name Complete 1200s - kubectl -n benchmark-operator logs $fio_pod > /tmp/$fio_pod.log - - - - - indexes="ripsaw-fio-results ripsaw-fio-log ripsaw-fio-analyzed-result" - if check_es "${long_uuid}" "${indexes}" - then - echo "${test_name} test: Success" - else - echo "Failed to find data for ${test_name} in ES" - kubectl logs "$fio_pod" -n benchmark-operator - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -kubectl label nodes -l node-role.kubernetes.io/worker= kernel-cache-dropper=yes --overwrite -functional_test_fio "Fio distributed" tests/test_crs/valid_fiod.yaml -openshift_storage_present=$(oc get namespace | awk '/openshift-storage/' | wc -l) -if [ $openshift_storage_present -gt 0 ] ; then - oc patch OCSInitialization ocsinit -n openshift-storage --type json --patch \ - '[{ "op": "replace", "path": "/spec/enableCephTools", "value": true }]' - drop_cache_pods=$(oc -n openshift-storage get pod | awk '/drop/' | awk '/unning/' | wc -l) - if [ $drop_cache_pods -eq 0 ] ; then - oc create -f roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml - kubectl wait --for=condition=Initialized pods/rook-ceph-osd-cache-drop -n openshift-storage --timeout=100s - fi - sleep 5 - functional_test_fio "Fio cache drop" tests/test_crs/valid_fiod_ocs_cache_drop.yaml -fi -functional_test_fio "Fio distributed - bsrange" tests/test_crs/valid_fiod_bsrange.yaml -functional_test_fio "Fio hostpath distributed" tests/test_crs/valid_fiod_hostpath.yaml diff --git a/tests/test_flent.sh b/tests/test_flent.sh deleted file mode 100755 index 84d059cfe..000000000 --- a/tests/test_flent.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up Flent" - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_flent { - test_name=$1 - cr=$2 - delete_benchmark $cr - echo "Performing: ${test_name}" - benchmark_name=$(get_benchmark_name $cr) - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - check_benchmark_for_desired_state $benchmark_name Complete 800s - - index="ripsaw-flent-results" - if check_es "${long_uuid}" "${index}" - then - echo "${test_name} test: Success" - else - echo "Failed to find data for ${test_name} in ES" - kubectl logs "$flent_client_pod" -n benchmark-operator - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_flent "Flent without resources definition" tests/test_crs/valid_flent.yaml -functional_test_flent "Flent with resources definition" tests/test_crs/valid_flent_resources.yaml diff --git a/tests/test_fs_drift.sh b/tests/test_fs_drift.sh deleted file mode 100755 index ac7d26d17..000000000 --- a/tests/test_fs_drift.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -set -xeo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up fs_drift" - wait_clean -} - - -trap error ERR -trap finish EXIT - -function functional_test_fs_drift { - test_name=$1 - cr=$2 - delete_benchmark $cr - echo "Performing: ${test_name}" - benchmark_name=$(get_benchmark_name $cr) - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - fsdrift_pod=$(get_pod "app=fs-drift-benchmark-$uuid" 300) - check_benchmark_for_desired_state $benchmark_name Complete 600s - indexes="ripsaw-fs-drift-results ripsaw-fs-drift-rsptimes ripsaw-fs-drift-rates-over-time" - if check_es "${long_uuid}" "${indexes}" - then - echo "${test_name} test: Success" - else - echo "Failed to find data for ${test_name} in ES" - kubectl logs "$fsdrift_pod" -n benchmark-operator - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_fs_drift "fs-drift" tests/test_crs/valid_fs_drift.yaml -functional_test_fs_drift "fs-drift hostpath" tests/test_crs/valid_fs_drift_hostpath.yaml diff --git a/tests/test_hammerdb.sh b/tests/test_hammerdb.sh deleted file mode 100755 index 78c7aa145..000000000 --- a/tests/test_hammerdb.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -xeo pipefail - -source tests/common.sh - -function initdb_pod { - echo "Setting up a MS-SQL DB Pod" - kubectl apply -f tests/mssql.yaml - mssql_pod=$(get_pod "app=mssql" 300 "sql-server") - kubectl wait --for=condition=Ready "pods/$mssql_pod" --namespace sql-server --timeout=300s -} - -function finish { - echo "Cleaning up hammerdb" - kubectl delete -f tests/mssql.yaml - wait_clean -} - -trap finish EXIT - -function functional_test_hammerdb { - initdb_pod - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - cr=tests/test_crs/valid_hammerdb.yaml - delete_benchmark $cr - benchmark_name=$(get_benchmark_name $cr) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - # Wait for the workload pod to run the actual workload - hammerdb_workload_pod=$(get_pod "app=hammerdb_workload-$uuid" 300) - check_benchmark_for_desired_state $benchmark_name Complete 900s - - index="ripsaw-hammerdb-results" - if check_es "${long_uuid}" "${index}" - then - echo "Hammerdb test: Success" - else - echo "Failed to find data for HammerDB test in ES" - kubectl logs "$hammerdb_workload_pod" --namespace benchmark-operator - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_hammerdb diff --git a/tests/test_image_pull.sh b/tests/test_image_pull.sh deleted file mode 100755 index 07a47309b..000000000 --- a/tests/test_image_pull.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up Image Pull Test" - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_image_pull { - test_name=$1 - cr=$2 - delete_benchmark $cr - echo "Performing: ${test_name}" - kubectl apply -f ${cr} - benchmark_name=$(get_benchmark_name $cr) - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - check_benchmark_for_desired_state $benchmark_name Complete 500s - - index="image-pull-results" - if check_es "${long_uuid}" "${index}" - then - echo "${test_name} test: Success" - else - echo "Failed to find data for ${test_name} in ES" - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_image_pull "Image Pull" tests/test_crs/valid_image_pull.yaml diff --git a/tests/test_iperf3.sh b/tests/test_iperf3.sh deleted file mode 100755 index 18efecbe0..000000000 --- a/tests/test_iperf3.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up iperf3" - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_iperf { - echo "Performing iperf3: ${1}" - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - cr=tests/test_crs/valid_iperf3.yaml - delete_benchmark $cr - benchmark_name=$(get_benchmark_name $cr) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" -e "s/hostnetwork:.*/${1}/g" $cr | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - iperf_server_pod=$(get_pod "app=iperf3-bench-server-$uuid" 300) - wait_for "kubectl -n benchmark-operator wait --for=condition=Initialized -l app=iperf3-bench-server-$uuid pods --timeout=300s" "300s" $iperf_server_pod - iperf_client_pod=$(get_pod "app=iperf3-bench-client-$uuid" 300) - check_benchmark_for_desired_state $benchmark_name Complete 600s - sleep 5 - # ensuring that iperf actually ran and we can access metrics - kubectl logs "$iperf_client_pod" --namespace benchmark-operator | grep "iperf Done." - echo "iperf ${1}: Success" - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_iperf "hostnetwork: false" -functional_test_iperf "hostnetwork: true" diff --git a/tests/test_kubeburner.sh b/tests/test_kubeburner.sh deleted file mode 100755 index 88f57883f..000000000 --- a/tests/test_kubeburner.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - [[ $check_logs == 1 ]] && kubectl logs -l app=kube-burner-benchmark-$uuid -n benchmark-operator - echo "Cleaning up kube-burner" - kubectl delete -f resources/kube-burner-role.yml --ignore-not-found - kubectl delete ns -l kube-burner-uuid=${long_uuid} - wait_clean -} - - -trap error ERR -trap finish EXIT - -function functional_test_kubeburner { - workload_name=$1 - metrics_profile=$2 - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - cr=tests/test_crs/valid_kube-burner.yaml - delete_benchmark $cr - benchmark_name=$(get_benchmark_name $cr) - check_logs=0 - kubectl apply -f resources/kube-burner-role.yml - echo "Performing kube-burner: ${workload_name}" - sed -e "s/WORKLOAD/${workload_name}/g" -e "s/PROMETHEUS_TOKEN/${token}/g" -e "s/METRICS_PROFILE/${metrics_profile}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - check_logs=1 - check_benchmark_for_desired_state $benchmark_name Complete 1800s - - index="ripsaw-kube-burner" - if check_es "${long_uuid}" "${index}" - then - echo "kube-burner ${workload_name}: Success" - else - echo "Failed to find data for kube-burner ${workload_name} in ES" - exit 1 - fi - kubectl delete ns -l kube-burner-uuid=${long_uuid} - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_kubeburner cluster-density metrics-aggregated.yaml -functional_test_kubeburner node-density metrics.yaml -functional_test_kubeburner node-density-heavy metrics.yaml -functional_test_kubeburner max-namespaces metrics-aggregated.yaml -functional_test_kubeburner max-services metrics-aggregated.yaml -functional_test_kubeburner concurrent-builds metrics-aggregated.yaml \ No newline at end of file diff --git a/tests/test_list b/tests/test_list deleted file mode 100755 index 0e0208c32..000000000 --- a/tests/test_list +++ /dev/null @@ -1,17 +0,0 @@ -test_backpack.sh -test_ycsb.sh -test_fiod.sh -test_fs-drift.sh -test_smallfile.sh -test_pgbench.sh -test_iperf3.sh -test_sysbench.sh -test_uperf.sh -test_byowl.sh -test_hammerdb.sh -test_vegeta.sh -test_scale_openshift.sh -test_stressng.sh -test_kubeburner.sh -test_flent.sh -test_log_generator.sh diff --git a/tests/test_log_generator.sh b/tests/test_log_generator.sh deleted file mode 100755 index 3d394f981..000000000 --- a/tests/test_log_generator.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up Log Generator Test" - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_log_generator { - test_name=$1 - cr=$2 - delete_benchmark $cr - benchmark_name=$(get_benchmark_name $cr) - echo "Performing: ${test_name}" - kubectl apply -f ${cr} - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - log_gen_pod=$(get_pod "app=log-generator-$uuid" 300) - check_benchmark_for_desired_state $benchmark_name Complete 800s - - index="log-generator-results" - if check_es "${long_uuid}" "${index}" - then - echo "${test_name} test: Success" - else - echo "Failed to find data for ${test_name} in ES" - kubectl logs "$log_gen_pod" -n benchmark-operator - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_log_generator "Log Generator" tests/test_crs/valid_log_generator.yaml diff --git a/tests/test_pgbench.sh b/tests/test_pgbench.sh deleted file mode 100755 index d8d33a249..000000000 --- a/tests/test_pgbench.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up pgbench" - wait_clean -} - -trap error ERR -trap finish EXIT - -# Note we don't test persistent storage here -function functional_test_pgbench { - # stand up postgres deployment -cat << EOF | kubectl apply -f - -apiVersion: v1 -kind: ConfigMap -metadata: - name: postgres-config - namespace: benchmark-operator - labels: - app: postgres -data: - POSTGRES_DB: cidb - POSTGRES_USER: ci - POSTGRES_PASSWORD: ci - PGDATA: /var/lib/postgresql/data/pgdata -EOF -cat << EOF | kubectl apply -f - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgres - namespace: benchmark-operator -spec: - selector: - matchLabels: - app: postgres - replicas: 1 - selector: - matchLabels: - app: postgres - template: - metadata: - labels: - app: postgres - spec: - containers: - - name: postgres - image: postgres:10.4 - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 5432 - envFrom: - - configMapRef: - name: postgres-config -EOF - postgres_pod=$(get_pod 'app=postgres' 300) - # get the postgres pod IP - postgres_ip=0 - counter=0 - until [[ $postgres_ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ || $counter -eq 10 ]]; do - let counter+=1 - postgres_ip=$(kubectl get pod -n benchmark-operator $postgres_pod --template={{.status.podIP}}) - sleep 2 - done - # deploy the test CR with the postgres pod IP - cr=tests/test_crs/valid_pgbench.yaml - delete_benchmark $cr - benchmark_name=$(get_benchmark_name $cr) - sed s/host:/host:\ ${postgres_ip}/ $cr | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - pgbench_pod=$(get_pod "app=pgbench-client-$uuid" 300) - check_benchmark_for_desired_state $benchmark_name Complete 500s - - index="ripsaw-pgbench-summary ripsaw-pgbench-raw" - if check_es "${long_uuid}" "${index}" - then - echo "pgbench test: Success" - else - echo "Failed to find data for PGBench in ES" - kubectl logs -n benchmark-operator $pgbench_pod - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_pgbench diff --git a/tests/test_scale_openshift.sh b/tests/test_scale_openshift.sh deleted file mode 100755 index 42b38ab79..000000000 --- a/tests/test_scale_openshift.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up Scale Test" - kubectl delete -f resources/scale_role.yaml --ignore-not-found - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_scale_openshift { - test_name=$1 - cr=$2 - benchmark_name=$(get_benchmark_name $cr) - delete_benchmark $cr - # Apply scale role and service account - kubectl apply -f resources/scale_role.yaml - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - echo "Performing: ${test_name}" - sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - scale_pod=$(get_pod "app=scale-$uuid" 300) - check_benchmark_for_desired_state $benchmark_name Complete 500s - index="openshift-cluster-timings" - if check_es "${long_uuid}" "${index}" - then - echo "${test_name} test: Success" - else - echo "Failed to find data for ${test_name} in ES" - kubectl logs "$scale_pod" -n benchmark-operator - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_scale_openshift "Scale Up" tests/test_crs/valid_scale_up.yaml -functional_test_scale_openshift "Scale Down" tests/test_crs/valid_scale_down.yaml diff --git a/tests/test_servicemesh.sh b/tests/test_servicemesh.sh deleted file mode 100755 index 6f5bf18d0..000000000 --- a/tests/test_servicemesh.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up ServiceMesh" - long_uuid=$(get_uuid 0) - uuid=${long_uuid:0:8} - kubectl delete --ignore-not-found=true namespace example-benchmark-controlplane-$uuid - kubectl delete --ignore-not-found=true namespace example-benchmark-workload-$uuid - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_servicemesh { - cr=tests/test_crs/valid_servicemesh.yaml - benchmark_name=$(get_benchmark_name $cr) - delete_benchmark $cr - kubectl apply -f $cr - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - # wait until the job appears - count=0 - max_count=60 - while [[ $count -lt $max_count ]] - do - if kubectl get --namespace benchmark-operator job example-benchmark-$uuid; then - break - fi - sleep 15 - count=$((count + 1)) - done - - wait_for "kubectl -n benchmark-operator wait --for=condition=complete jobs --timeout=300s example-benchmark-$uuid" "300s" - - job_pod=$(get_pod app=example-benchmark-$uuid 30) - # ensuring that uperf actually ran and we can access metrics - INFO=$(kubectl logs $job_pod --namespace benchmark-operator | jq .info) - if [ -n "$INFO" ]; then - echo "Successful: $INFO" - else - echo "Failed to verify benchmark results" - exit 1 - fi - delete_benchmark $cr -} - -functional_test_servicemesh diff --git a/tests/test_smallfile.sh b/tests/test_smallfile.sh deleted file mode 100755 index b921adfb2..000000000 --- a/tests/test_smallfile.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up smallfile" - wait_clean -} - - -trap error ERR -trap finish EXIT - -function functional_test_smallfile { - test_name=$1 - cr=$2 - delete_benchmark $cr - benchmark_name=$(get_benchmark_name $cr) - echo "Performing: ${test_name}" - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - check_benchmark_for_desired_state $benchmark_name Complete 500s - - indexes="ripsaw-smallfile-results ripsaw-smallfile-rsptimes" - if check_es "${long_uuid}" "${indexes}" - then - echo "${test_name} test: Success" - else - echo "Failed to find data for ${test_name} in ES" - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_smallfile "smallfile" tests/test_crs/valid_smallfile.yaml -functional_test_smallfile "smallfile hostpath" tests/test_crs/valid_smallfile_hostpath.yaml diff --git a/tests/test_stressng.sh b/tests/test_stressng.sh deleted file mode 100755 index d9c98c05c..000000000 --- a/tests/test_stressng.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -set -xeo pipefail - -source tests/common.sh - - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - echo "Cleaning up stressng" - wait_clean -} - -trap finish EXIT - -function functional_test_stressng { - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - cr=tests/test_crs/valid_stressng.yaml - delete_benchmark $cr - benchmark_name=$(get_benchmark_name $cr) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - # Wait for the workload pod to run the actual workload - stressng_workload_pod=$(get_pod "app=stressng_workload-$uuid" 300) - check_benchmark_for_desired_state $benchmark_name Complete 500s - - - index="ripsaw-stressng-results" - if check_es "${long_uuid}" "${index}" - then - echo "StressNG test: Success" - else - echo "Failed to find data for StressNG test in ES" - kubectl logs "$stressng_workload_pod" --namespace benchmark-operator - exit 1 - fi - - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_stressng diff --git a/tests/test_sysbench.sh b/tests/test_sysbench.sh deleted file mode 100755 index 78571fe2c..000000000 --- a/tests/test_sysbench.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up sysbench" - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_sysbench { - cr=tests/test_crs/valid_sysbench.yaml - delete_benchmark $cr - kubectl apply -f $cr - benchmark_name=$(get_benchmark_name $cr) - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - sysbench_pod=$(get_pod "app=sysbench-$uuid" 300) - check_benchmark_for_desired_state $benchmark_name Complete 800s - # sleep isn't needed as the sysbench is kind: job so once it's complete we can access logs - # ensuring the run has actually happened - kubectl logs "$sysbench_pod" --namespace benchmark-operator | grep "execution time" - echo "Sysbench test: Success" - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_sysbench diff --git a/tests/test_uperf.sh b/tests/test_uperf.sh deleted file mode 100755 index 3621268fe..000000000 --- a/tests/test_uperf.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up Uperf" - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_uperf { - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - test_name=$1 - cr=$2 - benchmark_name=$(get_benchmark_name $cr) - delete_benchmark $cr - echo "Performing: ${test_name}" - sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - check_benchmark_for_desired_state $benchmark_name Complete 1800s - - - - index="ripsaw-uperf-results" - if check_es "${long_uuid}" "${index}" - then - echo "${test_name} test: Success" - else - echo "Failed to find data for ${test_name} in ES" - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_uperf "Uperf without resources definition" tests/test_crs/valid_uperf.yaml -functional_test_uperf "Uperf with ServiceIP" tests/test_crs/valid_uperf_serviceip.yaml -functional_test_uperf "Uperf with resources definition and hostNetwork" tests/test_crs/valid_uperf_resources.yaml -functional_test_uperf "Uperf with networkpolicy" tests/test_crs/valid_uperf_networkpolicy.yaml diff --git a/tests/test_vegeta.sh b/tests/test_vegeta.sh deleted file mode 100755 index ca3d19a19..000000000 --- a/tests/test_vegeta.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - [[ $check_logs == 1 ]] && kubectl logs -l app=vegeta-benchmark-$uuid -n benchmark-operator - echo "Cleaning up vegeta" - wait_clean -} - - -trap error ERR -trap finish EXIT - -function functional_test_vegeta { - check_logs=0 - test_name=$1 - cr=$2 - benchmark_name=$(get_benchmark_name $cr) - delete_benchmark $cr - echo "Performing: ${test_name}" - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - check_benchmark_for_desired_state $benchmark_name Complete 500s - check_logs=1 - - index="ripsaw-vegeta-results" - if check_es "${long_uuid}" "${index}" - then - echo "${test_name} test: Success" - else - echo "Failed to find data for ${test_name} in ES" - exit 1 - fi - delete_benchmark $cr - -} - -figlet $(basename $0) -functional_test_vegeta "Vegeta benchmark" tests/test_crs/valid_vegeta.yaml -functional_test_vegeta "Vegeta benchmark hostnetwork" tests/test_crs/valid_vegeta_hostnetwork.yaml diff --git a/tests/test_ycsb.sh b/tests/test_ycsb.sh deleted file mode 100755 index dc113fca4..000000000 --- a/tests/test_ycsb.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up ycsb" - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_ycsb { - # stand up mongo deployment -cat << EOF | kubectl apply -f - -apiVersion: v1 -kind: Service -metadata: - name: mongo - namespace: benchmark-operator - labels: - name: mongo -spec: - ports: - - port: 27017 - targetPort: 27017 - clusterIP: None - selector: - role: mongo -EOF -cat << EOF | kubectl apply -f - -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mongo - namespace: benchmark-operator -spec: - selector: - matchLabels: - role: mongo - serviceName: "mongo" - replicas: 1 - selector: - matchLabels: - role: mongo - template: - metadata: - labels: - role: mongo - environment: test - spec: - terminationGracePeriodSeconds: 10 - containers: - - name: mongo - image: mongo - command: ["/bin/sh"] - args: ["-c", "mkdir -p /tmp/data/db; mongod --bind_ip 0.0.0.0 --dbpath /tmp/data/db"] - ports: - - containerPort: 27017 -EOF - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - cr=tests/test_crs/valid_ycsb-mongo.yaml - delete_benchmark $cr - benchmark_name=$(get_benchmark_name $cr) - sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - ycsb_load_pod=$(get_pod "name=ycsb-load-$uuid" 300) - check_benchmark_for_desired_state $benchmark_name Complete 500s - - indexes="ripsaw-ycsb-summary ripsaw-ycsb-results" - if check_es "${long_uuid}" "${indexes}" - then - echo "ycsb test: Success" - else - echo "Failed to find data for ${test_name} in ES" - kubectl logs -n benchmark-operator $ycsb_load_pod - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_ycsb From 67134f1d71ee689c285d6bc4c1c9b78b5a408568 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Fri, 17 Sep 2021 13:44:17 +0200 Subject: [PATCH 149/249] Concurrency for e2e workflow Signed-off-by: Raul Sevilla --- .github/workflows/e2e.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e67062530..9a7c0b6a0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -13,6 +13,8 @@ on: img: description: Benchmark-operator image location default: quay.io/rht_perf_ci/benchmark-operator:e2e +concurrency: + group: e2e-ci jobs: e2e: name: E2E testing From 0e3ccfdd6a3a5a0b08ee84e2a65a81757f5debcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Wed, 29 Sep 2021 10:54:01 +0200 Subject: [PATCH 150/249] Kube burner configmap config (#690) Get kube-burner config from configmap Signed-off-by: Raul Sevilla --- docs/e2e-ci.md | 2 +- docs/kube-burner.md | 37 ++ e2e/008-kube-burner.bats | 11 + e2e/kube-burner/concurrent-builds.yaml | 2 +- e2e/kube-burner/configmap-cfg.yaml | 79 +++++ e2e/kube-burner/configmap.yaml | 32 ++ e2e/kube-burner/max-namespaces.yaml | 2 +- e2e/kube-burner/max-services.yaml | 2 +- e2e/kube-burner/node-density-heavy.yaml | 2 +- e2e/kube-burner/node-density.yaml | 2 +- roles/kube-burner/tasks/main.yml | 325 +++++++++--------- .../kube-burner/templates/kube-burner.yml.j2 | 13 +- 12 files changed, 341 insertions(+), 168 deletions(-) create mode 100644 e2e/kube-burner/configmap-cfg.yaml create mode 100644 e2e/kube-burner/configmap.yaml diff --git a/docs/e2e-ci.md b/docs/e2e-ci.md index d4193eabe..cf0826dad 100644 --- a/docs/e2e-ci.md +++ b/docs/e2e-ci.md @@ -27,7 +27,7 @@ A Bats test file is a Bash script with special syntax for defining test cases. U } ``` -If every command in the test case exits with a 0 status code (success), the test passes. This workflow uses bats framework to execute end to end tests, these tests are defined in files with the termination *.bats under the [e2e directory](./e2e) +If every command in the test case exits with a 0 status code (success), the test passes. This workflow uses bats framework to execute end to end tests, these tests are defined in files with the termination *.bats under the project's e2e directory. ### Events diff --git a/docs/kube-burner.md b/docs/kube-burner.md index 329d5fbcc..5bee08142 100644 --- a/docs/kube-burner.md +++ b/docs/kube-burner.md @@ -253,3 +253,40 @@ It supports different severities: - critical: Prints a fatal message with the alarm description to stdout and exits execution inmediatly with rc != 0 More information can be found at the [Kube-burner docs site.](https://kube-burner.readthedocs.io/en/latest/alerting/) + +## Reading configuration from a configmap + +Kube-burner is able to fetch it's own configuration from a configmap. To do so you just have to set the argument `configmap` pointing to a configmap in the same namespace where kube-burner is in the CR. This configmap needs to have a config.yml file to hold the main kube-burner's configuration file(apart from the required object templates), and optionally can contain a metrics.yml and alerts.yml files. An example configuration CR would look like: + +```yaml +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-configmap-cfg + namespace: benchmark-operator +spec: + metadata: + collection: false + prometheus: + prom_token: ThisIsNotAValidToken + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + configmap: kube-burner-config + cleanup: true + pin_server: {"node-role.kubernetes.io/worker": ""} + image: quay.io/cloud-bulldozer/kube-burner:latest + log_level: info + step: 30s + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule +``` + +To create a configmap with the kube-burner configurations you can use `kubectl create configmap --from-file= kube-burner-config` + diff --git a/e2e/008-kube-burner.bats b/e2e/008-kube-burner.bats index c13f587b3..e5f7e7afc 100755 --- a/e2e/008-kube-burner.bats +++ b/e2e/008-kube-burner.bats @@ -61,13 +61,24 @@ ES_INDEX=ripsaw-kube-burner check_es } +@test "kube-burner-configmap" { + CR=kube-burner/configmap.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + setup_file() { basic_setup kubectl apply -f ../resources/kube-burner-role.yml + kubectl apply -f kube-burner/configmap-cfg.yaml } teardown_file() { kubectl delete -f ../resources/kube-burner-role.yml + kubectl delete -f kube-burner/configmap-cfg.yaml } teardown() { diff --git a/e2e/kube-burner/concurrent-builds.yaml b/e2e/kube-burner/concurrent-builds.yaml index 76e1a9813..b56c51f82 100644 --- a/e2e/kube-burner/concurrent-builds.yaml +++ b/e2e/kube-burner/concurrent-builds.yaml @@ -8,7 +8,7 @@ spec: elasticsearch: url: ${ES_SERVER} metadata: - collection: true + collection: false prometheus: es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} diff --git a/e2e/kube-burner/configmap-cfg.yaml b/e2e/kube-burner/configmap-cfg.yaml new file mode 100644 index 000000000..9c2579c5d --- /dev/null +++ b/e2e/kube-burner/configmap-cfg.yaml @@ -0,0 +1,79 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-burner-config + namespace: benchmark-operator +data: + alerts.yml: "- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m]))[5m:]) + > 0.01\n description: 5 minutes avg. etcd fsync latency on {{$labels.pod}} higher + than 10ms {{$value}}\n severity: warning\n\n- expr: avg_over_time(histogram_quantile(0.99, + rate(etcd_network_peer_round_trip_time_seconds_bucket[5m]))[5m:]) > 0.1\n description: + 5 minutes avg. etcd network peer round trip on {{$labels.pod}} higher than 100ms + {{$value}}\n severity: warning \n" + config.yml: | + --- + global: + writeToFile: false + metricsDirectory: collected-metrics + indexerConfig: + enabled: true + esServers: [{{ .ES_SERVER }}] + insecureSkipVerify: true + defaultIndex: ripsaw-kube-burner + type: elastic + measurements: + - name: podLatency + esIndex: ripsaw-kube-burner + + jobs: + - name: basic-job + jobIterations: 5 + qps: 20 + burst: 20 + namespacedIterations: false + namespace: basic-job + waitWhenFinished: true + podWait: false + objects: + + - objectTemplate: pod.yml + replicas: 1 + inputVars: + containerImage: gcr.io/google_containers/pause-amd64:3.0 + metrics.yml: | + - query: count(kube_namespace_created) + metricName: namespaceCount + instant: true + + - query: sum(kube_pod_status_phase{}) by (phase) + metricName: podStatusCount + instant: true + + - query: count(kube_secret_info{}) + metricName: secretCount + instant: true + + - query: count(kube_deployment_labels{}) + metricName: deploymentCount + instant: true + + - query: count(kube_configmap_info{}) + metricName: configmapCount + instant: true + pod.yml: | + kind: Pod + apiVersion: v1 + metadata: + name: basic-pod-{{.Iteration}} + spec: + nodeSelector: + node-role.kubernetes.io/worker: "" + containers: + - name: basic-container + image: {{.containerImage}} + ports: + - containerPort: 8080 + protocol: TCP + imagePullPolicy: IfNotPresent + securityContext: + privileged: false diff --git a/e2e/kube-burner/configmap.yaml b/e2e/kube-burner/configmap.yaml new file mode 100644 index 000000000..e6ee8b971 --- /dev/null +++ b/e2e/kube-burner/configmap.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-configmap-cfg + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + prometheus: + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + extra_env_vars: + ES_SERVER: ${ES_SERVER} + configmap: kube-burner-config + cleanup: true + pin_server: {"node-role.kubernetes.io/worker": ""} + image: quay.io/cloud-bulldozer/kube-burner:latest + log_level: info + step: 30s + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/max-namespaces.yaml b/e2e/kube-burner/max-namespaces.yaml index daaf3fe6e..db17594d1 100644 --- a/e2e/kube-burner/max-namespaces.yaml +++ b/e2e/kube-burner/max-namespaces.yaml @@ -8,7 +8,7 @@ spec: elasticsearch: url: ${ES_SERVER} metadata: - collection: true + collection: false prometheus: es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} diff --git a/e2e/kube-burner/max-services.yaml b/e2e/kube-burner/max-services.yaml index dfdc0e6d2..dc737d5ad 100644 --- a/e2e/kube-burner/max-services.yaml +++ b/e2e/kube-burner/max-services.yaml @@ -8,7 +8,7 @@ spec: elasticsearch: url: ${ES_SERVER} metadata: - collection: true + collection: false prometheus: es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} diff --git a/e2e/kube-burner/node-density-heavy.yaml b/e2e/kube-burner/node-density-heavy.yaml index 543b2fd0d..10d00b9b4 100644 --- a/e2e/kube-burner/node-density-heavy.yaml +++ b/e2e/kube-burner/node-density-heavy.yaml @@ -8,7 +8,7 @@ spec: elasticsearch: url: ${ES_SERVER} metadata: - collection: true + collection: false prometheus: es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} diff --git a/e2e/kube-burner/node-density.yaml b/e2e/kube-burner/node-density.yaml index 76775d6a4..4f00d84fd 100644 --- a/e2e/kube-burner/node-density.yaml +++ b/e2e/kube-burner/node-density.yaml @@ -8,7 +8,7 @@ spec: elasticsearch: url: ${ES_SERVER} metadata: - collection: true + collection: false prometheus: es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} diff --git a/roles/kube-burner/tasks/main.yml b/roles/kube-burner/tasks/main.yml index 0cf7e0eab..505639e41 100644 --- a/roles/kube-burner/tasks/main.yml +++ b/roles/kube-burner/tasks/main.yml @@ -15,167 +15,170 @@ - workload_args.remote_config is defined - workload_args.remote_config - - name: Create cluster-density configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'cluster-density.yml.j2')}}" - metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" - build.yml: "{{ lookup('file', 'build.yml')}}" - buildconfig.yml: "{{ lookup('file', 'buildconfig.yml')}}" - deployment.yml: "{{ lookup('file', 'deployment.yml')}}" - imagestream.yml: "{{ lookup('file', 'imagestream.yml')}}" - route.yml: "{{ lookup('file', 'route.yml')}}" - secret.yml: "{{ lookup('file', 'secret.yml')}}" - service.yml: "{{ lookup('file', 'service.yml')}}" - configmap.yml: "{{ lookup('file', 'configmap.yml')}}" - when: workload_args.workload == "cluster-density" - - - name: Create node-density or pod-density configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'node-pod-density.yml.j2')}}" - metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" - pod.yml: "{{ lookup('file', 'pod.yml')}}" - when: workload_args.workload == "node-density" or workload_args.workload == "pod-density" - - - name: Create node-density-heavy configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'node-density-heavy.yml.j2')}}" - metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" - app-deployment.yml: "{{ lookup('file', 'app-deployment.yml')}}" - postgres-deployment.yml: "{{ lookup('file', 'postgres-deployment.yml')}}" - postgres-service.yml: "{{ lookup('file', 'postgres-service.yml')}}" - when: workload_args.workload == "node-density-heavy" - - - name: Create max-namespaces configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'max-namespaces.yml.j2')}}" - metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" - app-deployment.yml: "{{ lookup('file', 'app-deployment.yml')}}" - postgres-deployment.yml: "{{ lookup('file', 'postgres-deployment.yml')}}" - postgres-service.yml: "{{ lookup('file', 'postgres-service.yml')}}" - secret.yml: "{{ lookup('file', 'secret.yml')}}" - when: workload_args.workload == "max-namespaces" - - - name: Create networkpolicy-multitenant configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'networkpolicy-multitenant.yml.j2')}}" - nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" - networkpolicy-ingress.yaml: "{{ lookup('file', 'networkpolicy-ingress.yaml')}}" - networkpolicy-monitoring.yaml: "{{ lookup('file', 'networkpolicy-monitoring.yaml')}}" - networkpolicy-same-ns.yaml: "{{ lookup('file', 'networkpolicy-same-ns.yaml')}}" - when: workload_args.workload == "networkpolicy-multitenant" - - - name: Create networkpolicy-case2 configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'networkpolicy-case2.yml.j2')}}" - nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" - case2-networkpolicy.yaml: "{{ lookup('file', 'case2-networkpolicy.yaml')}}" - networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" - when: workload_args.workload == "networkpolicy-case2" - - - name: Create networkpolicy-case3 configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'networkpolicy-case3.yml.j2')}}" - nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" - case3-networkpolicy.yaml: "{{ lookup('file', 'case3-networkpolicy.yaml')}}" - networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" - when: workload_args.workload == "networkpolicy-case3" - - - - name: Create networkpolicy-cross-ns configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'networkpolicy-cross-ns.yml.j2')}}" - nginx-cross-ns.yaml: "{{ lookup('file', 'nginx-cross-ns.yaml')}}" - cross-ns-networkpolicy.yaml: "{{ lookup('file', 'cross-ns-networkpolicy.yaml')}}" - networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" - when: workload_args.workload == "networkpolicy-cross-ns" - - - name: Create max-services configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'max-services.yml.j2')}}" - metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" - simple-deployment.yml: "{{ lookup('file', 'simple-deployment.yml')}}" - service.yml: "{{ lookup('file', 'service.yml')}}" - - when: workload_args.workload == "max-services" - - - name: Create concurrent builds - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'concurrent-builds.yml.j2')}}" - metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" - buildconfig_triggers.yml: "{{ lookup('file', 'buildconfig_triggers.yml')}}" - configmap.yml: "{{ lookup('file', 'configmap.yml')}}" - imagestream.yml: "{{ lookup('file', 'imagestream.yml')}}" - when: workload_args.workload == "concurrent-builds" + - block: + + - name: Create cluster-density configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'cluster-density.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" + build.yml: "{{ lookup('file', 'build.yml')}}" + buildconfig.yml: "{{ lookup('file', 'buildconfig.yml')}}" + deployment.yml: "{{ lookup('file', 'deployment.yml')}}" + imagestream.yml: "{{ lookup('file', 'imagestream.yml')}}" + route.yml: "{{ lookup('file', 'route.yml')}}" + secret.yml: "{{ lookup('file', 'secret.yml')}}" + service.yml: "{{ lookup('file', 'service.yml')}}" + configmap.yml: "{{ lookup('file', 'configmap.yml')}}" + when: workload_args.workload == "cluster-density" + + - name: Create node-density or pod-density configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'node-pod-density.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" + pod.yml: "{{ lookup('file', 'pod.yml')}}" + when: workload_args.workload == "node-density" or workload_args.workload == "pod-density" + + - name: Create node-density-heavy configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'node-density-heavy.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" + app-deployment.yml: "{{ lookup('file', 'app-deployment.yml')}}" + postgres-deployment.yml: "{{ lookup('file', 'postgres-deployment.yml')}}" + postgres-service.yml: "{{ lookup('file', 'postgres-service.yml')}}" + when: workload_args.workload == "node-density-heavy" + + - name: Create max-namespaces configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'max-namespaces.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" + app-deployment.yml: "{{ lookup('file', 'app-deployment.yml')}}" + postgres-deployment.yml: "{{ lookup('file', 'postgres-deployment.yml')}}" + postgres-service.yml: "{{ lookup('file', 'postgres-service.yml')}}" + secret.yml: "{{ lookup('file', 'secret.yml')}}" + when: workload_args.workload == "max-namespaces" + + - name: Create networkpolicy-multitenant configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-multitenant.yml.j2')}}" + nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" + networkpolicy-ingress.yaml: "{{ lookup('file', 'networkpolicy-ingress.yaml')}}" + networkpolicy-monitoring.yaml: "{{ lookup('file', 'networkpolicy-monitoring.yaml')}}" + networkpolicy-same-ns.yaml: "{{ lookup('file', 'networkpolicy-same-ns.yaml')}}" + when: workload_args.workload == "networkpolicy-multitenant" + + - name: Create networkpolicy-case2 configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-case2.yml.j2')}}" + nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" + case2-networkpolicy.yaml: "{{ lookup('file', 'case2-networkpolicy.yaml')}}" + networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" + when: workload_args.workload == "networkpolicy-case2" + + - name: Create networkpolicy-case3 configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-case3.yml.j2')}}" + nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" + case3-networkpolicy.yaml: "{{ lookup('file', 'case3-networkpolicy.yaml')}}" + networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" + when: workload_args.workload == "networkpolicy-case3" + + + - name: Create networkpolicy-cross-ns configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-cross-ns.yml.j2')}}" + nginx-cross-ns.yaml: "{{ lookup('file', 'nginx-cross-ns.yaml')}}" + cross-ns-networkpolicy.yaml: "{{ lookup('file', 'cross-ns-networkpolicy.yaml')}}" + networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" + when: workload_args.workload == "networkpolicy-cross-ns" + + - name: Create max-services configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'max-services.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" + simple-deployment.yml: "{{ lookup('file', 'simple-deployment.yml')}}" + service.yml: "{{ lookup('file', 'service.yml')}}" + + when: workload_args.workload == "max-services" + + - name: Create concurrent builds + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'concurrent-builds.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" + buildconfig_triggers.yml: "{{ lookup('file', 'buildconfig_triggers.yml')}}" + configmap.yml: "{{ lookup('file', 'configmap.yml')}}" + imagestream.yml: "{{ lookup('file', 'imagestream.yml')}}" + when: workload_args.workload == "concurrent-builds" + when: workload_args.configmap is not defined - name: Launching kube-burner job k8s: diff --git a/roles/kube-burner/templates/kube-burner.yml.j2 b/roles/kube-burner/templates/kube-burner.yml.j2 index f81bda7fc..f08c8e778 100644 --- a/roles/kube-burner/templates/kube-burner.yml.j2 +++ b/roles/kube-burner/templates/kube-burner.yml.j2 @@ -33,8 +33,12 @@ spec: imagePullPolicy: Always securityContext: readOnlyRootFilesystem: false -{% if workload_args.extra_env_vars is defined and workload_args.extra_env_vars is mapping %} env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{% if workload_args.extra_env_vars is defined and workload_args.extra_env_vars is mapping %} {% for name, value in workload_args.extra_env_vars.items() %} - name: {{ name }} value: "{{ value }}" @@ -47,6 +51,9 @@ spec: kube-burner init {% if workload_args.remote_config is defined and workload_args.remote_config %} -c {{ workload_args.remote_config }} +{% elif workload_args.configmap is defined and workload_args.configmap %} + --configmap={{ workload_args.configmap }} + --namespace=${POD_NAMESPACE} {% else %} -c config.yml {% endif %} @@ -54,13 +61,16 @@ spec: -u {{ prometheus.prom_url }} -t {{ prometheus.prom_token }} --step={{ workload_args.step|default("30s") }} +{% if workload_args.configmap is not defined %} -m {{ workload_args.remote_metrics_profile|default("metrics.yaml", true) }} {% if workload_args.remote_alert_profile is defined and workload_args.remote_alert_profile %} -a {{ workload_args.remote_alert_profile }} {% endif %} +{% endif %} {% endif %} --uuid={{ uuid }} --log-level={{ workload_args.log_level|default("info") }} +{% if workload_args.configmap is not defined %} volumeMounts: - name: kube-burner-config mountPath: /tmp/kube-burner @@ -68,4 +78,5 @@ spec: - name: kube-burner-config configMap: name: kube-burner-config-{{ trunc_uuid }} +{% endif %} {% include "metadata.yml.j2" %} From f8f00fa1c6b7d9ec27f3a2cbe6e17322077f9c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Thu, 30 Sep 2021 10:34:08 +0200 Subject: [PATCH 151/249] Revert tests directory (#691) Signed-off-by: Raul Sevilla --- tests/check_es.py | 61 ++++ tests/common.sh | 297 ++++++++++++++++++ tests/full_test_file_trigger | 7 + tests/mssql.yaml | 43 +++ tests/test_api_load.sh | 35 +++ tests/test_backpack.sh | 51 +++ tests/test_byowl.sh | 32 ++ tests/test_crs/configmap-cfg.yaml | 79 +++++ tests/test_crs/configmap.yaml | 32 ++ tests/test_crs/valid_api_load.yaml | 13 + tests/test_crs/valid_backpack_daemonset.yaml | 15 + tests/test_crs/valid_backpack_init.yaml | 18 ++ tests/test_crs/valid_byowl.yaml | 22 ++ tests/test_crs/valid_fiod.yaml | 85 +++++ tests/test_crs/valid_fiod_bsrange.yaml | 86 +++++ tests/test_crs/valid_fiod_hostpath.yaml | 85 +++++ tests/test_crs/valid_fiod_ocs_cache_drop.yaml | 85 +++++ tests/test_crs/valid_flent.yaml | 31 ++ tests/test_crs/valid_flent_resources.yaml | 39 +++ tests/test_crs/valid_fs_drift.yaml | 30 ++ tests/test_crs/valid_fs_drift_hostpath.yaml | 31 ++ tests/test_crs/valid_hammerdb.yaml | 70 +++++ tests/test_crs/valid_image_pull.yaml | 18 ++ tests/test_crs/valid_iperf3.yaml | 32 ++ tests/test_crs/valid_kube-burner.yaml | 48 +++ tests/test_crs/valid_log_generator.yaml | 18 ++ tests/test_crs/valid_pgbench.yaml | 37 +++ tests/test_crs/valid_scale_down.yaml | 24 ++ tests/test_crs/valid_scale_up.yaml | 24 ++ tests/test_crs/valid_servicemesh.yaml | 16 + tests/test_crs/valid_smallfile.yaml | 46 +++ tests/test_crs/valid_smallfile_hostpath.yaml | 31 ++ tests/test_crs/valid_stressng.yaml | 34 ++ tests/test_crs/valid_sysbench.yaml | 23 ++ tests/test_crs/valid_uperf.yaml | 45 +++ tests/test_crs/valid_uperf_networkpolicy.yaml | 46 +++ tests/test_crs/valid_uperf_resources.yaml | 52 +++ tests/test_crs/valid_uperf_serviceip.yaml | 44 +++ tests/test_crs/valid_vegeta.yaml | 37 +++ tests/test_crs/valid_vegeta_hostnetwork.yaml | 33 ++ tests/test_crs/valid_ycsb-mongo.yaml | 30 ++ tests/test_fiod.sh | 67 ++++ tests/test_flent.sh | 46 +++ tests/test_fs_drift.sh | 46 +++ tests/test_hammerdb.sh | 48 +++ tests/test_image_pull.sh | 43 +++ tests/test_iperf3.sh | 42 +++ tests/test_kubeburner.sh | 57 ++++ tests/test_list | 17 + tests/test_log_generator.sh | 45 +++ tests/test_pgbench.sh | 98 ++++++ tests/test_scale_openshift.sh | 49 +++ tests/test_servicemesh.sh | 57 ++++ tests/test_smallfile.sh | 45 +++ tests/test_stressng.sh | 46 +++ tests/test_sysbench.sh | 37 +++ tests/test_uperf.sh | 48 +++ tests/test_vegeta.sh | 49 +++ tests/test_ycsb.sh | 91 ++++++ 59 files changed, 2816 insertions(+) create mode 100755 tests/check_es.py create mode 100755 tests/common.sh create mode 100755 tests/full_test_file_trigger create mode 100755 tests/mssql.yaml create mode 100755 tests/test_api_load.sh create mode 100755 tests/test_backpack.sh create mode 100755 tests/test_byowl.sh create mode 100644 tests/test_crs/configmap-cfg.yaml create mode 100644 tests/test_crs/configmap.yaml create mode 100644 tests/test_crs/valid_api_load.yaml create mode 100644 tests/test_crs/valid_backpack_daemonset.yaml create mode 100644 tests/test_crs/valid_backpack_init.yaml create mode 100644 tests/test_crs/valid_byowl.yaml create mode 100644 tests/test_crs/valid_fiod.yaml create mode 100644 tests/test_crs/valid_fiod_bsrange.yaml create mode 100644 tests/test_crs/valid_fiod_hostpath.yaml create mode 100644 tests/test_crs/valid_fiod_ocs_cache_drop.yaml create mode 100644 tests/test_crs/valid_flent.yaml create mode 100644 tests/test_crs/valid_flent_resources.yaml create mode 100644 tests/test_crs/valid_fs_drift.yaml create mode 100644 tests/test_crs/valid_fs_drift_hostpath.yaml create mode 100644 tests/test_crs/valid_hammerdb.yaml create mode 100644 tests/test_crs/valid_image_pull.yaml create mode 100644 tests/test_crs/valid_iperf3.yaml create mode 100644 tests/test_crs/valid_kube-burner.yaml create mode 100644 tests/test_crs/valid_log_generator.yaml create mode 100644 tests/test_crs/valid_pgbench.yaml create mode 100644 tests/test_crs/valid_scale_down.yaml create mode 100644 tests/test_crs/valid_scale_up.yaml create mode 100644 tests/test_crs/valid_servicemesh.yaml create mode 100644 tests/test_crs/valid_smallfile.yaml create mode 100644 tests/test_crs/valid_smallfile_hostpath.yaml create mode 100644 tests/test_crs/valid_stressng.yaml create mode 100644 tests/test_crs/valid_sysbench.yaml create mode 100644 tests/test_crs/valid_uperf.yaml create mode 100644 tests/test_crs/valid_uperf_networkpolicy.yaml create mode 100644 tests/test_crs/valid_uperf_resources.yaml create mode 100644 tests/test_crs/valid_uperf_serviceip.yaml create mode 100644 tests/test_crs/valid_vegeta.yaml create mode 100644 tests/test_crs/valid_vegeta_hostnetwork.yaml create mode 100644 tests/test_crs/valid_ycsb-mongo.yaml create mode 100755 tests/test_fiod.sh create mode 100755 tests/test_flent.sh create mode 100755 tests/test_fs_drift.sh create mode 100755 tests/test_hammerdb.sh create mode 100755 tests/test_image_pull.sh create mode 100755 tests/test_iperf3.sh create mode 100755 tests/test_kubeburner.sh create mode 100755 tests/test_list create mode 100755 tests/test_log_generator.sh create mode 100755 tests/test_pgbench.sh create mode 100755 tests/test_scale_openshift.sh create mode 100755 tests/test_servicemesh.sh create mode 100755 tests/test_smallfile.sh create mode 100755 tests/test_stressng.sh create mode 100755 tests/test_sysbench.sh create mode 100755 tests/test_uperf.sh create mode 100755 tests/test_vegeta.sh create mode 100755 tests/test_ycsb.sh diff --git a/tests/check_es.py b/tests/check_es.py new file mode 100755 index 000000000..efd203629 --- /dev/null +++ b/tests/check_es.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import elasticsearch +import sys +import ssl +import urllib3 + + +def _check_index(server, uuid, index, es_ssl): + + if es_ssl == "true": + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + ssl_ctx = ssl.create_default_context() + ssl_ctx.check_hostname = False + ssl_ctx.verify_mode = ssl.CERT_NONE + es = elasticsearch.Elasticsearch([server], send_get_body_as='POST', ssl_context=ssl_ctx, use_ssl=True) + else: + es = elasticsearch.Elasticsearch([server], send_get_body_as='POST') + es.indices.refresh(index=index) + results = es.search(index=index, body={'query': {'term': {'uuid.keyword': uuid}}}, size=1) + if results['hits']['total']['value'] > 0: + return 0 + else: + print("No result found in ES") + return 1 + + +def main(): + parser = argparse.ArgumentParser(description="Script to verify uploads to ES") + parser.add_argument( + '-s', '--server', + help='Provide elastic server information') + parser.add_argument( + '-u', '--uuid', + help='UUID to provide to search') + parser.add_argument( + '-i', '--index', + help='Index to provide to search') + parser.add_argument( + '--sslskipverify', + help='if es is setup with ssl, but can disable tls cert verification', + default=False) + args = parser.parse_args() + + sys.exit(_check_index(args.server, args.uuid, args.index, args.sslskipverify)) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/common.sh b/tests/common.sh new file mode 100755 index 000000000..302e3d144 --- /dev/null +++ b/tests/common.sh @@ -0,0 +1,297 @@ +#!/usr/bin/env bash + +ERRORED=false +image_location=${RIPSAW_CI_IMAGE_LOCATION:-quay.io} +image_account=${RIPSAW_CI_IMAGE_ACCOUNT:-rht_perf_ci} +es_url=${ES_SERVER:-http://foo.esserver.com:9200} +echo "using container image location $image_location and account $image_account" + +function populate_test_list { + rm -f tests/iterate_tests + + for item in $@ + do + # Check for changes in roles + if [[ $(echo ${item} | grep 'roles/fs-drift') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/uperf') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/fio_distributed') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/iperf3') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/byowl') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/sysbench') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/pgbench') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/ycsb') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/backpack') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/hammerdb') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/smallfile') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/vegeta') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/stressng') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/scale_openshift') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/kube-burner') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/flent') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/log_generator') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/image_pull') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi + + # Check for changes in cr files + if [[ $(echo ${item} | grep 'valid_backpack*') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_byowl*') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_fiod*') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_fs_drift*') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_hammerdb*') ]]; then echo "test_hammerdb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_iperf3*') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_pgbench*') ]]; then echo "test_pgbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_smallfile*') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_sysbench*') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_uperf*') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_ycsb*') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_vegeta*') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_stressng*') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_scale*') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_kube-burner*') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_flent*') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_log_generator*') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_image_pull*') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi + + # Check for changes in test scripts + test_check=`echo $item | awk -F / '{print $2}'` + + if [[ $(echo ${test_check} | grep 'test_.*.sh') ]]; then echo ${test_check} >> tests/iterate_tests; fi + done +} + +function delete_benchmark { + kubectl delete -f $1 --ignore-not-found=true +} + +function wait_clean { + if [[ `kubectl get benchmarks.ripsaw.cloudbulldozer.io --all-namespaces` ]] + then + echo "skipping cleanup" + fi +} + +# The argument is 'timeout in seconds' +function get_uuid () { + sleep_time=20 + benchmark_name=$1 + sleep $sleep_time + counter=0 + counter_max=6 + uuid="False" + until [ $uuid != "False" ] ; do + uuid=$(kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath='{.status.uuid}') + if [ -z $uuid ]; then + sleep $sleep_time + uuid="False" + fi + counter=$(( counter+1 )) + if [ $counter -eq $counter_max ]; then + return 1 + fi + done + echo $uuid + return 0 +} + + +# Two arguments are 'pod label' and 'timeout in seconds' +function get_pod () { + counter=0 + sleep_time=5 + counter_max=$(( $2 / sleep_time )) + pod_name="False" + until [ $pod_name != "False" ] ; do + sleep $sleep_time + pod_name=$(kubectl get pods -l $1 --namespace ${3:-benchmark-operator} -o name | cut -d/ -f2) + if [ -z $pod_name ]; then + pod_name="False" + fi + counter=$(( counter+1 )) + if [ $counter -eq $counter_max ]; then + return 1 + fi + done + echo $pod_name + return 0 +} + +# Three arguments are 'pod label', 'expected count', and 'timeout in seconds' +function pod_count () { + counter=0 + sleep_time=5 + counter_max=$(( $3 / sleep_time )) + pod_count=0 + export $1 + until [ $pod_count == $2 ] ; do + sleep $sleep_time + pod_count=$(kubectl get pods -n benchmark-operator -l $1 -o name | wc -l) + if [ -z $pod_count ]; then + pod_count=0 + fi + counter=$(( counter+1 )) + if [ $counter -eq $counter_max ]; then + return 1 + fi + done + echo $pod_count + return 0 +} + +function apply_operator { + tag_name="${NODE_NAME:-master}" + make image-build image-push deploy IMG=$image_location/$image_account/benchmark-operator:$tag_name + kubectl wait --for=condition=available "deployment/benchmark-controller-manager" -n benchmark-operator --timeout=300s +} + +function delete_operator { + make undeploy +} + + +function backpack_requirements { + kubectl apply -f resources/backpack_role.yaml + if [[ `command -v oc` ]] + then + if [[ `oc get securitycontextconstraints.security.openshift.io` ]] + then + oc adm policy -n benchmark-operator add-scc-to-user privileged -z benchmark-operator + oc adm policy -n benchmark-operator add-scc-to-user privileged -z backpack-view + fi + fi +} + + +function cleanup_operator_resources { + delete_operator + wait_clean +} + +function check_log(){ + for i in {1..10}; do + if kubectl logs -f $1 --namespace benchmark-operator | grep -q $2 ; then + break; + else + sleep 10 + fi + done +} + +function get_benchmark_name() { + benchmark_file=$1 + echo $(yq e '.metadata.name' $benchmark_file) +} + +function get_benchmark_state() { + benchmark_name=$1 + kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath={.status.state} +} + +function wait_for_benchmark() { + benchmark_name=$1 + desired_state=$2 + until [[ $(get_benchmark_state $benchmark_name) == "$desired_state" ]]; do + if [[ $(get_benchmark_state $benchmark_name) == "Failed" ]]; then + echo "Benchmark $benchmark_name failed" + return 1 + fi + sleep 5 + done +} + +function check_benchmark_for_desired_state(){ + benchmark_name=$1 + desired_state=$2 + timeout=${3:-500s} + export -f wait_for_benchmark + export -f get_benchmark_state + if ! timeout -k $timeout $timeout bash -c "wait_for_benchmark $benchmark_name $desired_state" + then + echo "Timeout exceeded for: "$benchmark_name + + counter=3 + until [ $counter -gt $# ] + do + echo "Logs from "${@:$counter} + kubectl -n benchmark-operator logs --tail=40 ${@:$counter} + counter=$(( counter+1 )) + done + return 1 + fi + echo $(kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath={.status.state}) + +} + +# Takes 2 or more arguments: 'command to run', 'time to wait until true' +# Any additional arguments will be passed to kubectl -n benchmark-operator logs to provide logging if a timeout occurs +function wait_for() { + if ! timeout -k $2 $2 $1 + then + echo "Timeout exceeded for: "$1 + + counter=3 + until [ $counter -gt $# ] + do + echo "Logs from "${@:$counter} + kubectl -n benchmark-operator logs --tail=40 ${@:$counter} + counter=$(( counter+1 )) + done + return 1 + fi + return 0 +} + +function error { + if [[ $ERRORED == "true" ]] + then + exit + fi + + ERRORED=true + + echo "Error caught. Dumping logs before exiting" + echo "Benchmark operator Logs" + kubectl -n benchmark-operator logs --tail=200 -l control-plane=controller-manager -c manager +} + + + +function wait_for_backpack() { + echo "Waiting for backpack to complete before starting benchmark test" + + uuid=$1 + count=0 + max_count=60 + while [[ $count -lt $max_count ]] + do + if [[ `kubectl -n benchmark-operator get daemonsets backpack-$uuid` ]] + then + desired=`kubectl -n benchmark-operator get daemonsets backpack-$uuid | grep -v NAME | awk '{print $2}'` + ready=`kubectl -n benchmark-operator get daemonsets backpack-$uuid | grep -v NAME | awk '{print $4}'` + if [[ $desired -eq $ready ]] + then + echo "Backpack complete. Starting benchmark" + break + fi + fi + count=$((count + 1)) + if [[ $count -ne $max_count ]] + then + sleep 6 + else + echo "Backpack failed to complete. Exiting" + exit 1 + fi + done +} + +function check_es() { + if [[ ${#} != 2 ]]; then + echo "Wrong number of arguments: ${#}" + return 1 + fi + local uuid=$1 + local index=${@:2} + for my_index in $index; do + python3 tests/check_es.py -s $es_url -u $uuid -i $my_index \ + || return 1 + done +} diff --git a/tests/full_test_file_trigger b/tests/full_test_file_trigger new file mode 100755 index 000000000..7859a425c --- /dev/null +++ b/tests/full_test_file_trigger @@ -0,0 +1,7 @@ +build/Dockerfile +group_vars/all.yml +resources/operator.yaml +tests/common.sh +playbook.yml +test.sh +watches.yaml diff --git a/tests/mssql.yaml b/tests/mssql.yaml new file mode 100755 index 000000000..1649d7fad --- /dev/null +++ b/tests/mssql.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: sql-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mssql-deployment + namespace: sql-server +spec: + replicas: 1 + selector: + matchLabels: + app: mssql + template: + metadata: + labels: + app: mssql + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mssql + image: quay.io/cloud-bulldozer/mssql:latest + ports: + - containerPort: 1433 + resources: + requests: + memory: "2048Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: mssql-deployment + namespace: sql-server +spec: + selector: + app: mssql + ports: + - protocol: TCP + port: 1433 + targetPort: 1433 diff --git a/tests/test_api_load.sh b/tests/test_api_load.sh new file mode 100755 index 000000000..2e0fceb89 --- /dev/null +++ b/tests/test_api_load.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up API load Test" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_api_load { + test_name=$1 + cr=$2 + delete_benchmark $cr + echo "Performing: ${test_name}" + kubectl apply -f ${cr} + benchmark_name=$(get_benchmark_name $cr) + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + check_benchmark_for_desired_state $benchmark_name Complete 500s + + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_api_load "API load" tests/test_crs/valid_api_load.yaml diff --git a/tests/test_backpack.sh b/tests/test_backpack.sh new file mode 100755 index 000000000..28c1d6b8e --- /dev/null +++ b/tests/test_backpack.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up backpack" + kubectl delete -f resources/backpack_role.yaml + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_backpack { + backpack_requirements + cr=$1 + delete_benchmark $cr + kubectl apply -f $cr + benchmark_name=$(get_benchmark_name $cr) + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + if [[ $1 == "tests/test_crs/valid_backpack_daemonset.yaml" ]] + then + wait_for_backpack $uuid + else + byowl_pod=$(get_pod "app=byowl-$uuid" 300) + wait_for "kubectl -n benchmark-operator wait --for=condition=Initialized pods/$byowl_pod --timeout=500s" "500s" $byowl_pod + wait_for "kubectl -n benchmark-operator wait --for=condition=complete -l app=byowl-$uuid jobs --timeout=500s" "500s" $byowl_pod + fi + + indexes="cpu_vulnerabilities-metadata cpuinfo-metadata dmidecode-metadata k8s_nodes-metadata lspci-metadata meminfo-metadata sysctl-metadata ocp_network_operator-metadata ocp_install_config-metadata ocp_kube_apiserver-metadata ocp_dns-metadata ocp_kube_controllermanager-metadata" + if check_es "${long_uuid}" "${indexes}" + then + echo "Backpack test: Success" + else + echo "Failed to find data in ES" + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_backpack tests/test_crs/valid_backpack_daemonset.yaml +functional_test_backpack tests/test_crs/valid_backpack_init.yaml diff --git a/tests/test_byowl.sh b/tests/test_byowl.sh new file mode 100755 index 000000000..b210c9f44 --- /dev/null +++ b/tests/test_byowl.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up byowl" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_byowl { + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + cr=tests/test_crs/valid_byowl.yaml + + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + check_benchmark_for_desired_state $benchmark_name Complete 500s + echo "BYOWL test: Success" + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_byowl diff --git a/tests/test_crs/configmap-cfg.yaml b/tests/test_crs/configmap-cfg.yaml new file mode 100644 index 000000000..9c2579c5d --- /dev/null +++ b/tests/test_crs/configmap-cfg.yaml @@ -0,0 +1,79 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-burner-config + namespace: benchmark-operator +data: + alerts.yml: "- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m]))[5m:]) + > 0.01\n description: 5 minutes avg. etcd fsync latency on {{$labels.pod}} higher + than 10ms {{$value}}\n severity: warning\n\n- expr: avg_over_time(histogram_quantile(0.99, + rate(etcd_network_peer_round_trip_time_seconds_bucket[5m]))[5m:]) > 0.1\n description: + 5 minutes avg. etcd network peer round trip on {{$labels.pod}} higher than 100ms + {{$value}}\n severity: warning \n" + config.yml: | + --- + global: + writeToFile: false + metricsDirectory: collected-metrics + indexerConfig: + enabled: true + esServers: [{{ .ES_SERVER }}] + insecureSkipVerify: true + defaultIndex: ripsaw-kube-burner + type: elastic + measurements: + - name: podLatency + esIndex: ripsaw-kube-burner + + jobs: + - name: basic-job + jobIterations: 5 + qps: 20 + burst: 20 + namespacedIterations: false + namespace: basic-job + waitWhenFinished: true + podWait: false + objects: + + - objectTemplate: pod.yml + replicas: 1 + inputVars: + containerImage: gcr.io/google_containers/pause-amd64:3.0 + metrics.yml: | + - query: count(kube_namespace_created) + metricName: namespaceCount + instant: true + + - query: sum(kube_pod_status_phase{}) by (phase) + metricName: podStatusCount + instant: true + + - query: count(kube_secret_info{}) + metricName: secretCount + instant: true + + - query: count(kube_deployment_labels{}) + metricName: deploymentCount + instant: true + + - query: count(kube_configmap_info{}) + metricName: configmapCount + instant: true + pod.yml: | + kind: Pod + apiVersion: v1 + metadata: + name: basic-pod-{{.Iteration}} + spec: + nodeSelector: + node-role.kubernetes.io/worker: "" + containers: + - name: basic-container + image: {{.containerImage}} + ports: + - containerPort: 8080 + protocol: TCP + imagePullPolicy: IfNotPresent + securityContext: + privileged: false diff --git a/tests/test_crs/configmap.yaml b/tests/test_crs/configmap.yaml new file mode 100644 index 000000000..e6ee8b971 --- /dev/null +++ b/tests/test_crs/configmap.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-configmap-cfg + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + prometheus: + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + extra_env_vars: + ES_SERVER: ${ES_SERVER} + configmap: kube-burner-config + cleanup: true + pin_server: {"node-role.kubernetes.io/worker": ""} + image: quay.io/cloud-bulldozer/kube-burner:latest + log_level: info + step: 30s + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/tests/test_crs/valid_api_load.yaml b/tests/test_crs/valid_api_load.yaml new file mode 100644 index 000000000..d49a497a1 --- /dev/null +++ b/tests/test_crs/valid_api_load.yaml @@ -0,0 +1,13 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: api-load + namespace: benchmark-operator +spec: + elasticsearch: + url: ES_SERVER + index_name: "api-load" + workload: + name: api_load + args: + override: version diff --git a/tests/test_crs/valid_backpack_daemonset.yaml b/tests/test_crs/valid_backpack_daemonset.yaml new file mode 100644 index 000000000..97fb74314 --- /dev/null +++ b/tests/test_crs/valid_backpack_daemonset.yaml @@ -0,0 +1,15 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: backpack + namespace: benchmark-operator +spec: + elasticsearch: + url: ES_SERVER + metadata: + collection: true + targeted: false + privileged: true + serviceaccount: backpack-view + workload: + name: backpack diff --git a/tests/test_crs/valid_backpack_init.yaml b/tests/test_crs/valid_backpack_init.yaml new file mode 100644 index 000000000..6886d3faa --- /dev/null +++ b/tests/test_crs/valid_backpack_init.yaml @@ -0,0 +1,18 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: backpack + namespace: benchmark-operator +spec: + elasticsearch: + url: ES_SERVER + metadata: + collection: true + privileged: true + serviceaccount: backpack-view + workload: + name: byowl + args: + image: "quay.io/jtaleric/uperf:testing" + clients: 1 + commands: "echo Test" diff --git a/tests/test_crs/valid_byowl.yaml b/tests/test_crs/valid_byowl.yaml new file mode 100644 index 000000000..621d08dc8 --- /dev/null +++ b/tests/test_crs/valid_byowl.yaml @@ -0,0 +1,22 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: byowl-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + metadata: + collection: true + workload: + name: byowl + args: + image: "quay.io/jtaleric/uperf:testing" + clients: 1 + commands: "echo Test" diff --git a/tests/test_crs/valid_fiod.yaml b/tests/test_crs/valid_fiod.yaml new file mode 100644 index 000000000..9df7dfe1c --- /dev/null +++ b/tests/test_crs/valid_fiod.yaml @@ -0,0 +1,85 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fiod-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-fio + metadata: + collection: true + cleanup: false + workload: + name: "fio_distributed" + args: + samples: 2 + servers: 2 + jobs: + - write + - read + - randwrite + - randread + - randrw + bs: + - 4KiB + numjobs: + - 2 + iodepth: 1 + read_runtime: 10 + write_runtime: 10 + read_ramp_time: 1 + write_ramp_time: 1 + filesize: 10MiB + log_sample_rate: 2000 + storagesize: 16Mi + debug: true +####################################### +# EXPERT AREA - MODIFY WITH CAUTION # +####################################### + job_params: + - jobname_match: write + params: + - time_based=1 + - fsync_on_close=1 + - create_on_open=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: read + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: rw + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: readwrite + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randread + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randwrite + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: randrw + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} diff --git a/tests/test_crs/valid_fiod_bsrange.yaml b/tests/test_crs/valid_fiod_bsrange.yaml new file mode 100644 index 000000000..2c5cdbb26 --- /dev/null +++ b/tests/test_crs/valid_fiod_bsrange.yaml @@ -0,0 +1,86 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fiod-bsrange-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-fio + metadata: + collection: true + cleanup: false + workload: + name: "fio_distributed" + args: + samples: 2 + servers: 2 + jobs: + - write + - read + - randwrite + - randread + - randrw + bsrange: + - 4KiB-16KiB + numjobs: + - 2 + iodepth: 1 + read_runtime: 10 + write_runtime: 10 + read_ramp_time: 1 + write_ramp_time: 1 + filesize: 10MiB + log_sample_rate: 2000 + log_hist_msec: 3000 + storagesize: 16Mi + debug: true +####################################### +# EXPERT AREA - MODIFY WITH CAUTION # +####################################### + job_params: + - jobname_match: write + params: + - time_based=1 + - fsync_on_close=1 + - create_on_open=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: read + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: rw + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: readwrite + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randread + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randwrite + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: randrw + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} diff --git a/tests/test_crs/valid_fiod_hostpath.yaml b/tests/test_crs/valid_fiod_hostpath.yaml new file mode 100644 index 000000000..cc56c4cbd --- /dev/null +++ b/tests/test_crs/valid_fiod_hostpath.yaml @@ -0,0 +1,85 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fiod-hostpath-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-fio + metadata: + collection: true + cleanup: false + hostpath: /mnt/vda1/workload_args/ + workload: + name: "fio_distributed" + args: + drop_cache_kernel: true + samples: 1 + servers: 2 + prefill: true + jobs: + - write + - randread + - randrw + bs: + - 4KiB + numjobs: + - 2 + iodepth: 2 + read_runtime: 6 + write_runtime: 6 + read_ramp_time: 1 + write_ramp_time: 1 + filesize: 10MiB + debug: false +####################################### +# EXPERT AREA - MODIFY WITH CAUTION # +####################################### + job_params: + - jobname_match: write + params: + - time_based=1 + - fsync_on_close=1 + - create_on_open=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: read + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: rw + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: readwrite + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randread + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randwrite + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: randrw + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + diff --git a/tests/test_crs/valid_fiod_ocs_cache_drop.yaml b/tests/test_crs/valid_fiod_ocs_cache_drop.yaml new file mode 100644 index 000000000..2ebba6a0e --- /dev/null +++ b/tests/test_crs/valid_fiod_ocs_cache_drop.yaml @@ -0,0 +1,85 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fiod-ocs-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-fio + metadata: + collection: true + cleanup: false + workload: + name: "fio_distributed" + args: + drop_cache_rook_ceph: true + drop_cache_kernel: true + samples: 2 + servers: 2 + jobs: + - randwrite + - randread + - randrw + bs: + - 4KiB + numjobs: + - 2 + iodepth: 1 + read_runtime: 10 + write_runtime: 10 + read_ramp_time: 1 + write_ramp_time: 1 + filesize: 10MiB + log_sample_rate: 2000 + storagesize: 16Mi + debug: false +####################################### +# EXPERT AREA - MODIFY WITH CAUTION # +####################################### + job_params: + - jobname_match: write + params: + - time_based=1 + - fsync_on_close=1 + - create_on_open=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: read + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: rw + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: readwrite + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randread + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randwrite + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: randrw + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} diff --git a/tests/test_crs/valid_flent.yaml b/tests/test_crs/valid_flent.yaml new file mode 100644 index 000000000..4d17d4ce4 --- /dev/null +++ b/tests/test_crs/valid_flent.yaml @@ -0,0 +1,31 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: flent-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-flent + metadata: + collection: true + clustername: myk8scluster + workload: + # cleanup: true + name: flent + args: + hostnetwork: false + multus: + enabled: false + pin: false + pair: 1 + test_types: + - tcp_download + runtime: 2 + debug: true diff --git a/tests/test_crs/valid_flent_resources.yaml b/tests/test_crs/valid_flent_resources.yaml new file mode 100644 index 000000000..bef6ba7ab --- /dev/null +++ b/tests/test_crs/valid_flent_resources.yaml @@ -0,0 +1,39 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: flent-resources-benchmark + namespace: benchmark-operator +spec: + clustername: myk8scluster + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-flent + workload: + # cleanup: true + name: flent + args: + client_resources: + requests: + cpu: 100m + memory: 100Mi + server_resources: + requests: + cpu: 100m + memory: 100Mi + pin: false + pin_server: "node-0" + pin_client: "node-1" + hostnetwork: false + multus: + enabled: false + pair: 1 + test_types: + - tcp_download + runtime: 2 + debug: true diff --git a/tests/test_crs/valid_fs_drift.yaml b/tests/test_crs/valid_fs_drift.yaml new file mode 100644 index 000000000..9ccd180ce --- /dev/null +++ b/tests/test_crs/valid_fs_drift.yaml @@ -0,0 +1,30 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fs-drift-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + # where elastic search is running + elasticsearch: + url: ES_SERVER + index_name: ripsaw-fs-drift + metadata: + collection: true + workload: + name: fs-drift + args: + worker_pods: 1 + threads: 5 + max_file_size_kb: 4 + max_files: 1000 + duration: 15 + debug: true diff --git a/tests/test_crs/valid_fs_drift_hostpath.yaml b/tests/test_crs/valid_fs_drift_hostpath.yaml new file mode 100644 index 000000000..91c8f65bb --- /dev/null +++ b/tests/test_crs/valid_fs_drift_hostpath.yaml @@ -0,0 +1,31 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fs-drift-hostpath-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + # where elastic search is running + elasticsearch: + url: ES_SERVER + index_name: ripsaw-fs-drift + metadata: + collection: true + hostpath: /mnt/vda1/fs_drift + workload: + name: fs-drift + args: + worker_pods: 1 + threads: 5 + max_file_size_kb: 4 + max_files: 1000 + duration: 15 + debug: true diff --git a/tests/test_crs/valid_hammerdb.yaml b/tests/test_crs/valid_hammerdb.yaml new file mode 100644 index 000000000..6c3d1b7f3 --- /dev/null +++ b/tests/test_crs/valid_hammerdb.yaml @@ -0,0 +1,70 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-hammerdb + metadata: + collection: true + workload: + name: "hammerdb" + args: + # image: "quay.io/test/hammerdb:latest" + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + db_init: false + db_benchmark: true + db_server: "mssql-deployment.sql-server" + db_port: "1433" + db_warehouses: 1 + db_num_workers: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 10000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mysql: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + debug: true diff --git a/tests/test_crs/valid_image_pull.yaml b/tests/test_crs/valid_image_pull.yaml new file mode 100644 index 000000000..c39cf5e63 --- /dev/null +++ b/tests/test_crs/valid_image_pull.yaml @@ -0,0 +1,18 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: image-pull + namespace: benchmark-operator +spec: + elasticsearch: + url: ES_SERVER + index_name: "image-pull" + workload: + name: image_pull + args: + pod_count: 2 + timeout: 100 + retries: 1 + image_list: + - docker://quay.io/cloud-bulldozer/backpack + - docker://quay.io/cloud-bulldozer/fio diff --git a/tests/test_crs/valid_iperf3.yaml b/tests/test_crs/valid_iperf3.yaml new file mode 100644 index 000000000..d0ebe5b91 --- /dev/null +++ b/tests/test_crs/valid_iperf3.yaml @@ -0,0 +1,32 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: iperf3-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + metadata: + collection: true + workload: + name: iperf3 + args: + pairs: 1 + hostnetwork: false + port: 5201 + transmit_type: time + transmit_value: 60 + omit_start: 0 + length_buffer: 128K + window_size: 64k + ip_tos: 0 + mss: 900 + streams: 1 + extra_options_server: '--forceflush ' + extra_options_client: '-R ' diff --git a/tests/test_crs/valid_kube-burner.yaml b/tests/test_crs/valid_kube-burner.yaml new file mode 100644 index 000000000..476067b96 --- /dev/null +++ b/tests/test_crs/valid_kube-burner.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: ES_SERVER + metadata: + collection: false + prometheus: + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + workload: WORKLOAD + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 1 + job_timeout: 300 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: METRICS_PROFILE + app: django + source_strat_env: PIP_INDEX_URL + source_strat_from: python + source_strat_from_version: latest + post_commit_script: "./manage.py test" + build_image_stream: django-psql-example + git_url: https://github.com/sclorg/django-ex.git + build_image: image-registry.openshift-image-registry.svc:5000/svt-django/django-psql-example + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/tests/test_crs/valid_log_generator.yaml b/tests/test_crs/valid_log_generator.yaml new file mode 100644 index 000000000..72b926db2 --- /dev/null +++ b/tests/test_crs/valid_log_generator.yaml @@ -0,0 +1,18 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: benchmark-operator +spec: + elasticsearch: + url: ES_SERVER + index_name: log-generator + workload: + name: log_generator + args: + pod_count: 1 + size: 512 + messages_per_second: 1 + duration: 1 + es_url: ES_SERVER + timeout: 600 diff --git a/tests/test_crs/valid_pgbench.yaml b/tests/test_crs/valid_pgbench.yaml new file mode 100644 index 000000000..908a4e58b --- /dev/null +++ b/tests/test_crs/valid_pgbench.yaml @@ -0,0 +1,37 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: pgbench-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-pgbench + metadata: + collection: true + workload: + name: "pgbench" + args: + timeout: 5 + clients: + - 1 + - 2 + threads: 1 + transactions: 10 + cmd_flags: '' + init_cmd_flags: '' + scaling_factor: 1 + samples: 2 + num_databases_pattern: 'all' + databases: + - host: + user: ci + password: ci + db_name: cidb + debug: true diff --git a/tests/test_crs/valid_scale_down.yaml b/tests/test_crs/valid_scale_down.yaml new file mode 100644 index 000000000..4b1d03b04 --- /dev/null +++ b/tests/test_crs/valid_scale_down.yaml @@ -0,0 +1,24 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: scale-down + namespace: benchmark-operator +spec: + metadata: + collection: true + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: openshift-cluster-timings + workload: + name: scale_openshift + args: + scale: 0 + serviceaccount: scaler + poll_interval: 2 + debug: true diff --git a/tests/test_crs/valid_scale_up.yaml b/tests/test_crs/valid_scale_up.yaml new file mode 100644 index 000000000..e1f1aceff --- /dev/null +++ b/tests/test_crs/valid_scale_up.yaml @@ -0,0 +1,24 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: scale-up + namespace: benchmark-operator +spec: + metadata: + collection: true + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: openshift-cluster-timings + workload: + name: scale_openshift + args: + scale: 1 + serviceaccount: scaler + poll_interval: 2 + debug: true diff --git a/tests/test_crs/valid_servicemesh.yaml b/tests/test_crs/valid_servicemesh.yaml new file mode 100644 index 000000000..c41fab396 --- /dev/null +++ b/tests/test_crs/valid_servicemesh.yaml @@ -0,0 +1,16 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: servicemesh-benchmark + namespace: benchmark-operator +spec: + workload: + name: servicemesh + args: + deployments: 2 + pods_per_deployment: 1 + workload_cpu: 1 + hyperfoil_threads: 1 + test_name: closedmodel + closedmodel: + steadystate_duration: 10s diff --git a/tests/test_crs/valid_smallfile.yaml b/tests/test_crs/valid_smallfile.yaml new file mode 100644 index 000000000..f37b85731 --- /dev/null +++ b/tests/test_crs/valid_smallfile.yaml @@ -0,0 +1,46 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: smallfile-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + # where elastic search is running + elasticsearch: + url: ES_SERVER + index_name: ripsaw-smallfile + metadata: + collection: true + workload: + name: smallfile + args: + clients: 2 + operation: ["create", "read", "delete", "cleanup"] + threads: 1 + file_size: 2 + record_size: 1 + xattr_size: 50 + pause: 0 + auto_pause: y + stonewall: n + finish: y + prefix: abc + hash_into_dirs: true + same_dir: y + incompressible: y + verify_read: y + xattr_count: 10 + fsync: false + files_per_dir: 2000 + dirs_per_dir: 4 + files: 100000 + cleanup_delay_usec_per_file: 20 + debug: true diff --git a/tests/test_crs/valid_smallfile_hostpath.yaml b/tests/test_crs/valid_smallfile_hostpath.yaml new file mode 100644 index 000000000..daf1c273b --- /dev/null +++ b/tests/test_crs/valid_smallfile_hostpath.yaml @@ -0,0 +1,31 @@ + +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: smallfile-hostpath-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-smallfile + metadata: + collection: true + hostpath: /mnt/vda1/smallfile + workload: + name: smallfile + args: + clients: 1 + operation: ["create", "read", "append", "delete"] + threads: 1 + file_size: 0 + files: 100000 + debug: true diff --git a/tests/test_crs/valid_stressng.yaml b/tests/test_crs/valid_stressng.yaml new file mode 100644 index 000000000..220004b18 --- /dev/null +++ b/tests/test_crs/valid_stressng.yaml @@ -0,0 +1,34 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: stressng + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-stressng + metadata: + collection: true + workload: + name: "stressng" + args: + # general options + runtype: "parallel" + timeout: "30" + instances: 1 + # nodeselector: + # cpu stressor options + cpu_stressors: "1" + cpu_percentage: "100" + # vm stressor option + vm_stressors: "1" + vm_bytes: "128M" + # mem stressor options + mem_stressors: "1" + debug: true diff --git a/tests/test_crs/valid_sysbench.yaml b/tests/test_crs/valid_sysbench.yaml new file mode 100644 index 000000000..9bd1aca1d --- /dev/null +++ b/tests/test_crs/valid_sysbench.yaml @@ -0,0 +1,23 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: sysbench-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + metadata: + collection: true + workload: + name: sysbench + args: + tests: + - name: cpu + parameters: + cpu-max-prime: 200 diff --git a/tests/test_crs/valid_uperf.yaml b/tests/test_crs/valid_uperf.yaml new file mode 100644 index 000000000..a0ff00c5e --- /dev/null +++ b/tests/test_crs/valid_uperf.yaml @@ -0,0 +1,45 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: false + pin: false + pin_server: "node-0" + pin_client: "node-1" + multus: + enabled: false + samples: 2 + pair: 1 + test_types: + - stream + - rr + - bidirec + protos: + - tcp + - udp + sizes: + - 1024 + - [1024, 512] + nthrs: + - 1 + - 2 + runtime: 2 + debug: true diff --git a/tests/test_crs/valid_uperf_networkpolicy.yaml b/tests/test_crs/valid_uperf_networkpolicy.yaml new file mode 100644 index 000000000..54cb87e1f --- /dev/null +++ b/tests/test_crs/valid_uperf_networkpolicy.yaml @@ -0,0 +1,46 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-networkpolicy + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: false + networkpolicy: true + pin: false + pin_server: "node-0" + pin_client: "node-1" + multus: + enabled: false + samples: 2 + pair: 1 + test_types: + - stream + - rr + - bidirec + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + - 2 + runtime: 2 + debug: true diff --git a/tests/test_crs/valid_uperf_resources.yaml b/tests/test_crs/valid_uperf_resources.yaml new file mode 100644 index 000000000..da3f60392 --- /dev/null +++ b/tests/test_crs/valid_uperf_resources.yaml @@ -0,0 +1,52 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-resources + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf + args: + client_resources: + requests: + cpu: 100m + memory: 100Mi + server_resources: + requests: + cpu: 100m + memory: 100Mi + hostnetwork: true + serviceip: false + pin: false + pin_server: "node-0" + pin_client: "node-1" + multus: + enabled: false + samples: 2 + pair: 1 + test_types: + - stream + - rr + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + - 2 + runtime: 2 + debug: true diff --git a/tests/test_crs/valid_uperf_serviceip.yaml b/tests/test_crs/valid_uperf_serviceip.yaml new file mode 100644 index 000000000..9837bbe7c --- /dev/null +++ b/tests/test_crs/valid_uperf_serviceip.yaml @@ -0,0 +1,44 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-serviceip + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: true + pin: false + pin_server: "node-0" + pin_client: "node-1" + multus: + enabled: false + samples: 2 + pair: 1 + test_types: + - stream + - rr + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + - 2 + runtime: 2 + debug: true diff --git a/tests/test_crs/valid_vegeta.yaml b/tests/test_crs/valid_vegeta.yaml new file mode 100644 index 000000000..4d9e79f5e --- /dev/null +++ b/tests/test_crs/valid_vegeta.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: vegeta-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-vegeta + metadata: + collection: true + workload: + name: vegeta + args: + clients: 3 + targets: + - name: 2w-ka + urls: + - GET https://1.1.1.1 + - GET http://1.1.1.1 + samples: 3 + workers: 2 + duration: 5 + keepalive: true + - name: 2w-noka + urls: + - GET https://1.1.1.1 + workers: 2 + duration: 5 + debug: true diff --git a/tests/test_crs/valid_vegeta_hostnetwork.yaml b/tests/test_crs/valid_vegeta_hostnetwork.yaml new file mode 100644 index 000000000..6d3f1ecb8 --- /dev/null +++ b/tests/test_crs/valid_vegeta_hostnetwork.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: vegeta-benchmark-hostnetwork + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-vegeta + metadata: + collection: true + serviceaccount: benchmark-operator + workload: + name: vegeta + args: + hostnetwork: true + clients: 2 + targets: + - name: 2w-ka + urls: + - GET https://1.1.1.1 + samples: 1 + workers: 2 + duration: 5 + keepalive: true + debug: true diff --git a/tests/test_crs/valid_ycsb-mongo.yaml b/tests/test_crs/valid_ycsb-mongo.yaml new file mode 100644 index 000000000..3e8a56cda --- /dev/null +++ b/tests/test_crs/valid_ycsb-mongo.yaml @@ -0,0 +1,30 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: ycsb-mongo-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-ycsb + metadata: + collection: true + workload: + name: ycsb + args: + infra: mongodb + driver: mongodb + recordcount: 100 + operationcount: 100 + requestdistribution: zipfian + workloads: + - workloada + options_load: '-p mongodb.url="mongodb://mongo-0.mongo/ycsb?"' #passed as is to ycsb when loading database + options_run: '-p mongodb.url="mongodb://mongo-0.mongo/ycsb?" -threads 10 -target 100' + debug: true diff --git a/tests/test_fiod.sh b/tests/test_fiod.sh new file mode 100755 index 000000000..78fb7d8af --- /dev/null +++ b/tests/test_fiod.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up fio" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_fio { + wait_clean + test_name=$1 + cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr + echo "Performing: ${test_name}" + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + pod_count "app=fio-benchmark-$uuid" 2 300 + fio_pod=$(get_pod "app=fiod-client-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 1200s + kubectl -n benchmark-operator logs $fio_pod > /tmp/$fio_pod.log + + + + + indexes="ripsaw-fio-results ripsaw-fio-log ripsaw-fio-analyzed-result" + if check_es "${long_uuid}" "${indexes}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + kubectl logs "$fio_pod" -n benchmark-operator + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +kubectl label nodes -l node-role.kubernetes.io/worker= kernel-cache-dropper=yes --overwrite +functional_test_fio "Fio distributed" tests/test_crs/valid_fiod.yaml +openshift_storage_present=$(oc get namespace | awk '/openshift-storage/' | wc -l) +if [ $openshift_storage_present -gt 0 ] ; then + oc patch OCSInitialization ocsinit -n openshift-storage --type json --patch \ + '[{ "op": "replace", "path": "/spec/enableCephTools", "value": true }]' + drop_cache_pods=$(oc -n openshift-storage get pod | awk '/drop/' | awk '/unning/' | wc -l) + if [ $drop_cache_pods -eq 0 ] ; then + oc create -f roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml + kubectl wait --for=condition=Initialized pods/rook-ceph-osd-cache-drop -n openshift-storage --timeout=100s + fi + sleep 5 + functional_test_fio "Fio cache drop" tests/test_crs/valid_fiod_ocs_cache_drop.yaml +fi +functional_test_fio "Fio distributed - bsrange" tests/test_crs/valid_fiod_bsrange.yaml +functional_test_fio "Fio hostpath distributed" tests/test_crs/valid_fiod_hostpath.yaml diff --git a/tests/test_flent.sh b/tests/test_flent.sh new file mode 100755 index 000000000..84d059cfe --- /dev/null +++ b/tests/test_flent.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up Flent" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_flent { + test_name=$1 + cr=$2 + delete_benchmark $cr + echo "Performing: ${test_name}" + benchmark_name=$(get_benchmark_name $cr) + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + check_benchmark_for_desired_state $benchmark_name Complete 800s + + index="ripsaw-flent-results" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + kubectl logs "$flent_client_pod" -n benchmark-operator + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_flent "Flent without resources definition" tests/test_crs/valid_flent.yaml +functional_test_flent "Flent with resources definition" tests/test_crs/valid_flent_resources.yaml diff --git a/tests/test_fs_drift.sh b/tests/test_fs_drift.sh new file mode 100755 index 000000000..ac7d26d17 --- /dev/null +++ b/tests/test_fs_drift.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -xeo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up fs_drift" + wait_clean +} + + +trap error ERR +trap finish EXIT + +function functional_test_fs_drift { + test_name=$1 + cr=$2 + delete_benchmark $cr + echo "Performing: ${test_name}" + benchmark_name=$(get_benchmark_name $cr) + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + fsdrift_pod=$(get_pod "app=fs-drift-benchmark-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 600s + indexes="ripsaw-fs-drift-results ripsaw-fs-drift-rsptimes ripsaw-fs-drift-rates-over-time" + if check_es "${long_uuid}" "${indexes}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + kubectl logs "$fsdrift_pod" -n benchmark-operator + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_fs_drift "fs-drift" tests/test_crs/valid_fs_drift.yaml +functional_test_fs_drift "fs-drift hostpath" tests/test_crs/valid_fs_drift_hostpath.yaml diff --git a/tests/test_hammerdb.sh b/tests/test_hammerdb.sh new file mode 100755 index 000000000..78c7aa145 --- /dev/null +++ b/tests/test_hammerdb.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -xeo pipefail + +source tests/common.sh + +function initdb_pod { + echo "Setting up a MS-SQL DB Pod" + kubectl apply -f tests/mssql.yaml + mssql_pod=$(get_pod "app=mssql" 300 "sql-server") + kubectl wait --for=condition=Ready "pods/$mssql_pod" --namespace sql-server --timeout=300s +} + +function finish { + echo "Cleaning up hammerdb" + kubectl delete -f tests/mssql.yaml + wait_clean +} + +trap finish EXIT + +function functional_test_hammerdb { + initdb_pod + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + cr=tests/test_crs/valid_hammerdb.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + # Wait for the workload pod to run the actual workload + hammerdb_workload_pod=$(get_pod "app=hammerdb_workload-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 900s + + index="ripsaw-hammerdb-results" + if check_es "${long_uuid}" "${index}" + then + echo "Hammerdb test: Success" + else + echo "Failed to find data for HammerDB test in ES" + kubectl logs "$hammerdb_workload_pod" --namespace benchmark-operator + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_hammerdb diff --git a/tests/test_image_pull.sh b/tests/test_image_pull.sh new file mode 100755 index 000000000..07a47309b --- /dev/null +++ b/tests/test_image_pull.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up Image Pull Test" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_image_pull { + test_name=$1 + cr=$2 + delete_benchmark $cr + echo "Performing: ${test_name}" + kubectl apply -f ${cr} + benchmark_name=$(get_benchmark_name $cr) + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + check_benchmark_for_desired_state $benchmark_name Complete 500s + + index="image-pull-results" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_image_pull "Image Pull" tests/test_crs/valid_image_pull.yaml diff --git a/tests/test_iperf3.sh b/tests/test_iperf3.sh new file mode 100755 index 000000000..18efecbe0 --- /dev/null +++ b/tests/test_iperf3.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up iperf3" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_iperf { + echo "Performing iperf3: ${1}" + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + cr=tests/test_crs/valid_iperf3.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" -e "s/hostnetwork:.*/${1}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + iperf_server_pod=$(get_pod "app=iperf3-bench-server-$uuid" 300) + wait_for "kubectl -n benchmark-operator wait --for=condition=Initialized -l app=iperf3-bench-server-$uuid pods --timeout=300s" "300s" $iperf_server_pod + iperf_client_pod=$(get_pod "app=iperf3-bench-client-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 600s + sleep 5 + # ensuring that iperf actually ran and we can access metrics + kubectl logs "$iperf_client_pod" --namespace benchmark-operator | grep "iperf Done." + echo "iperf ${1}: Success" + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_iperf "hostnetwork: false" +functional_test_iperf "hostnetwork: true" diff --git a/tests/test_kubeburner.sh b/tests/test_kubeburner.sh new file mode 100755 index 000000000..88f57883f --- /dev/null +++ b/tests/test_kubeburner.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + [[ $check_logs == 1 ]] && kubectl logs -l app=kube-burner-benchmark-$uuid -n benchmark-operator + echo "Cleaning up kube-burner" + kubectl delete -f resources/kube-burner-role.yml --ignore-not-found + kubectl delete ns -l kube-burner-uuid=${long_uuid} + wait_clean +} + + +trap error ERR +trap finish EXIT + +function functional_test_kubeburner { + workload_name=$1 + metrics_profile=$2 + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + cr=tests/test_crs/valid_kube-burner.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + check_logs=0 + kubectl apply -f resources/kube-burner-role.yml + echo "Performing kube-burner: ${workload_name}" + sed -e "s/WORKLOAD/${workload_name}/g" -e "s/PROMETHEUS_TOKEN/${token}/g" -e "s/METRICS_PROFILE/${metrics_profile}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + check_logs=1 + check_benchmark_for_desired_state $benchmark_name Complete 1800s + + index="ripsaw-kube-burner" + if check_es "${long_uuid}" "${index}" + then + echo "kube-burner ${workload_name}: Success" + else + echo "Failed to find data for kube-burner ${workload_name} in ES" + exit 1 + fi + kubectl delete ns -l kube-burner-uuid=${long_uuid} + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_kubeburner cluster-density metrics-aggregated.yaml +functional_test_kubeburner node-density metrics.yaml +functional_test_kubeburner node-density-heavy metrics.yaml +functional_test_kubeburner max-namespaces metrics-aggregated.yaml +functional_test_kubeburner max-services metrics-aggregated.yaml +functional_test_kubeburner concurrent-builds metrics-aggregated.yaml \ No newline at end of file diff --git a/tests/test_list b/tests/test_list new file mode 100755 index 000000000..0e0208c32 --- /dev/null +++ b/tests/test_list @@ -0,0 +1,17 @@ +test_backpack.sh +test_ycsb.sh +test_fiod.sh +test_fs-drift.sh +test_smallfile.sh +test_pgbench.sh +test_iperf3.sh +test_sysbench.sh +test_uperf.sh +test_byowl.sh +test_hammerdb.sh +test_vegeta.sh +test_scale_openshift.sh +test_stressng.sh +test_kubeburner.sh +test_flent.sh +test_log_generator.sh diff --git a/tests/test_log_generator.sh b/tests/test_log_generator.sh new file mode 100755 index 000000000..3d394f981 --- /dev/null +++ b/tests/test_log_generator.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up Log Generator Test" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_log_generator { + test_name=$1 + cr=$2 + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + echo "Performing: ${test_name}" + kubectl apply -f ${cr} + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + log_gen_pod=$(get_pod "app=log-generator-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 800s + + index="log-generator-results" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + kubectl logs "$log_gen_pod" -n benchmark-operator + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_log_generator "Log Generator" tests/test_crs/valid_log_generator.yaml diff --git a/tests/test_pgbench.sh b/tests/test_pgbench.sh new file mode 100755 index 000000000..d8d33a249 --- /dev/null +++ b/tests/test_pgbench.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up pgbench" + wait_clean +} + +trap error ERR +trap finish EXIT + +# Note we don't test persistent storage here +function functional_test_pgbench { + # stand up postgres deployment +cat << EOF | kubectl apply -f - +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-config + namespace: benchmark-operator + labels: + app: postgres +data: + POSTGRES_DB: cidb + POSTGRES_USER: ci + POSTGRES_PASSWORD: ci + PGDATA: /var/lib/postgresql/data/pgdata +EOF +cat << EOF | kubectl apply -f - +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: benchmark-operator +spec: + selector: + matchLabels: + app: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:10.4 + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5432 + envFrom: + - configMapRef: + name: postgres-config +EOF + postgres_pod=$(get_pod 'app=postgres' 300) + # get the postgres pod IP + postgres_ip=0 + counter=0 + until [[ $postgres_ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ || $counter -eq 10 ]]; do + let counter+=1 + postgres_ip=$(kubectl get pod -n benchmark-operator $postgres_pod --template={{.status.podIP}}) + sleep 2 + done + # deploy the test CR with the postgres pod IP + cr=tests/test_crs/valid_pgbench.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed s/host:/host:\ ${postgres_ip}/ $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + pgbench_pod=$(get_pod "app=pgbench-client-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 500s + + index="ripsaw-pgbench-summary ripsaw-pgbench-raw" + if check_es "${long_uuid}" "${index}" + then + echo "pgbench test: Success" + else + echo "Failed to find data for PGBench in ES" + kubectl logs -n benchmark-operator $pgbench_pod + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_pgbench diff --git a/tests/test_scale_openshift.sh b/tests/test_scale_openshift.sh new file mode 100755 index 000000000..42b38ab79 --- /dev/null +++ b/tests/test_scale_openshift.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up Scale Test" + kubectl delete -f resources/scale_role.yaml --ignore-not-found + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_scale_openshift { + test_name=$1 + cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr + # Apply scale role and service account + kubectl apply -f resources/scale_role.yaml + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + echo "Performing: ${test_name}" + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + scale_pod=$(get_pod "app=scale-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 500s + index="openshift-cluster-timings" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + kubectl logs "$scale_pod" -n benchmark-operator + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_scale_openshift "Scale Up" tests/test_crs/valid_scale_up.yaml +functional_test_scale_openshift "Scale Down" tests/test_crs/valid_scale_down.yaml diff --git a/tests/test_servicemesh.sh b/tests/test_servicemesh.sh new file mode 100755 index 000000000..6f5bf18d0 --- /dev/null +++ b/tests/test_servicemesh.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up ServiceMesh" + long_uuid=$(get_uuid 0) + uuid=${long_uuid:0:8} + kubectl delete --ignore-not-found=true namespace example-benchmark-controlplane-$uuid + kubectl delete --ignore-not-found=true namespace example-benchmark-workload-$uuid + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_servicemesh { + cr=tests/test_crs/valid_servicemesh.yaml + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr + kubectl apply -f $cr + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + # wait until the job appears + count=0 + max_count=60 + while [[ $count -lt $max_count ]] + do + if kubectl get --namespace benchmark-operator job example-benchmark-$uuid; then + break + fi + sleep 15 + count=$((count + 1)) + done + + wait_for "kubectl -n benchmark-operator wait --for=condition=complete jobs --timeout=300s example-benchmark-$uuid" "300s" + + job_pod=$(get_pod app=example-benchmark-$uuid 30) + # ensuring that uperf actually ran and we can access metrics + INFO=$(kubectl logs $job_pod --namespace benchmark-operator | jq .info) + if [ -n "$INFO" ]; then + echo "Successful: $INFO" + else + echo "Failed to verify benchmark results" + exit 1 + fi + delete_benchmark $cr +} + +functional_test_servicemesh diff --git a/tests/test_smallfile.sh b/tests/test_smallfile.sh new file mode 100755 index 000000000..b921adfb2 --- /dev/null +++ b/tests/test_smallfile.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up smallfile" + wait_clean +} + + +trap error ERR +trap finish EXIT + +function functional_test_smallfile { + test_name=$1 + cr=$2 + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + echo "Performing: ${test_name}" + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + check_benchmark_for_desired_state $benchmark_name Complete 500s + + indexes="ripsaw-smallfile-results ripsaw-smallfile-rsptimes" + if check_es "${long_uuid}" "${indexes}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_smallfile "smallfile" tests/test_crs/valid_smallfile.yaml +functional_test_smallfile "smallfile hostpath" tests/test_crs/valid_smallfile_hostpath.yaml diff --git a/tests/test_stressng.sh b/tests/test_stressng.sh new file mode 100755 index 000000000..d9c98c05c --- /dev/null +++ b/tests/test_stressng.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -xeo pipefail + +source tests/common.sh + + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + echo "Cleaning up stressng" + wait_clean +} + +trap finish EXIT + +function functional_test_stressng { + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + cr=tests/test_crs/valid_stressng.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + # Wait for the workload pod to run the actual workload + stressng_workload_pod=$(get_pod "app=stressng_workload-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 500s + + + index="ripsaw-stressng-results" + if check_es "${long_uuid}" "${index}" + then + echo "StressNG test: Success" + else + echo "Failed to find data for StressNG test in ES" + kubectl logs "$stressng_workload_pod" --namespace benchmark-operator + exit 1 + fi + + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_stressng diff --git a/tests/test_sysbench.sh b/tests/test_sysbench.sh new file mode 100755 index 000000000..78571fe2c --- /dev/null +++ b/tests/test_sysbench.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up sysbench" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_sysbench { + cr=tests/test_crs/valid_sysbench.yaml + delete_benchmark $cr + kubectl apply -f $cr + benchmark_name=$(get_benchmark_name $cr) + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + sysbench_pod=$(get_pod "app=sysbench-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 800s + # sleep isn't needed as the sysbench is kind: job so once it's complete we can access logs + # ensuring the run has actually happened + kubectl logs "$sysbench_pod" --namespace benchmark-operator | grep "execution time" + echo "Sysbench test: Success" + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_sysbench diff --git a/tests/test_uperf.sh b/tests/test_uperf.sh new file mode 100755 index 000000000..3621268fe --- /dev/null +++ b/tests/test_uperf.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up Uperf" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_uperf { + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + test_name=$1 + cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr + echo "Performing: ${test_name}" + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + check_benchmark_for_desired_state $benchmark_name Complete 1800s + + + + index="ripsaw-uperf-results" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_uperf "Uperf without resources definition" tests/test_crs/valid_uperf.yaml +functional_test_uperf "Uperf with ServiceIP" tests/test_crs/valid_uperf_serviceip.yaml +functional_test_uperf "Uperf with resources definition and hostNetwork" tests/test_crs/valid_uperf_resources.yaml +functional_test_uperf "Uperf with networkpolicy" tests/test_crs/valid_uperf_networkpolicy.yaml diff --git a/tests/test_vegeta.sh b/tests/test_vegeta.sh new file mode 100755 index 000000000..ca3d19a19 --- /dev/null +++ b/tests/test_vegeta.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + [[ $check_logs == 1 ]] && kubectl logs -l app=vegeta-benchmark-$uuid -n benchmark-operator + echo "Cleaning up vegeta" + wait_clean +} + + +trap error ERR +trap finish EXIT + +function functional_test_vegeta { + check_logs=0 + test_name=$1 + cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr + echo "Performing: ${test_name}" + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + check_benchmark_for_desired_state $benchmark_name Complete 500s + check_logs=1 + + index="ripsaw-vegeta-results" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + exit 1 + fi + delete_benchmark $cr + +} + +figlet $(basename $0) +functional_test_vegeta "Vegeta benchmark" tests/test_crs/valid_vegeta.yaml +functional_test_vegeta "Vegeta benchmark hostnetwork" tests/test_crs/valid_vegeta_hostnetwork.yaml diff --git a/tests/test_ycsb.sh b/tests/test_ycsb.sh new file mode 100755 index 000000000..dc113fca4 --- /dev/null +++ b/tests/test_ycsb.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up ycsb" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_ycsb { + # stand up mongo deployment +cat << EOF | kubectl apply -f - +apiVersion: v1 +kind: Service +metadata: + name: mongo + namespace: benchmark-operator + labels: + name: mongo +spec: + ports: + - port: 27017 + targetPort: 27017 + clusterIP: None + selector: + role: mongo +EOF +cat << EOF | kubectl apply -f - +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mongo + namespace: benchmark-operator +spec: + selector: + matchLabels: + role: mongo + serviceName: "mongo" + replicas: 1 + selector: + matchLabels: + role: mongo + template: + metadata: + labels: + role: mongo + environment: test + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mongo + image: mongo + command: ["/bin/sh"] + args: ["-c", "mkdir -p /tmp/data/db; mongod --bind_ip 0.0.0.0 --dbpath /tmp/data/db"] + ports: + - containerPort: 27017 +EOF + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + cr=tests/test_crs/valid_ycsb-mongo.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + ycsb_load_pod=$(get_pod "name=ycsb-load-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 500s + + indexes="ripsaw-ycsb-summary ripsaw-ycsb-results" + if check_es "${long_uuid}" "${indexes}" + then + echo "ycsb test: Success" + else + echo "Failed to find data for ${test_name} in ES" + kubectl logs -n benchmark-operator $ycsb_load_pod + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_ycsb From 1bcdebf1c72108e81311426cc786b2e2416b81c6 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 6 Sep 2021 17:40:59 +0200 Subject: [PATCH 152/249] Migration to generic_workload Signed-off-by: Raul Sevilla --- roles/byowl/tasks/main.yml | 24 ++++---------- roles/generic_workload/tasks/main.yml | 21 ++++++++++++ roles/image_pull/tasks/main.yml | 24 ++++---------- roles/log_generator/tasks/main.yml | 24 ++++---------- roles/oslat/tasks/main.yml | 47 ++++++++++----------------- roles/scale_openshift/tasks/main.yml | 24 ++++---------- roles/vegeta/tasks/main.yml | 30 +++++++---------- 7 files changed, 74 insertions(+), 120 deletions(-) create mode 100644 roles/generic_workload/tasks/main.yml diff --git a/roles/byowl/tasks/main.yml b/roles/byowl/tasks/main.yml index 4ce2fd544..b161868cb 100644 --- a/roles/byowl/tasks/main.yml +++ b/roles/byowl/tasks/main.yml @@ -1,20 +1,8 @@ --- -- block: +- set_fact: + resources: + - "{{ role_path }}/templates/workload.yml" - - name: Start byowl job - k8s: - state: present - definition: "{{ lookup('template', 'workload.yml') | from_yaml }}" - - - include_role: - name: benchmark_state - tasks_from: set_state - vars: - state: Running - - when: benchmark_state.resources[0].status.state == "Building" - -- include_role: - name: benchmark_state - tasks_from: completed.yml - when: benchmark_state.resources[0].status.state == "Running" +- name: Generic workload - byowl + include_role: + name: generic_workload diff --git a/roles/generic_workload/tasks/main.yml b/roles/generic_workload/tasks/main.yml new file mode 100644 index 000000000..665656608 --- /dev/null +++ b/roles/generic_workload/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- block: + + - name: Create workload resources + k8s: + state: present + definition: "{{ lookup('template', item) }}" + loop: "{{ resources }}" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running + + when: benchmark_state.resources[0].status.state == "Building" + +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/image_pull/tasks/main.yml b/roles/image_pull/tasks/main.yml index 793d90d76..ac199f0af 100644 --- a/roles/image_pull/tasks/main.yml +++ b/roles/image_pull/tasks/main.yml @@ -1,20 +1,8 @@ --- -- block: - - - name: Image pull pods - k8s: - state: present - definition: "{{ lookup('template', 'image_pull.yml') }}" +- set_fact: + resources: + - "{{ role_path }}/templates/image_pull.yml" - - include_role: - name: benchmark_state - tasks_from: set_state - vars: - state: Running - - when: benchmark_state.resources[0].status.state == "Building" - -- include_role: - name: benchmark_state - tasks_from: completed.yml - when: benchmark_state.resources[0].status.state == "Running" +- name: Generic workload - image_pull + include_role: + name: generic_workload diff --git a/roles/log_generator/tasks/main.yml b/roles/log_generator/tasks/main.yml index bbf93d717..5f39129f3 100644 --- a/roles/log_generator/tasks/main.yml +++ b/roles/log_generator/tasks/main.yml @@ -1,20 +1,8 @@ --- -- block: +- set_fact: + resources: + - "{{ role_path }}/templates/log_generator.yml" - - name: Create log generator pods - k8s: - state: present - definition: "{{ lookup('template', 'log_generator.yml') }}" - - - include_role: - name: benchmark_state - tasks_from: set_state - vars: - state: Running - - when: benchmark_state.resources[0].status.state == "Building" - -- include_role: - name: benchmark_state - tasks_from: completed.yml - when: benchmark_state.resources[0].status.state == "Running" +- name: Generic workload - log_generator + include_role: + name: generic_workload diff --git a/roles/oslat/tasks/main.yml b/roles/oslat/tasks/main.yml index 64c20dafc..4a8c0df6a 100644 --- a/roles/oslat/tasks/main.yml +++ b/roles/oslat/tasks/main.yml @@ -1,32 +1,21 @@ --- -- block: - - - name: Add oslat test scripts - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' - namespace: '{{ operator_namespace }}' - data: - oslat.sh: "{{ lookup('file', 'oslat.sh') }}" - functions.sh: "{{ lookup('file', 'functions.sh') }}" - - - name: Start cyclitest job - k8s: - state: present - definition: "{{ lookup('template', 'oslatjob.yaml') | from_yaml }}" - - - include_role: - name: benchmark_state - tasks_from: set_state - vars: - state: Running - +- name: Add oslat test scripts + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' + data: + oslat.sh: "{{ lookup('file', 'oslat.sh') }}" + functions.sh: "{{ lookup('file', 'functions.sh') }}" when: benchmark_state.resources[0].status.state == "Building" -- include_role: - name: benchmark_state - tasks_from: completed.yml - when: benchmark_state.resources[0].status.state == "Running" +- set_fact: + resources: + - "{{ role_path }}/templates/oslatjob.yaml" + +- name: Generic workload - oslat + include_role: + name: generic_workload diff --git a/roles/scale_openshift/tasks/main.yml b/roles/scale_openshift/tasks/main.yml index cabb1fc2e..e680236d9 100644 --- a/roles/scale_openshift/tasks/main.yml +++ b/roles/scale_openshift/tasks/main.yml @@ -1,20 +1,8 @@ --- -- block: - - - name: Create scale pod - k8s: - state: present - definition: "{{ lookup('template', 'scale.yml') }}" +- set_fact: + resources: + - "{{ role_path }}/templates/scale.yml" - - include_role: - name: benchmark_state - tasks_from: set_state - vars: - state: Running - - when: benchmark_state.resources[0].status.state == "Building" - -- include_role: - name: benchmark_state - tasks_from: completed.yml - when: benchmark_state.resources[0].status.state == "Running" +- name: Generic workload - scale_openshift + include_role: + name: generic_workload diff --git a/roles/vegeta/tasks/main.yml b/roles/vegeta/tasks/main.yml index c5fb5e559..3edfa084f 100644 --- a/roles/vegeta/tasks/main.yml +++ b/roles/vegeta/tasks/main.yml @@ -1,23 +1,15 @@ --- -- block: - - name: Create targets configmap - k8s: - definition: "{{ lookup('template', 'targets.yml.j2') | from_yaml }}" - - - name: Create vegeta clients - k8s: - definition: "{{ lookup('template', 'vegeta.yml.j2') | from_yaml }}" - loop: "{{ range(0, workload_args.clients|default(1)|int)|list }}" +- name: Create vegeta clients + k8s: + definition: "{{ lookup('template', 'vegeta.yml.j2') | from_yaml }}" + loop: "{{ range(0, workload_args.clients|default(1)|int)|list }}" + when: benchmark_state.resources[0].status.state == "Building" - - include_role: - name: benchmark_state - tasks_from: set_state - vars: - state: Running +- set_fact: + resources: + - "{{ role_path }}/templates/targets.yml.j2" - when: benchmark_state.resources[0].status.state == "Building" -- include_role: - name: benchmark_state - tasks_from: completed.yml - when: benchmark_state.resources[0].status.state == "Running" +- name: Generic workload - vegeta + include_role: + name: generic_workload From 52d83ae122f411c3eaaf4df53b43fca717b6fe40 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 21 Sep 2021 12:10:19 +0200 Subject: [PATCH 153/249] Migrate API load benchmark Signed-off-by: Raul Sevilla --- roles/api_load/tasks/main.yml | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/roles/api_load/tasks/main.yml b/roles/api_load/tasks/main.yml index a5f90c980..a9280456f 100644 --- a/roles/api_load/tasks/main.yml +++ b/roles/api_load/tasks/main.yml @@ -1,20 +1,8 @@ --- -- block: +- set_fact: + resources: + - "{{ role_path }}/templates/api_load.yml" - - name: Api load pods - k8s: - state: present - definition: "{{ lookup('template', 'api_load.yml') | from_yaml }}" - - - include_role: - name: benchmark_state - tasks_from: set_state - vars: - state: Running - - when: benchmark_state.resources[0].status.state == "Building" - -- include_role: - name: benchmark_state - tasks_from: completed.yml - when: benchmark_state.resources[0].status.state == "Running" +- name: Generic workload - API load + include_role: + name: generic_workload From 248edd33d15944084891d9dc99e58d60c836a9d5 Mon Sep 17 00:00:00 2001 From: morenod Date: Tue, 5 Oct 2021 14:59:21 +0200 Subject: [PATCH 154/249] update sleep execution after scale tu capture run_snafu exit code correctly --- docs/e2e-ci.md | 1 + roles/scale_openshift/templates/scale.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/e2e-ci.md b/docs/e2e-ci.md index cf0826dad..cff4c41e1 100644 --- a/docs/e2e-ci.md +++ b/docs/e2e-ci.md @@ -68,3 +68,4 @@ Running tests in local has several requirements: - make - kubectl - oc +- yq diff --git a/roles/scale_openshift/templates/scale.yml b/roles/scale_openshift/templates/scale.yml index 73661ec99..7e8222846 100755 --- a/roles/scale_openshift/templates/scale.yml +++ b/roles/scale_openshift/templates/scale.yml @@ -72,7 +72,7 @@ spec: - name: AWS_DEFAULT_REGION value: "{{ workload_args.aws.default_region | default() }}" {% endif %} - command: ["/bin/sh", "-c"] + command: ["/bin/sh", "-e", "-c"] args: - "run_snafu --tool scale --scale {{workload_args.scale}} -u {{uuid}} --user {{test_user|default("ripsaw")}} --incluster true --poll_interval {{workload_args.poll_interval|default(5)}} \ {% if workload_args.debug is defined and workload_args.debug %} From 901a492c05b5ef5739a5690c9d1b30f81debdce2 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 5 Oct 2021 13:21:26 +0200 Subject: [PATCH 155/249] Make completed.yml able to detect job timeouts Signed-off-by: Raul Sevilla --- roles/benchmark_state/tasks/completed.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/benchmark_state/tasks/completed.yml b/roles/benchmark_state/tasks/completed.yml index eff33efbd..1ad95db48 100644 --- a/roles/benchmark_state/tasks/completed.yml +++ b/roles/benchmark_state/tasks/completed.yml @@ -18,7 +18,7 @@ status: state: Complete complete: true - when: job_state|json_query('resources[].status[]')|selectattr('succeeded', 'defined')|list|length == job_state.resources|length + when: job_state|json_query('resources[].status[].conditions[]')|selectattr('type', 'equalto', 'Complete')|list # Set Benchmark as failed when one of the jobs fails - name: Setting benchmark state - Failed @@ -30,7 +30,7 @@ status: state: Failed complete: true - when: job_state|json_query('resources[].status[]')|selectattr('failed', 'defined')|list|length > 0 + when: job_state|json_query('resources[].status[].conditions[]')|selectattr('type', 'equalto', 'Failed')|list - name: Get current state k8s_info: From 8e009df19f1d1946a249236d3bad83aec589f16a Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Thu, 7 Oct 2021 17:41:38 -0500 Subject: [PATCH 156/249] Update node selector for RT workloads Signed-off-by: Sai Sindhur Malleni --- docs/cyclictest.md | 1 + docs/oslat.md | 1 + roles/cyclictest/templates/cyclictestjob.yaml | 8 ++++---- roles/oslat/templates/oslatjob.yaml | 8 ++++---- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/cyclictest.md b/docs/cyclictest.md index e1c1b9c40..8c58acf34 100644 --- a/docs/cyclictest.md +++ b/docs/cyclictest.md @@ -28,6 +28,7 @@ spec: workload: name: "cyclictest" args: + node_selector: "" duration: "1m" disable_cpu_balance: true stressng: false diff --git a/docs/oslat.md b/docs/oslat.md index 70c3b450a..039fb371b 100644 --- a/docs/oslat.md +++ b/docs/oslat.md @@ -27,6 +27,7 @@ spec: workload: name: "oslat" args: + node_selector: "" runtime: "1m" disable_cpu_balance: true use_taskset: true diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index 55d08865c..3582a45df 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -14,6 +14,10 @@ spec: runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} ttlSecondsAfterFinished: 600 +{% if workload_args.nodeselector is defined %} + nodeSelector: + '{{ workload_args.node_selector.split("=")[0] }}': '{{ workload_args.node_selector.split("=")[1] }}' +{% endif %} containers: - name: cyclictest image: {{ workload_args.image | default('quay.io/cloud-bulldozer/cyclictest:latest') }} @@ -25,10 +29,6 @@ spec: {% endif %} ; imagePullPolicy: Always -{% if workload_args.nodeselector is defined %} - nodeSelector: - "{{ workload_args.nodeselector }}" -{% endif %} resources: requests: memory: {{ workload_args.pod.requests.memory }} diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index 9c48a8ced..8095b405d 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -14,6 +14,10 @@ spec: spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.nodeselector is defined %} + nodeSelector: + '{{ workload_args.node_selector.split("=")[0] }}': '{{ workload_args.node_selector.split("=")[1] }}' {% endif %} containers: - name: oslat @@ -26,10 +30,6 @@ spec: {% endif %} ; imagePullPolicy: Always -{% if workload_args.nodeselector is defined %} - nodeSelector: - "{{ workload_args.nodeselector }}" -{% endif %} resources: requests: memory: {{ workload_args.pod.requests.memory }} From fd2e24f971f78ebc5dd03e9fe4b6adf76853cb3d Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Fri, 15 Oct 2021 12:13:09 +0200 Subject: [PATCH 157/249] Handle log dumping for pods with multiple containers Signed-off-by: Raul Sevilla --- e2e/helpers.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/helpers.bash b/e2e/helpers.bash index 3c2e4b2f8..c53453a8f 100644 --- a/e2e/helpers.bash +++ b/e2e/helpers.bash @@ -76,7 +76,7 @@ die() { for pod in $(kubectl_exec get pod -l benchmark-uuid=${uuid} -o custom-columns="name:.metadata.name" --no-headers); do log_file=${TEST_ARTIFACTS}/${pod}.log echo "Saving log from pod ${pod} in ${log_file}" - kubectl_exec logs --tail=-1 ${pod} > ${log_file} + kubectl_exec logs --tail=-1 ${pod} --all-containers --prefix --all-containers --prefix > ${log_file} done false } From 3d520acc0e3337ad4eb6aeb3fe0d48358b9c39de Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Fri, 15 Oct 2021 12:49:57 +0200 Subject: [PATCH 158/249] LOGS_FLAGS Signed-off-by: Raul Sevilla --- e2e/helpers.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/helpers.bash b/e2e/helpers.bash index c53453a8f..7e8a4414e 100644 --- a/e2e/helpers.bash +++ b/e2e/helpers.bash @@ -69,6 +69,7 @@ check_benchmark() { die() { printf "\nError message: ${1}\n" local TEST_ARTIFACTS=${ARTIFACTS_DIR}/${CR_NAME} + local LOGS_FLAGS="--tail=-1 --all-containers --prefix" mkdir -p ${TEST_ARTIFACTS} echo "Dumping logs at ${TEST_ARTIFACTS}" kubectl_exec get benchmark ${CR_NAME} -o yaml --ignore-not-found > ${TEST_ARTIFACTS}/${CR_NAME}.yaml @@ -76,7 +77,7 @@ die() { for pod in $(kubectl_exec get pod -l benchmark-uuid=${uuid} -o custom-columns="name:.metadata.name" --no-headers); do log_file=${TEST_ARTIFACTS}/${pod}.log echo "Saving log from pod ${pod} in ${log_file}" - kubectl_exec logs --tail=-1 ${pod} --all-containers --prefix --all-containers --prefix > ${log_file} + kubectl_exec logs ${LOGS_FLAGS} ${pod} > ${log_file} done false } From d0604aecd8273fad720f564b1a20a9b52877703d Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 26 Oct 2021 10:12:57 +0200 Subject: [PATCH 159/249] Bump to bats v1.5.0 Signed-off-by: Raul Sevilla --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index af0d3b64f..2fca5571e 100644 --- a/Makefile +++ b/Makefile @@ -215,4 +215,4 @@ e2e-tests: ## Triggers e2e testing, by default all e2e/*.bats tests are executed .PHONY: install-bats install-bats: - VERSION=v1.4.1 ./hack/install-bats.sh + VERSION=v1.5.0 ./hack/install-bats.sh From 8b06b37a1c20bae6b5cfc1fadee4a0e7c3f9aed8 Mon Sep 17 00:00:00 2001 From: Robert Krawitz Date: Tue, 26 Oct 2021 14:55:11 -0400 Subject: [PATCH 160/249] Add optional annotations to workloads --- docs/fio_distributed.md | 6 ++++++ docs/fs-drift.md | 3 +++ docs/hammerdb.md | 7 ++++++- docs/iperf3.md | 3 +++ docs/kube-burner.md | 1 + docs/oslat.md | 3 +++ docs/pgbench.md | 3 +++ docs/scale_openshift.md | 2 ++ docs/servicemesh.md | 3 +++ docs/stressng.md | 3 +++ docs/sysbench.md | 3 +++ docs/uperf.md | 6 ++++++ docs/vegeta.md | 3 +++ docs/ycsb.md | 2 ++ roles/api_load/templates/api_load.yml | 6 ++++++ roles/byowl/templates/workload.yml | 6 ++++++ roles/cyclictest/templates/cyclictestjob.yaml | 6 ++++++ roles/fio_distributed/templates/client.yaml | 9 +++++++++ roles/fio_distributed/templates/server_vm.yml.j2 | 9 +++++++++ roles/fio_distributed/templates/servers.yaml | 9 +++++++++ roles/flent/templates/server.yml.j2 | 10 +++++++++- roles/flent/templates/workload.yml.j2 | 12 ++++++++++-- roles/fs-drift/templates/workload_job.yml.j2 | 6 ++++++ roles/hammerdb/templates/db_creation.yml | 9 +++++++++ roles/hammerdb/templates/db_creation_mariadb_vm.yml | 9 +++++++++ roles/hammerdb/templates/db_creation_mssql_vm.yml | 9 +++++++++ roles/hammerdb/templates/db_creation_postgres_vm.yml | 11 ++++++++++- roles/hammerdb/templates/db_mariadb_workload.yml.j2 | 9 +++++++++ roles/hammerdb/templates/db_mssql_workload.yml.j2 | 9 +++++++++ roles/hammerdb/templates/db_postgres_workload.yml.j2 | 9 +++++++++ roles/image_pull/templates/image_pull.yml | 6 ++++++ roles/iperf3/templates/client.yml.j2 | 9 +++++++++ roles/iperf3/templates/client_store.yml.j2 | 9 +++++++++ roles/iperf3/templates/server.yml.j2 | 9 +++++++++ roles/kube-burner/templates/kube-burner.yml.j2 | 6 ++++++ roles/log_generator/templates/log_generator.yml | 6 ++++++ roles/oslat/templates/oslatjob.yaml | 6 ++++++ roles/pgbench/templates/workload.yml.j2 | 6 ++++++ roles/scale_openshift/templates/scale.yml | 6 ++++++ roles/servicemesh/templates/job.yaml.j2 | 8 +++++++- roles/smallfile/templates/workload_job.yml.j2 | 6 ++++++ roles/stressng/templates/stressng_workload.yml.j2 | 6 ++++++ roles/stressng/templates/stressng_workload_vm.yml.j2 | 6 ++++++ roles/sysbench/templates/workload.yml | 6 ++++++ roles/sysbench/templates/workload_vm.yml | 6 ++++++ roles/system-metrics/templates/kube-burner.yml.j2 | 6 ++++++ roles/uperf/templates/server.yml.j2 | 9 +++++++++ roles/uperf/templates/server_vm.yml.j2 | 9 +++++++++ roles/uperf/templates/workload.yml.j2 | 9 +++++++++ roles/uperf/templates/workload_vm.yml.j2 | 9 +++++++++ roles/vegeta/templates/vegeta.yml.j2 | 6 ++++++ roles/ycsb/templates/ycsb_load.yaml | 6 ++++++ roles/ycsb/templates/ycsb_run.yaml | 6 ++++++ 53 files changed, 341 insertions(+), 6 deletions(-) diff --git a/docs/fio_distributed.md b/docs/fio_distributed.md index cfbb96e3b..58e3314a8 100644 --- a/docs/fio_distributed.md +++ b/docs/fio_distributed.md @@ -100,6 +100,12 @@ The workload loops are nested as such from the CR options: > to instantiate the files themselves prior to beginning the benchmark workload. - **runtime_class** : If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. > Note: For Kata containers +- **annotations** : If this is set, the benchmark-operator will set + > the specified annotations on the pods' metadata. +- **server_annotations** : If this is set, the benchmark-operator will set + > the specified annotations on the server pods' metadata. +- **client_annotations** : If this is set, the benchmark-operator will set + > the specified annotations on the client pods' metadata. - **kind**: Can either be `pod` or `vm` to determine if the fio workload is run in a Pod or in a VM > Note: For VM workloads, you need to install Openshift Virtualization first - **vm_image**: Whether to use a pre-defined VM image with pre-installed requirements. Necessary for disconnected installs. diff --git a/docs/fs-drift.md b/docs/fs-drift.md index 2fc7f6c9b..8ff3e796e 100644 --- a/docs/fs-drift.md +++ b/docs/fs-drift.md @@ -53,6 +53,9 @@ The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + Once done creating/editing the CR file below, one can run it by: ```bash diff --git a/docs/hammerdb.md b/docs/hammerdb.md index 3dd8bfc5f..e19582429 100644 --- a/docs/hammerdb.md +++ b/docs/hammerdb.md @@ -150,6 +150,11 @@ The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. To set annotations only on the +workload, use **client_annotations**; to set annotations only on the +database server, use **server_annotations**. + Once done creating/editing the resource file, you can run it by: ```bash @@ -241,4 +246,4 @@ podman push You can either access results by indexing them directly or by accessing the console. The results are stored in /tmp/ directory -You can console into VM by running `virtctl console vmi_name` \ No newline at end of file +You can console into VM by running `virtctl console vmi_name` diff --git a/docs/iperf3.md b/docs/iperf3.md index 51c035a4d..ce9d91d48 100644 --- a/docs/iperf3.md +++ b/docs/iperf3.md @@ -50,6 +50,9 @@ then you'd probably give a `retries` of 60, as 60*15 is 900s. runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +`annotations` can be set to apply the specified +annotations to the pod metadata. + The rest of the args are compulsory arguments that need to be passed and can cause issues if missed, they are: diff --git a/docs/kube-burner.md b/docs/kube-burner.md index 5bee08142..2a4ae3000 100644 --- a/docs/kube-burner.md +++ b/docs/kube-burner.md @@ -103,6 +103,7 @@ Where key defaults to __node-role.kubernetes.io/worker__ and value defaults to e - **``step``**: Prometheus step size, useful for long benchmarks. Defaults to 30s - **``metrics_profile``**: kube-burner metric profile that indicates what prometheus metrics kube-burner will collect. Defaults to `metrics.yaml` in node-density workloads and `metrics-aggregated.yaml` in the remaining. Detailed in the [Metrics section](#Metrics) of this document - **``runtime_class``** : If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. +- **``annotations``** : If this is set, the benchmark-operator will set the specified annotations on the pod's metadata. - **``extra_env_vars``** : This dictionary defines a set of fields that will be injected to the kube-burner pod as environment variables. e.g. `extra_env_vars: {"foo": "bar", "foo2": "bar2"}` kube-burner is able to collect complex prometheus metrics and index them in a ElasticSearch. This feature can be configured by the prometheus object of kube-burner's CR. diff --git a/docs/oslat.md b/docs/oslat.md index 039fb371b..e53d671bf 100644 --- a/docs/oslat.md +++ b/docs/oslat.md @@ -13,6 +13,9 @@ The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + An example CR might look like this ```yaml diff --git a/docs/pgbench.md b/docs/pgbench.md index 740073f38..784634d45 100644 --- a/docs/pgbench.md +++ b/docs/pgbench.md @@ -78,6 +78,9 @@ The `runtime_class` option can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option `annotations` can be set to apply the specified +annotations to the pod metadata. + Once done creating/editing the resource file, you can run it by: ```bash diff --git a/docs/scale_openshift.md b/docs/scale_openshift.md index ab2306c7b..a24d8aa68 100644 --- a/docs/scale_openshift.md +++ b/docs/scale_openshift.md @@ -28,6 +28,8 @@ Optional variables: `runtime_class` If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. +`annotations` If this is set, the benchmark-operator will set the specified annotations on the pods' metadata. + `rosa` For clusters installed using ROSA. Following parameters will be required: `rosa.cluster_name` Name of cluster as it is shown on `rosa list clusters` command diff --git a/docs/servicemesh.md b/docs/servicemesh.md index 7e4ec1543..2b916b59d 100644 --- a/docs/servicemesh.md +++ b/docs/servicemesh.md @@ -20,6 +20,9 @@ The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + ## Running the benchmark Here is an example of the [benchmark CR](../config/samples/servicemesh/cr.yaml): diff --git a/docs/stressng.md b/docs/stressng.md index 61ec360ce..3354bea3f 100644 --- a/docs/stressng.md +++ b/docs/stressng.md @@ -11,6 +11,9 @@ The optional argument **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + An example CR might look like this ```yaml diff --git a/docs/sysbench.md b/docs/sysbench.md index 88d80d951..43b1f37c3 100644 --- a/docs/sysbench.md +++ b/docs/sysbench.md @@ -11,6 +11,9 @@ The optional argument **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + The `pin_node` parameter allows to place the sysbench pod on a specific node, using the `hostname` label. diff --git a/docs/uperf.md b/docs/uperf.md index 1a186283a..ec37a53c0 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -68,6 +68,12 @@ spec: *Note:* `runtime_class` has only been tested with Kata containers. Only include `runtime_class` if using Kata containers. +`annotations` If this is set, the benchmark-operator will set the specified annotations on the pods' metadata. + +`server_annotations` If this is set, the benchmark-operator will set the specified annotations on the server pods' metadata. + +`client_annotations` If this is set, the benchmark-operator will set the specified annotations on the client pods' metadata. + `hostnetwork` will test the performance of the node the pod will run on. `networkpolicy` will create a simple networkpolicy for ingress diff --git a/docs/vegeta.md b/docs/vegeta.md index c769c908e..743ed9f03 100644 --- a/docs/vegeta.md +++ b/docs/vegeta.md @@ -11,6 +11,9 @@ The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + The **node_selector** option can be used to limit the nodes where the vegeta pods are deployed. diff --git a/docs/ycsb.md b/docs/ycsb.md index c3a5fb3ab..268109a19 100644 --- a/docs/ycsb.md +++ b/docs/ycsb.md @@ -68,6 +68,8 @@ so that the ycsb pod can access the API of database. `runtime_class`: If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. +`annotations` If this is set, the benchmark-operator will set the specified annotations on the pods' metadata. + Once done creating/editing the resource file, you can run it by: ```bash diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index 1cb6e94d1..65dbb15da 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -13,6 +13,12 @@ spec: labels: app: api-load-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/byowl/templates/workload.yml b/roles/byowl/templates/workload.yml index c8f353c2a..834b12509 100755 --- a/roles/byowl/templates/workload.yml +++ b/roles/byowl/templates/workload.yml @@ -12,6 +12,12 @@ spec: labels: app: byowl-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index 3582a45df..be8514a1b 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -9,6 +9,12 @@ spec: labels: app: cyclictest-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/fio_distributed/templates/client.yaml b/roles/fio_distributed/templates/client.yaml index 3256e4bf7..965c6f555 100644 --- a/roles/fio_distributed/templates/client.yaml +++ b/roles/fio_distributed/templates/client.yaml @@ -12,6 +12,15 @@ spec: labels: benchmark-uuid: {{ uuid }} app: fiod-client-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/fio_distributed/templates/server_vm.yml.j2 b/roles/fio_distributed/templates/server_vm.yml.j2 index c302516d3..27a28aa23 100644 --- a/roles/fio_distributed/templates/server_vm.yml.j2 +++ b/roles/fio_distributed/templates/server_vm.yml.j2 @@ -7,6 +7,15 @@ metadata: labels: app: fio-benchmark-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: cpu: diff --git a/roles/fio_distributed/templates/servers.yaml b/roles/fio_distributed/templates/servers.yaml index 062c2f42c..42af4ef60 100644 --- a/roles/fio_distributed/templates/servers.yaml +++ b/roles/fio_distributed/templates/servers.yaml @@ -7,6 +7,15 @@ metadata: labels: benchmark-uuid: {{ uuid }} app: fio-benchmark-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: affinity: podAntiAffinity: diff --git a/roles/flent/templates/server.yml.j2 b/roles/flent/templates/server.yml.j2 index 18ffa49af..07529840c 100644 --- a/roles/flent/templates/server.yml.j2 +++ b/roles/flent/templates/server.yml.j2 @@ -8,10 +8,18 @@ metadata: app : flent-bench-server-{{item}}-{{ trunc_uuid }} type : flent-bench-server-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} -{% if workload_args.multus.enabled is sameas true %} +{% if workload_args.multus.enabled is sameas true or workload_args.annotations is defined or workload_args.server_annotations is defined %} annotations: +{% if workload_args.multus.enabled is sameas true %} k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} {% endif %} +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/flent/templates/workload.yml.j2 b/roles/flent/templates/workload.yml.j2 index e9c0ed7e2..767f8c4b2 100644 --- a/roles/flent/templates/workload.yml.j2 +++ b/roles/flent/templates/workload.yml.j2 @@ -11,9 +11,17 @@ spec: app: flent-bench-client-{{ trunc_uuid }} clientfor: {{ item.metadata.labels.app }} benchmark-uuid: {{ uuid }} +{% if workload_args.multus.enabled is sameas true or workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: {% if workload_args.multus.enabled is sameas true %} - annotations: - k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} +{% endif %} +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} {% endif %} spec: {% if workload_args.hostnetwork is sameas true %} diff --git a/roles/fs-drift/templates/workload_job.yml.j2 b/roles/fs-drift/templates/workload_job.yml.j2 index 699b38af6..406b7a4d5 100644 --- a/roles/fs-drift/templates/workload_job.yml.j2 +++ b/roles/fs-drift/templates/workload_job.yml.j2 @@ -12,6 +12,12 @@ spec: labels: app: fs-drift-benchmark-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/hammerdb/templates/db_creation.yml b/roles/hammerdb/templates/db_creation.yml index c85670350..5be9d77b8 100644 --- a/roles/hammerdb/templates/db_creation.yml +++ b/roles/hammerdb/templates/db_creation.yml @@ -13,6 +13,15 @@ spec: app: hammerdb_creator-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-creator-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.pin is sameas true %} nodeSelector: diff --git a/roles/hammerdb/templates/db_creation_mariadb_vm.yml b/roles/hammerdb/templates/db_creation_mariadb_vm.yml index ddbc0007f..ecdd6a14f 100644 --- a/roles/hammerdb/templates/db_creation_mariadb_vm.yml +++ b/roles/hammerdb/templates/db_creation_mariadb_vm.yml @@ -8,6 +8,15 @@ metadata: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: cpu: diff --git a/roles/hammerdb/templates/db_creation_mssql_vm.yml b/roles/hammerdb/templates/db_creation_mssql_vm.yml index c6a018d84..66ce8ddc0 100644 --- a/roles/hammerdb/templates/db_creation_mssql_vm.yml +++ b/roles/hammerdb/templates/db_creation_mssql_vm.yml @@ -8,6 +8,15 @@ metadata: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: cpu: diff --git a/roles/hammerdb/templates/db_creation_postgres_vm.yml b/roles/hammerdb/templates/db_creation_postgres_vm.yml index ad3c4582c..c0e496bd7 100644 --- a/roles/hammerdb/templates/db_creation_postgres_vm.yml +++ b/roles/hammerdb/templates/db_creation_postgres_vm.yml @@ -8,6 +8,15 @@ metadata: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: cpu: @@ -119,4 +128,4 @@ spec: capacity: {{ workload_args.client_vm.hostpath_storagesize | default("10Gi") }} type: DiskOrCreate {% endif %} -status: {} \ No newline at end of file +status: {} diff --git a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 index 434aabb58..3556a204d 100644 --- a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 @@ -13,6 +13,15 @@ spec: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.pin is sameas true %} nodeSelector: diff --git a/roles/hammerdb/templates/db_mssql_workload.yml.j2 b/roles/hammerdb/templates/db_mssql_workload.yml.j2 index 152353797..617becdde 100644 --- a/roles/hammerdb/templates/db_mssql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mssql_workload.yml.j2 @@ -13,6 +13,15 @@ spec: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.pin is sameas true %} nodeSelector: diff --git a/roles/hammerdb/templates/db_postgres_workload.yml.j2 b/roles/hammerdb/templates/db_postgres_workload.yml.j2 index 9c5baa8f6..31d6758fb 100644 --- a/roles/hammerdb/templates/db_postgres_workload.yml.j2 +++ b/roles/hammerdb/templates/db_postgres_workload.yml.j2 @@ -13,6 +13,15 @@ spec: app: hammerdb_workload-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.pin is sameas true %} nodeSelector: diff --git a/roles/image_pull/templates/image_pull.yml b/roles/image_pull/templates/image_pull.yml index ad2434cda..0ce495890 100644 --- a/roles/image_pull/templates/image_pull.yml +++ b/roles/image_pull/templates/image_pull.yml @@ -13,6 +13,12 @@ spec: labels: app: image-pull-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/iperf3/templates/client.yml.j2 b/roles/iperf3/templates/client.yml.j2 index 02d083848..57d686bb5 100644 --- a/roles/iperf3/templates/client.yml.j2 +++ b/roles/iperf3/templates/client.yml.j2 @@ -12,6 +12,15 @@ spec: labels: app: iperf3-bench-client-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: affinity: podAntiAffinity: diff --git a/roles/iperf3/templates/client_store.yml.j2 b/roles/iperf3/templates/client_store.yml.j2 index 8c54a2f44..f856b2d33 100644 --- a/roles/iperf3/templates/client_store.yml.j2 +++ b/roles/iperf3/templates/client_store.yml.j2 @@ -11,6 +11,15 @@ spec: labels: app: iperf3-bench-client-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: affinity: podAntiAffinity: diff --git a/roles/iperf3/templates/server.yml.j2 b/roles/iperf3/templates/server.yml.j2 index 56761f358..aded32b49 100644 --- a/roles/iperf3/templates/server.yml.j2 +++ b/roles/iperf3/templates/server.yml.j2 @@ -7,6 +7,15 @@ metadata: labels: benchmark-uuid: {{ uuid }} app: iperf3-bench-server-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: affinity: podAntiAffinity: diff --git a/roles/kube-burner/templates/kube-burner.yml.j2 b/roles/kube-burner/templates/kube-burner.yml.j2 index f08c8e778..c6b37e69f 100644 --- a/roles/kube-burner/templates/kube-burner.yml.j2 +++ b/roles/kube-burner/templates/kube-burner.yml.j2 @@ -12,6 +12,12 @@ spec: labels: app: kube-burner-benchmark-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/log_generator/templates/log_generator.yml b/roles/log_generator/templates/log_generator.yml index 24cbb9899..76fd91cd4 100644 --- a/roles/log_generator/templates/log_generator.yml +++ b/roles/log_generator/templates/log_generator.yml @@ -13,6 +13,12 @@ spec: labels: app: log-generator-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index 8095b405d..9e21f3155 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -11,6 +11,12 @@ spec: labels: app: oslat-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/pgbench/templates/workload.yml.j2 b/roles/pgbench/templates/workload.yml.j2 index 7cdb51858..c4eef932a 100644 --- a/roles/pgbench/templates/workload.yml.j2 +++ b/roles/pgbench/templates/workload.yml.j2 @@ -12,6 +12,12 @@ spec: labels: app: pgbench-client-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/scale_openshift/templates/scale.yml b/roles/scale_openshift/templates/scale.yml index 7e8222846..9e264bb2d 100755 --- a/roles/scale_openshift/templates/scale.yml +++ b/roles/scale_openshift/templates/scale.yml @@ -12,6 +12,12 @@ spec: labels: app: scale-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/servicemesh/templates/job.yaml.j2 b/roles/servicemesh/templates/job.yaml.j2 index 2c08feeb0..190169afd 100644 --- a/roles/servicemesh/templates/job.yaml.j2 +++ b/roles/servicemesh/templates/job.yaml.j2 @@ -10,6 +10,12 @@ spec: metadata: labels: app: "{{ ansible_operator_meta.name }}-{{ trunc_uuid }}" +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -31,4 +37,4 @@ spec: - name: files configMap: defaultMode: 493 - name: "{{ ansible_operator_meta.name }}-test-{{ trunc_uuid }}" \ No newline at end of file + name: "{{ ansible_operator_meta.name }}-test-{{ trunc_uuid }}" diff --git a/roles/smallfile/templates/workload_job.yml.j2 b/roles/smallfile/templates/workload_job.yml.j2 index c8a36c769..21e82fea7 100644 --- a/roles/smallfile/templates/workload_job.yml.j2 +++ b/roles/smallfile/templates/workload_job.yml.j2 @@ -12,6 +12,12 @@ spec: labels: app: smallfile-benchmark-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/stressng/templates/stressng_workload.yml.j2 b/roles/stressng/templates/stressng_workload.yml.j2 index 917bac5bc..24841e56f 100644 --- a/roles/stressng/templates/stressng_workload.yml.j2 +++ b/roles/stressng/templates/stressng_workload.yml.j2 @@ -15,6 +15,12 @@ spec: type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} benchmark-operator-workload: stressng +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.pin is sameas true %} nodeSelector: diff --git a/roles/stressng/templates/stressng_workload_vm.yml.j2 b/roles/stressng/templates/stressng_workload_vm.yml.j2 index 9a1ad6a2a..0eb67a3fd 100644 --- a/roles/stressng/templates/stressng_workload_vm.yml.j2 +++ b/roles/stressng/templates/stressng_workload_vm.yml.j2 @@ -9,6 +9,12 @@ metadata: type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} benchmark-operator-workload: stressng +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: cpu: diff --git a/roles/sysbench/templates/workload.yml b/roles/sysbench/templates/workload.yml index a736f7c23..f7aa605d4 100644 --- a/roles/sysbench/templates/workload.yml +++ b/roles/sysbench/templates/workload.yml @@ -13,6 +13,12 @@ spec: labels: app: sysbench-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.pin_node is defined %} nodeSelector: diff --git a/roles/sysbench/templates/workload_vm.yml b/roles/sysbench/templates/workload_vm.yml index e3416a999..40de88329 100644 --- a/roles/sysbench/templates/workload_vm.yml +++ b/roles/sysbench/templates/workload_vm.yml @@ -8,6 +8,12 @@ metadata: labels: kubevirt-vm: vm-sysbench-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: devices: diff --git a/roles/system-metrics/templates/kube-burner.yml.j2 b/roles/system-metrics/templates/kube-burner.yml.j2 index 665f10f99..10a7aa2ad 100644 --- a/roles/system-metrics/templates/kube-burner.yml.j2 +++ b/roles/system-metrics/templates/kube-burner.yml.j2 @@ -12,6 +12,12 @@ spec: labels: app: system-metrics-collector-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index c5285192e..bfb1f520f 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -13,6 +13,15 @@ items: name: 'uperf-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }}' {% endif %} namespace: "{{ operator_namespace }}" +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: ttlSecondsAfterFinished: 600 backoffLimit: 0 diff --git a/roles/uperf/templates/server_vm.yml.j2 b/roles/uperf/templates/server_vm.yml.j2 index d006baf5b..48ba5703f 100644 --- a/roles/uperf/templates/server_vm.yml.j2 +++ b/roles/uperf/templates/server_vm.yml.j2 @@ -10,6 +10,15 @@ metadata: benchmark-operator-role: server app : uperf-bench-server-{{ item }}-{{ trunc_uuid }} type : {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: cpu: diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 990254949..ee260924a 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -13,6 +13,15 @@ items: name: 'uperf-client-{{item.status.podIP}}-{{ trunc_uuid }}' {% endif %} namespace: '{{ operator_namespace }}' +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: template: metadata: diff --git a/roles/uperf/templates/workload_vm.yml.j2 b/roles/uperf/templates/workload_vm.yml.j2 index 0053db676..7f548e2b6 100644 --- a/roles/uperf/templates/workload_vm.yml.j2 +++ b/roles/uperf/templates/workload_vm.yml.j2 @@ -10,6 +10,15 @@ metadata: benchmark-operator-role: client app: uperf-bench-client-{{ trunc_uuid }} type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: cpu: diff --git a/roles/vegeta/templates/vegeta.yml.j2 b/roles/vegeta/templates/vegeta.yml.j2 index d0ed029b5..6730c5abd 100644 --- a/roles/vegeta/templates/vegeta.yml.j2 +++ b/roles/vegeta/templates/vegeta.yml.j2 @@ -12,6 +12,12 @@ spec: labels: app: vegeta-benchmark-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.node_selector is defined %} nodeSelector: diff --git a/roles/ycsb/templates/ycsb_load.yaml b/roles/ycsb/templates/ycsb_load.yaml index 4985315f6..78d13ad21 100644 --- a/roles/ycsb/templates/ycsb_load.yaml +++ b/roles/ycsb/templates/ycsb_load.yaml @@ -11,6 +11,12 @@ spec: namespace: '{{ operator_namespace }}' labels: name: 'ycsb-load-{{ trunc_uuid }}' +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" diff --git a/roles/ycsb/templates/ycsb_run.yaml b/roles/ycsb/templates/ycsb_run.yaml index 5d5cfbbc1..f4397a401 100644 --- a/roles/ycsb/templates/ycsb_run.yaml +++ b/roles/ycsb/templates/ycsb_run.yaml @@ -11,6 +11,12 @@ spec: namespace: '{{ operator_namespace }}' labels: name: 'ycsb-run-{{ trunc_uuid }}' +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" From a0492bb752b71071326f964f138018aa7b7b941a Mon Sep 17 00:00:00 2001 From: Robert Krawitz Date: Wed, 27 Oct 2021 15:11:07 -0400 Subject: [PATCH 161/249] Fix indentation --- roles/uperf/templates/server.yml.j2 | 6 +++--- roles/uperf/templates/server_vm.yml.j2 | 6 +++--- roles/uperf/templates/workload.yml.j2 | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index bfb1f520f..f55feabb5 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -14,12 +14,12 @@ items: {% endif %} namespace: "{{ operator_namespace }}" {% if workload_args.annotations is defined or workload_args.server_annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% for annotation, value in workload_args.server_annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} spec: diff --git a/roles/uperf/templates/server_vm.yml.j2 b/roles/uperf/templates/server_vm.yml.j2 index 48ba5703f..ad3f438f3 100644 --- a/roles/uperf/templates/server_vm.yml.j2 +++ b/roles/uperf/templates/server_vm.yml.j2 @@ -11,12 +11,12 @@ metadata: app : uperf-bench-server-{{ item }}-{{ trunc_uuid }} type : {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} {% if workload_args.annotations is defined or workload_args.server_annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% for annotation, value in workload_args.server_annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} spec: diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index ee260924a..aa8b9325f 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -14,12 +14,12 @@ items: {% endif %} namespace: '{{ operator_namespace }}' {% if workload_args.annotations is defined or workload_args.server_annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% for annotation, value in workload_args.server_annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} spec: From ae117009213f322b069a324649ed9916b41a006d Mon Sep 17 00:00:00 2001 From: Robert Krawitz Date: Wed, 27 Oct 2021 15:50:19 -0400 Subject: [PATCH 162/249] Fix more indentation problems --- roles/fio_distributed/templates/server_vm.yml.j2 | 6 +++--- roles/fio_distributed/templates/servers.yaml | 6 +++--- roles/hammerdb/templates/db_creation_mariadb_vm.yml | 6 +++--- roles/hammerdb/templates/db_creation_mssql_vm.yml | 6 +++--- roles/hammerdb/templates/db_creation_postgres_vm.yml | 6 +++--- roles/stressng/templates/stressng_workload_vm.yml.j2 | 4 ++-- roles/sysbench/templates/workload_vm.yml | 4 ++-- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/roles/fio_distributed/templates/server_vm.yml.j2 b/roles/fio_distributed/templates/server_vm.yml.j2 index 27a28aa23..1aa1fa0bc 100644 --- a/roles/fio_distributed/templates/server_vm.yml.j2 +++ b/roles/fio_distributed/templates/server_vm.yml.j2 @@ -8,12 +8,12 @@ metadata: app: fio-benchmark-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} {% if workload_args.annotations is defined or workload_args.server_annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% for annotation, value in workload_args.server_annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} spec: diff --git a/roles/fio_distributed/templates/servers.yaml b/roles/fio_distributed/templates/servers.yaml index 42af4ef60..37c3a1197 100644 --- a/roles/fio_distributed/templates/servers.yaml +++ b/roles/fio_distributed/templates/servers.yaml @@ -8,12 +8,12 @@ metadata: benchmark-uuid: {{ uuid }} app: fio-benchmark-{{ trunc_uuid }} {% if workload_args.annotations is defined or workload_args.server_annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% for annotation, value in workload_args.server_annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} spec: diff --git a/roles/hammerdb/templates/db_creation_mariadb_vm.yml b/roles/hammerdb/templates/db_creation_mariadb_vm.yml index ecdd6a14f..a28d83e91 100644 --- a/roles/hammerdb/templates/db_creation_mariadb_vm.yml +++ b/roles/hammerdb/templates/db_creation_mariadb_vm.yml @@ -9,12 +9,12 @@ metadata: type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} {% if workload_args.annotations is defined or workload_args.server_annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% for annotation, value in workload_args.server_annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} spec: diff --git a/roles/hammerdb/templates/db_creation_mssql_vm.yml b/roles/hammerdb/templates/db_creation_mssql_vm.yml index 66ce8ddc0..3f4c34032 100644 --- a/roles/hammerdb/templates/db_creation_mssql_vm.yml +++ b/roles/hammerdb/templates/db_creation_mssql_vm.yml @@ -9,12 +9,12 @@ metadata: type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} {% if workload_args.annotations is defined or workload_args.server_annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% for annotation, value in workload_args.server_annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} spec: diff --git a/roles/hammerdb/templates/db_creation_postgres_vm.yml b/roles/hammerdb/templates/db_creation_postgres_vm.yml index c0e496bd7..102e4a0b5 100644 --- a/roles/hammerdb/templates/db_creation_postgres_vm.yml +++ b/roles/hammerdb/templates/db_creation_postgres_vm.yml @@ -9,12 +9,12 @@ metadata: type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} {% if workload_args.annotations is defined or workload_args.server_annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% for annotation, value in workload_args.server_annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} spec: diff --git a/roles/stressng/templates/stressng_workload_vm.yml.j2 b/roles/stressng/templates/stressng_workload_vm.yml.j2 index 0eb67a3fd..994ae5709 100644 --- a/roles/stressng/templates/stressng_workload_vm.yml.j2 +++ b/roles/stressng/templates/stressng_workload_vm.yml.j2 @@ -10,9 +10,9 @@ metadata: benchmark-uuid: {{ uuid }} benchmark-operator-workload: stressng {% if workload_args.annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} spec: diff --git a/roles/sysbench/templates/workload_vm.yml b/roles/sysbench/templates/workload_vm.yml index 40de88329..5f03efd73 100644 --- a/roles/sysbench/templates/workload_vm.yml +++ b/roles/sysbench/templates/workload_vm.yml @@ -9,9 +9,9 @@ metadata: kubevirt-vm: vm-sysbench-{{ trunc_uuid }} benchmark-uuid: {{ uuid }} {% if workload_args.annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} spec: From 126d76ca793bb672ca74b512a931e0aaa5f869f2 Mon Sep 17 00:00:00 2001 From: Marko Karg Date: Thu, 28 Oct 2021 16:19:41 +0200 Subject: [PATCH 163/249] fixing a naming issue --- roles/cyclictest/templates/cyclictestjob.yaml | 2 +- roles/oslat/templates/oslatjob.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index be8514a1b..30cc68455 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -20,7 +20,7 @@ spec: runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} ttlSecondsAfterFinished: 600 -{% if workload_args.nodeselector is defined %} +{% if workload_args.node_selector is defined %} nodeSelector: '{{ workload_args.node_selector.split("=")[0] }}': '{{ workload_args.node_selector.split("=")[1] }}' {% endif %} diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index 9e21f3155..01b74359c 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -21,7 +21,7 @@ spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} -{% if workload_args.nodeselector is defined %} +{% if workload_args.node_selector is defined %} nodeSelector: '{{ workload_args.node_selector.split("=")[0] }}': '{{ workload_args.node_selector.split("=")[1] }}' {% endif %} From 6b20f59efd68b90998c62571d6613c8b0c7fe148 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Thu, 16 Sep 2021 18:42:17 +0200 Subject: [PATCH 164/249] Adding multi test run to benchmark Signed-off-by: Vicente Zepeda Mas --- roles/api_load/templates/api_load.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index 65dbb15da..2c338addf 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -67,6 +67,7 @@ spec: command: ["/bin/sh", "-c"] args: - > +{% for key, value in workload_args.test_list.items() %} ocm-load-test {% if workload_args.override is defined %} {{ workload_args.override }} @@ -74,14 +75,25 @@ spec: --test-id={{ uuid }} --gateway-url {{ workload_args.gateway_url }} --ocm-token={{ workload_args.ocm_token }} +{% if value.duration is defined %} + --duration={{ value.duration}} +{% else %} --duration={{ workload_args.duration}} +{% endif %} +{% if value.rate is defined %} + --rate={{ value.rate}} +{% else %} --rate={{ workload_args.rate}} +{% endif %} --output-path={{ workload_args.output_path}} - --test-names {{ workload_args.test_name }} + --test-names {{ key }} --aws-access-key {{ workload_args.aws_access_key }} --aws-access-secret {{ workload_args.aws_access_secret }} --aws-account-id {{ workload_args.aws_account_id }}; + echo "Cooldown for {{ workload_args.cooldown }}"; + sleep {{ workload_args.cooldown }}; {% endif %} +{% endfor %} volumeMounts: - mountPath: /tmp/results name: results From f779d1813b96e3cbef9ba0d6058a04313f9f5a77 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Mon, 11 Oct 2021 15:03:57 +0200 Subject: [PATCH 165/249] Initial data-collection pod Signed-off-by: Vicente Zepeda Mas --- .../ripsaw.cloudbulldozer.io_benchmarks.yaml | 15 +++- config/manager/kustomization.yaml | 2 +- roles/api_load/templates/api_load.yml | 82 +++++++++++++------ tests/test_crs/valid_api_load.yaml | 4 + 4 files changed, 78 insertions(+), 25 deletions(-) diff --git a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml index 473f720a8..b4bb9efda 100644 --- a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml +++ b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml @@ -21,7 +21,7 @@ spec: spec: type: object properties: - uuid: + uuid: type: string elasticsearch: default: {} @@ -38,6 +38,19 @@ spec: verify_cert: type: boolean default: true + snappy: + default: {} + type: object + properties: + url: + type: string + default: "" + user: + type: string + default: "" + password: + type: string + default: "" prometheus: type: object properties: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index f6241927a..6a0c6fd37 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: quay.io/cloud-bulldozer/benchmark-operator + newName: quay.io/chentex/benchmark-operator newTag: latest diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index 2c338addf..d50bd335b 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -41,8 +41,8 @@ spec: - {{ workload_args.label.value }} {% endif %} containers: - - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/ocm-api-load:latest') }} - name: pull-image + - name: api-load + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/ocm-api-load:latest') }} env: - name: my_node_name valueFrom: @@ -54,19 +54,11 @@ spec: fieldPath: metadata.name - name: uuid value: "{{ uuid }}" -{% if elasticsearch is defined %} - - name: es - value: "{{ elasticsearch.url }}" - - name: es_index - value: "{{ elasticsearch.index_name | default("api-load") }}" - - name: es_verify_cert - value: "{{ elasticsearch.verify_cert | default(true) }}" - - name: parallel - value: "{{ elasticsearch.parallel | default(false) }}" -{% endif %} command: ["/bin/sh", "-c"] args: - > + echo "Seting redis status"; + redis-cli -h {{ bo.resources[0].status.podIP }} SET "{{ uuid }}-status" "running"; {% for key, value in workload_args.test_list.items() %} ocm-load-test {% if workload_args.override is defined %} @@ -75,17 +67,9 @@ spec: --test-id={{ uuid }} --gateway-url {{ workload_args.gateway_url }} --ocm-token={{ workload_args.ocm_token }} -{% if value.duration is defined %} - --duration={{ value.duration}} -{% else %} - --duration={{ workload_args.duration}} -{% endif %} -{% if value.rate is defined %} - --rate={{ value.rate}} -{% else %} - --rate={{ workload_args.rate}} -{% endif %} - --output-path={{ workload_args.output_path}} + --duration={{ value.duration | default(workload_args.duration) }} + --rate={{ value.rate | default(workload_args.rate) }} + --output-path={{ workload_args.output_path }} --test-names {{ key }} --aws-access-key {{ workload_args.aws_access_key }} --aws-access-secret {{ workload_args.aws_access_secret }} @@ -94,6 +78,58 @@ spec: sleep {{ workload_args.cooldown }}; {% endif %} {% endfor %} + redis-cli -h {{ bo.resources[0].status.podIP }} SET "{{ uuid }}-status" "ready"; + volumeMounts: + - mountPath: /tmp/results + name: results + - name: data-collector + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/ocm-api-load:latest') }} + env: + - name: my_node_name + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: my_pod_name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: uuid + value: "{{ uuid }}" +{% if elasticsearch is defined %} + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("api-load") }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(false) }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" +{% endif %} +{% if snappy is defined %} + - name: snappy_server + value: "{{ snappy.url }}" + - name: snappy_user + value: "{{ snappy.user }}" + - name: snappy_password + value: "{{ snappy.password }}" +{% endif %} + command: ["/bin/sh", "-c"] + args: + - > + echo "Waiting for test to end running"; + echo {{ es }} + status=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "{{ uuid }}-status"`; + while [[ $status != "ready" ]]; do + echo "Sleeping for 5 min"; + sleep 5; + echo "Testing test status"; + status=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "{{ uuid }}-status"`; + done; + echo "Starting with ES bulk upload"; + python3 automation.py esbulk --dir {{ workload_args.output_path }} --index {{ es_index }}; + echo "Starting with Snappy upload"; + python3 automation.py upload --dir {{ workload_args.output_path }} --server {{ snappy_server }} --user {{ snappy_user }} --password {{ snappy_password }}; + echo "Process finished"; volumeMounts: - mountPath: /tmp/results name: results diff --git a/tests/test_crs/valid_api_load.yaml b/tests/test_crs/valid_api_load.yaml index d49a497a1..d075ada49 100644 --- a/tests/test_crs/valid_api_load.yaml +++ b/tests/test_crs/valid_api_load.yaml @@ -7,6 +7,10 @@ spec: elasticsearch: url: ES_SERVER index_name: "api-load" + snappy: + url: SNAPPY_SERVER + user: realuser + password: realpassword workload: name: api_load args: From ae95aeadcc5b69573ae634bfa51f1406504d6059 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Tue, 12 Oct 2021 23:40:10 +0200 Subject: [PATCH 166/249] ES indexing and upload working Signed-off-by: Vicente Zepeda Mas --- config/manager/kustomization.yaml | 2 +- roles/api_load/templates/api_load.yml | 39 +++++++-------------------- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 6a0c6fd37..f6241927a 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: quay.io/chentex/benchmark-operator + newName: quay.io/cloud-bulldozer/benchmark-operator newTag: latest diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index d50bd335b..a30949ecb 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -57,7 +57,7 @@ spec: command: ["/bin/sh", "-c"] args: - > - echo "Seting redis status"; + echo "Setting redis status"; redis-cli -h {{ bo.resources[0].status.podIP }} SET "{{ uuid }}-status" "running"; {% for key, value in workload_args.test_list.items() %} ocm-load-test @@ -95,41 +95,22 @@ spec: fieldPath: metadata.name - name: uuid value: "{{ uuid }}" -{% if elasticsearch is defined %} - - name: es + - name: ES value: "{{ elasticsearch.url }}" - - name: es_index - value: "{{ elasticsearch.index_name | default("api-load") }}" - - name: es_verify_cert - value: "{{ elasticsearch.verify_cert | default(false) }}" - - name: parallel - value: "{{ elasticsearch.parallel | default(false) }}" -{% endif %} -{% if snappy is defined %} - - name: snappy_server - value: "{{ snappy.url }}" - - name: snappy_user - value: "{{ snappy.user }}" - - name: snappy_password - value: "{{ snappy.password }}" -{% endif %} command: ["/bin/sh", "-c"] args: - > - echo "Waiting for test to end running"; - echo {{ es }} + echo "Getting redis status"; status=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "{{ uuid }}-status"`; - while [[ $status != "ready" ]]; do - echo "Sleeping for 5 min"; - sleep 5; - echo "Testing test status"; + while [ $status != "ready" ]; do + sleep {{ workload_args.sleep }}; + echo "Testing readiness"; status=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "{{ uuid }}-status"`; done; - echo "Starting with ES bulk upload"; - python3 automation.py esbulk --dir {{ workload_args.output_path }} --index {{ es_index }}; - echo "Starting with Snappy upload"; - python3 automation.py upload --dir {{ workload_args.output_path }} --server {{ snappy_server }} --user {{ snappy_user }} --password {{ snappy_password }}; - echo "Process finished"; + echo "Uploading to ES..."; + python automation.py esbulk --dir {{ workload_args.output_path }} --index {{ elasticsearch.index_name }}; + echo "Uploading RAW files..."; + python automation.py upload --dir {{ workload_args.output_path }} --server {{ snappy.url }} --user {{ snappy.user }} --password {{ snappy.password }}; volumeMounts: - mountPath: /tmp/results name: results From 9783d78f8d09d2f1e08155e8b46c4073c468a30b Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Wed, 13 Oct 2021 11:43:54 +0200 Subject: [PATCH 167/249] Fixing api_load tests Signed-off-by: Vicente Zepeda Mas --- e2e/019-api_load.bats | 2 +- e2e/api_load/api_load.yaml | 19 +++++++++++++++++-- roles/api_load/templates/api_load.yml | 8 ++++---- tests/common.sh | 4 ++-- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/e2e/019-api_load.bats b/e2e/019-api_load.bats index 0c9790a57..6dd4a602b 100755 --- a/e2e/019-api_load.bats +++ b/e2e/019-api_load.bats @@ -10,7 +10,7 @@ load helpers.bash CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 120 } setup_file() { diff --git a/e2e/api_load/api_load.yaml b/e2e/api_load/api_load.yaml index 626508f59..f3a6ff9b4 100644 --- a/e2e/api_load/api_load.yaml +++ b/e2e/api_load/api_load.yaml @@ -5,12 +5,27 @@ metadata: name: api-load namespace: benchmark-operator spec: + uuid: "bb4ada0d-ff70-41e0-acc2-cd152ce4ac85" elasticsearch: url: ${ES_SERVER} index_name: "api-load" - metadata: - collection: true + snappy: + url: SNAPPY_SERVER + user: realuser + password: realpassword workload: name: api_load args: override: version + gateway_url: http://localhost:8080 + ocm_token: notARealToken + duration: 1 + rate: 1/s + output_path: /tmp/results + aws_access_key: empty + aws_access_secret: empty + aws_account_id: empty + cooldown: 10 + sleep: 5 + test_list: + self-access-token: diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index a30949ecb..5dd393e49 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -57,13 +57,13 @@ spec: command: ["/bin/sh", "-c"] args: - > +{% if workload_args.override is defined %} + ocm-load-test {{ workload_args.override }}; +{% else %} echo "Setting redis status"; redis-cli -h {{ bo.resources[0].status.podIP }} SET "{{ uuid }}-status" "running"; {% for key, value in workload_args.test_list.items() %} ocm-load-test -{% if workload_args.override is defined %} - {{ workload_args.override }} -{% else %} --test-id={{ uuid }} --gateway-url {{ workload_args.gateway_url }} --ocm-token={{ workload_args.ocm_token }} @@ -76,8 +76,8 @@ spec: --aws-account-id {{ workload_args.aws_account_id }}; echo "Cooldown for {{ workload_args.cooldown }}"; sleep {{ workload_args.cooldown }}; -{% endif %} {% endfor %} +{% endif %} redis-cli -h {{ bo.resources[0].status.podIP }} SET "{{ uuid }}-status" "ready"; volumeMounts: - mountPath: /tmp/results diff --git a/tests/common.sh b/tests/common.sh index 302e3d144..3843df9d5 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -188,13 +188,13 @@ function get_benchmark_state() { function wait_for_benchmark() { benchmark_name=$1 desired_state=$2 - until [[ $(get_benchmark_state $benchmark_name) == "$desired_state" ]]; do + until [[ $(get_benchmark_state $benchmark_name) == "$desired_state" ]]; do if [[ $(get_benchmark_state $benchmark_name) == "Failed" ]]; then echo "Benchmark $benchmark_name failed" return 1 fi sleep 5 - done + done } function check_benchmark_for_desired_state(){ From 1797d70532924b606584d679046f115281d61d33 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Wed, 13 Oct 2021 17:00:12 +0200 Subject: [PATCH 168/249] Fixing api_load test Signed-off-by: Vicente Zepeda Mas --- e2e/api_load/api_load.yaml | 2 +- roles/api_load/templates/api_load.yml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/e2e/api_load/api_load.yaml b/e2e/api_load/api_load.yaml index f3a6ff9b4..01a1bce6c 100644 --- a/e2e/api_load/api_load.yaml +++ b/e2e/api_load/api_load.yaml @@ -5,7 +5,7 @@ metadata: name: api-load namespace: benchmark-operator spec: - uuid: "bb4ada0d-ff70-41e0-acc2-cd152ce4ac85" + uuid: "" elasticsearch: url: ${ES_SERVER} index_name: "api-load" diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index 5dd393e49..261c139be 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -107,10 +107,14 @@ spec: echo "Testing readiness"; status=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "{{ uuid }}-status"`; done; +{% if workload_args.override is defined %} + echo "succeeded" +{% else %} echo "Uploading to ES..."; python automation.py esbulk --dir {{ workload_args.output_path }} --index {{ elasticsearch.index_name }}; echo "Uploading RAW files..."; python automation.py upload --dir {{ workload_args.output_path }} --server {{ snappy.url }} --user {{ snappy.user }} --password {{ snappy.password }}; +{% endif %} volumeMounts: - mountPath: /tmp/results name: results From e865bd7c8db3b352502218e617fbdd062c0985f2 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Wed, 27 Oct 2021 13:45:15 +0200 Subject: [PATCH 169/249] Adding documentation for api load scenario Signed-off-by: Vicente Zepeda Mas --- docs/api_load.md | 98 ++++++++++++++++++++++++++++++++++++++ e2e/api_load/api_load.yaml | 1 - 2 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 docs/api_load.md diff --git a/docs/api_load.md b/docs/api_load.md new file mode 100644 index 000000000..2c85bf9c5 --- /dev/null +++ b/docs/api_load.md @@ -0,0 +1,98 @@ +# API Load + +## What does it do? + +The API Load workload will execute a load test using [OCM API Load](https://github.com/cloud-bulldozer/ocm-api-load) +After running it will perform an ES indexig for all the request that have been generated and upload all the files to a snappy server. + +## Required sections + +`elasticsearch` section, it is needed to index all the documents that are used for results. + +`snappy` section, it is needed to upload the raw files in case we need them for further analysis. + +## Variables + +### Required variables + +`test_list` a list of the test that are goping to be run + +`gateway_url` url for openshift API + +`ocm_token` authorization token for API + +`duration` default duration of each the attack + +`rate` default rate of each attack + +`output_path` path were to write the results + +`aws_access_key` access key for AWS auhtentication + +`aws_access_secret` access secret for AWS auhtentication + +`aws_account_id` 12 digit account number for AWS auhtentication + +`cooldown` time in seconds, wait before the next attack occurs + +`sleep` time in seconds, checks redis status each seconds + +### Optional variables + +Each test can be provided with it's own configurations + +`duration` default duration of each the attack + +`rate` default rate of each attack + +### Example CR + +Your resource file with test list and custom configuraiton for some attacks: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: api-load + namespace: benchmark-operator +spec: + elasticsearch: + url: http://elastic.search.server + index_name: "api-load" + snappy: + url: http://snappy.files.server + user: user + password: password + workload: + name: api_load + args: + gateway_url: https://api.openshift.com + ocm_token: realTokenHere + duration: 1 + rate: 5/s + output_path: /tmp/results + aws_access_key: realKey + aws_access_secret: realSecret + aws_account_id: 12DigitAccountNumber + cooldown: 10 + sleep: 300 + test_list: + self-access-token: + rate: "1/s" + duration: 1 + list-subscriptions: + duration: 1 + access-review: + rate: "7/s" + register-new-cluster: + register-existing-cluster: + create-cluster: + list-clusters: + get-current-account: + quota-cost: + resource-review: + cluster-authorizations: + self-terms-review: + certificates: + +``` diff --git a/e2e/api_load/api_load.yaml b/e2e/api_load/api_load.yaml index 01a1bce6c..48661a35a 100644 --- a/e2e/api_load/api_load.yaml +++ b/e2e/api_load/api_load.yaml @@ -5,7 +5,6 @@ metadata: name: api-load namespace: benchmark-operator spec: - uuid: "" elasticsearch: url: ${ES_SERVER} index_name: "api-load" From 399e7e4b65264444f035e5e220f5f367590c2c3f Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Thu, 28 Oct 2021 18:40:13 +0200 Subject: [PATCH 170/249] Setting snappy section as optional Setting sleep, cooldown and output_path as optional variables Updating documentation Signed-off-by: Vicente Zepeda Mas --- docs/api_load.md | 22 ++++++++++++++-------- roles/api_load/templates/api_load.yml | 14 ++++++++------ 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/docs/api_load.md b/docs/api_load.md index 2c85bf9c5..9d6412230 100644 --- a/docs/api_load.md +++ b/docs/api_load.md @@ -3,13 +3,17 @@ ## What does it do? The API Load workload will execute a load test using [OCM API Load](https://github.com/cloud-bulldozer/ocm-api-load) -After running it will perform an ES indexig for all the request that have been generated and upload all the files to a snappy server. +After running it will perform an ES indexing for all the requests that have been generated and upload all the files to a snappy server. -## Required sections +## Sections + +### Required sections `elasticsearch` section, it is needed to index all the documents that are used for results. -`snappy` section, it is needed to upload the raw files in case we need them for further analysis. +### Optional sections + +[`snappy`](https://github.com/cloud-bulldozer/snappy-data-server) section, it is needed to upload the raw files in case we need them for further analysis. ## Variables @@ -25,19 +29,21 @@ After running it will perform an ES indexig for all the request that have been g `rate` default rate of each attack -`output_path` path were to write the results - `aws_access_key` access key for AWS auhtentication `aws_access_secret` access secret for AWS auhtentication `aws_account_id` 12 digit account number for AWS auhtentication -`cooldown` time in seconds, wait before the next attack occurs +### Optional variables -`sleep` time in seconds, checks redis status each seconds +`cooldown` time in seconds, wait before the next attack occurs, default 60 seconds -### Optional variables +`output_path` path were to write the results, default `/tmp/results` + +`override` used to run a specific sub-command of `ocm-api-load`, like `version` or `help` + +`sleep` time in seconds, checks redis status each seconds, default 360 seconds Each test can be provided with it's own configurations diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index 261c139be..6fc3d4b01 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -69,13 +69,13 @@ spec: --ocm-token={{ workload_args.ocm_token }} --duration={{ value.duration | default(workload_args.duration) }} --rate={{ value.rate | default(workload_args.rate) }} - --output-path={{ workload_args.output_path }} + --output-path={{ workload_args.output_path | default('/tmp/results') }} --test-names {{ key }} --aws-access-key {{ workload_args.aws_access_key }} --aws-access-secret {{ workload_args.aws_access_secret }} --aws-account-id {{ workload_args.aws_account_id }}; - echo "Cooldown for {{ workload_args.cooldown }}"; - sleep {{ workload_args.cooldown }}; + echo "Cooldown for {{ workload_args.cooldown | default(60) }}"; + sleep {{ workload_args.cooldown | default(60) }}; {% endfor %} {% endif %} redis-cli -h {{ bo.resources[0].status.podIP }} SET "{{ uuid }}-status" "ready"; @@ -103,7 +103,7 @@ spec: echo "Getting redis status"; status=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "{{ uuid }}-status"`; while [ $status != "ready" ]; do - sleep {{ workload_args.sleep }}; + sleep {{ workload_args.sleep | default(360) }}; echo "Testing readiness"; status=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "{{ uuid }}-status"`; done; @@ -111,9 +111,11 @@ spec: echo "succeeded" {% else %} echo "Uploading to ES..."; - python automation.py esbulk --dir {{ workload_args.output_path }} --index {{ elasticsearch.index_name }}; + python automation.py esbulk --dir {{ workload_args.output_path | default('/tmp/results') }} --index {{ elasticsearch.index_name }}; +{% if snappy.url %} echo "Uploading RAW files..."; - python automation.py upload --dir {{ workload_args.output_path }} --server {{ snappy.url }} --user {{ snappy.user }} --password {{ snappy.password }}; + python automation.py upload --dir {{ workload_args.output_path | default('/tmp/results') }} --server {{ snappy.url }} --user {{ snappy.user }} --password {{ snappy.password }}; +{% endif %} {% endif %} volumeMounts: - mountPath: /tmp/results From c8e5ca99ce8f6db5e0a861726f846a07e4d00149 Mon Sep 17 00:00:00 2001 From: Joe Talerico Date: Fri, 29 Oct 2021 12:38:06 -0400 Subject: [PATCH 171/249] Updating active members to contact Adding Sebastian --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 959f12668..e48ca12fc 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ a performance baseline of Kubernetes cluster on your provider. ## Installation (Default) -The easiest way to install the operator is through the operator-sdk methods provided in the `Makefile`. +The easiest way to install the operator is through the operator-sdk methods provided in the `Makefile`. ```bash git clone https://github.com/cloud-bulldozer/benchmark-operator @@ -21,9 +21,9 @@ cd benchmark-operator make image-build image-push deploy IMG=$YOUR_IMAGE ``` -> Note: building the image requires podman +> Note: building the image requires podman -## Installation (Helm) +## Installation (Helm) Installing the benchmark-operator via Helm can be done with the following commands. This requires your machine to have Helm installed. [Install Helm](https://helm.sh/docs/intro/install/) @@ -35,8 +35,8 @@ your machine to have Helm installed. [Install Helm](https://helm.sh/docs/intro/i ```bash git clone https://github.com/cloud-bulldozer/benchmark-operator cd benchmark-operator/charts/benchmark-operator -kubectl create namespace benchmark-operator -oc adm policy -n benchmark-operator add-scc-to-user privileged -z benchmark-operator # Openshift Only +kubectl create namespace benchmark-operator +oc adm policy -n benchmark-operator add-scc-to-user privileged -z benchmark-operator # Openshift Only helm install benchmark-operator . -n benchmark-operator --create-namespace ``` @@ -116,7 +116,7 @@ spec: Workloads that are triggered through [benchmark-wrapper](https://github.com/cloud-bulldozer/benchmark-wrapper) can optionally pass the debug flag through the workload CR. -NOTE: This is not a required arguement. If omitted it will default to the default logging level of +NOTE: This is not a required arguement. If omitted it will default to the default logging level of the benchmark-wrapper For Example: @@ -180,6 +180,6 @@ spec: [Cache dropping](docs/cache_dropping.md) ## Community -Key Members(slack_usernames): ravi, mohit, dry923, rsevilla or rook +Key Members(slack_usernames): sejug, mohit, dry923, rsevilla or rook * [**#sig-scalability on Kubernetes Slack**](https://kubernetes.slack.com) * [**#forum-perfscale on CoreOS Slack**](https://coreos.slack.com) From 7df4f2b17c2623b8e1d6468b67370f56e8365602 Mon Sep 17 00:00:00 2001 From: ebattat <73884315+ebattat@users.noreply.github.com> Date: Wed, 10 Nov 2021 13:21:07 +0200 Subject: [PATCH 172/249] add upgrade pip for CNV templates (#707) Only impacts CNV workload templates. --- roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 | 1 + roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 | 1 + roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 | 1 + 3 files changed, 3 insertions(+) diff --git a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 index 9e837ee79..62c07e3bd 100644 --- a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 @@ -10,6 +10,7 @@ dnf install -y ethtool; ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; {% endif %} dnf install -y epel-release git python3-pip; +pip3 install --upgrade pip; pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper; export db_type={{workload_args.db_type}}; export timed_test={{workload_args.timed_test}}; diff --git a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 index 938415535..d5b29c65b 100644 --- a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 @@ -10,6 +10,7 @@ dnf install -y ethtool; ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; {% endif %} dnf install -y epel-release git python3-pip; +pip3 install --upgrade pip; pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper; export db_type={{workload_args.db_type}}; export timed_test={{workload_args.timed_test}}; diff --git a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 index fa7aea661..d2a0d7b32 100644 --- a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 @@ -10,6 +10,7 @@ dnf install -y ethtool; ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; {% endif %} dnf install -y epel-release git python3-pip; +pip3 install --upgrade pip; pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper; export db_type={{workload_args.db_type}}; export timed_test={{workload_args.timed_test}}; From 186c8a259ade9de1d50a9a01a18c5fef47795fbb Mon Sep 17 00:00:00 2001 From: dry923 Date: Wed, 17 Nov 2021 10:35:56 -0500 Subject: [PATCH 173/249] Fix indentation on image pull that was causing the pull policy to be ignored --- roles/image_pull/templates/image_pull.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/image_pull/templates/image_pull.yml b/roles/image_pull/templates/image_pull.yml index 0ce495890..0cc738231 100644 --- a/roles/image_pull/templates/image_pull.yml +++ b/roles/image_pull/templates/image_pull.yml @@ -105,6 +105,6 @@ spec: --retries {{ workload_args.retries | default(0) | int }} --image "{{ my_image }}"; {% endfor %} - imagePullPolicy: Always + imagePullPolicy: Always restartPolicy: Never {% include "metadata.yml.j2" %} From 7c3d45cd425d348fb30c7124bfaf0fcf44db1fe1 Mon Sep 17 00:00:00 2001 From: dry923 Date: Wed, 17 Nov 2021 10:43:44 -0500 Subject: [PATCH 174/249] Also fix api_load for the same error --- roles/api_load/templates/api_load.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index 6fc3d4b01..f0dfdc97d 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -84,6 +84,7 @@ spec: name: results - name: data-collector image: {{ workload_args.image | default('quay.io/cloud-bulldozer/ocm-api-load:latest') }} + imagePullPolicy: Always env: - name: my_node_name valueFrom: @@ -126,6 +127,5 @@ spec: path: /tmp/results capacity: 10Gi type: DiskOrCreate - imagePullPolicy: Always restartPolicy: Never {% include "metadata.yml.j2" %} From 9c1ce52dbc493a480f28d75d19324fadc1d8b42c Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Wed, 17 Nov 2021 10:09:46 +0100 Subject: [PATCH 175/249] Standardizes using latest tag for images Signed-off-by: Vicente Zepeda Mas --- roles/fs-drift/tasks/main.yml | 2 +- roles/fs-drift/templates/workload_job.yml.j2 | 2 +- roles/hammerdb/templates/db_mariadb_workload.yml.j2 | 2 +- roles/hammerdb/templates/db_mssql_workload.yml.j2 | 2 +- roles/hammerdb/templates/db_postgres_workload.yml.j2 | 2 +- roles/smallfile/tasks/main.yml | 2 +- roles/smallfile/templates/workload_job.yml.j2 | 12 ++++++------ 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/roles/fs-drift/tasks/main.yml b/roles/fs-drift/tasks/main.yml index cded37064..b12dad3f9 100644 --- a/roles/fs-drift/tasks/main.yml +++ b/roles/fs-drift/tasks/main.yml @@ -59,7 +59,7 @@ spec: containers: - name: publisher-container - image: quay.io/cloud-bulldozer/fs-drift:master + image: quay.io/cloud-bulldozer/fs-drift:latest tty: true command: ["/bin/sh", "-c"] workingDir: /root/fs-drift-master/ diff --git a/roles/fs-drift/templates/workload_job.yml.j2 b/roles/fs-drift/templates/workload_job.yml.j2 index 406b7a4d5..71d30b9ad 100644 --- a/roles/fs-drift/templates/workload_job.yml.j2 +++ b/roles/fs-drift/templates/workload_job.yml.j2 @@ -41,7 +41,7 @@ spec: securityContext: privileged: true {% endif %} - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/fs-drift:master') }} + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/fs-drift:latest') }} # example of what to do when debugging image # image: quay.io/bengland2/fs-drift:latest imagePullPolicy: Always diff --git a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 index 3556a204d..f4e8b41ac 100644 --- a/roles/hammerdb/templates/db_mariadb_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 @@ -41,7 +41,7 @@ spec: cpu: {{ workload_args.limits_cpu }} memory: {{ workload_args.limits_memory }} {% endif %} - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:master') }} + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:latest') }} imagePullPolicy: Always env: - name: uuid diff --git a/roles/hammerdb/templates/db_mssql_workload.yml.j2 b/roles/hammerdb/templates/db_mssql_workload.yml.j2 index 617becdde..1c9b011fa 100644 --- a/roles/hammerdb/templates/db_mssql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mssql_workload.yml.j2 @@ -41,7 +41,7 @@ spec: cpu: {{ workload_args.limits_cpu }} memory: {{ workload_args.limits_memory }} {% endif %} - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:master') }} + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:latest') }} imagePullPolicy: Always env: - name: uuid diff --git a/roles/hammerdb/templates/db_postgres_workload.yml.j2 b/roles/hammerdb/templates/db_postgres_workload.yml.j2 index 31d6758fb..85941db7d 100644 --- a/roles/hammerdb/templates/db_postgres_workload.yml.j2 +++ b/roles/hammerdb/templates/db_postgres_workload.yml.j2 @@ -41,7 +41,7 @@ spec: cpu: {{ workload_args.limits_cpu }} memory: {{ workload_args.limits_memory }} {% endif %} - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:master') }} + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:latest') }} imagePullPolicy: Always env: - name: uuid diff --git a/roles/smallfile/tasks/main.yml b/roles/smallfile/tasks/main.yml index 5fba6ada7..801a92afb 100644 --- a/roles/smallfile/tasks/main.yml +++ b/roles/smallfile/tasks/main.yml @@ -58,7 +58,7 @@ spec: containers: - name: publisher-container - image: "{{ workload_args.image | default('quay.io/cloud-bulldozer/smallfile:master') }}" + image: "{{ workload_args.image | default('quay.io/cloud-bulldozer/smallfile:latest') }}" tty: true command: ["/bin/sh", "-c"] workingDir: /root/smallfile-master/ diff --git a/roles/smallfile/templates/workload_job.yml.j2 b/roles/smallfile/templates/workload_job.yml.j2 index 21e82fea7..9d0e198f6 100644 --- a/roles/smallfile/templates/workload_job.yml.j2 +++ b/roles/smallfile/templates/workload_job.yml.j2 @@ -41,7 +41,7 @@ spec: securityContext: privileged: true {% endif %} - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/smallfile:master') }} + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/smallfile:latest') }} imagePullPolicy: Always command: ["/bin/sh", "-c"] workingDir: /opt/snafu @@ -103,11 +103,11 @@ spec: {% if workload_args.samples is defined %} --samples {{ workload_args.samples }} {% endif %} - --tool smallfile - --operations $arr - --top {{ smallfile_path }}/smallfile_test_data - --dir /var/tmp/RESULTS - --yaml-input-file /tmp/smallfile/smallfilejob + --tool smallfile + --operations $arr + --top {{ smallfile_path }}/smallfile_test_data + --dir /var/tmp/RESULTS + --yaml-input-file /tmp/smallfile/smallfilejob {% if workload_args.debug is defined and workload_args.debug %} -v {% endif %} From a5bf5318ae29434809f89699c591c74f430650c6 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Fri, 19 Nov 2021 11:59:59 +0100 Subject: [PATCH 176/249] Adding parametrization to fs_drift image Signed-off-by: Vicente Zepeda Mas --- roles/fs-drift/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/fs-drift/tasks/main.yml b/roles/fs-drift/tasks/main.yml index b12dad3f9..dfb80e06e 100644 --- a/roles/fs-drift/tasks/main.yml +++ b/roles/fs-drift/tasks/main.yml @@ -59,7 +59,7 @@ spec: containers: - name: publisher-container - image: quay.io/cloud-bulldozer/fs-drift:latest + image: "{{ workload_args.image | default('quay.io/cloud-bulldozer/fs-drift:latest') }}" tty: true command: ["/bin/sh", "-c"] workingDir: /root/fs-drift-master/ From b942b0a5c26a9e99dd791b01c04db84bc1f4570c Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Fri, 19 Nov 2021 14:15:30 +0100 Subject: [PATCH 177/249] Use arg override in image for pgbench Signed-off-by: Vicente Zepeda Mas --- roles/pgbench/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/pgbench/defaults/main.yml b/roles/pgbench/defaults/main.yml index ef4224582..5625a5c03 100644 --- a/roles/pgbench/defaults/main.yml +++ b/roles/pgbench/defaults/main.yml @@ -1,6 +1,6 @@ --- db_port: 5432 -pgb_base_image: "quay.io/cloud-bulldozer/pgbench:latest" +pgb_base_image: "{{ workload_args.image | default('quay.io/cloud-bulldozer/pgbench:latest') }}" timeout_min: 300 num_databases_pattern: "{{ workload_args.num_databases_pattern|default('all') }}" run_time: "{{ workload_args.run_time|default('') }}" From 6e7cc6c5d7a73cf1a0df1d74d2dce11278a0ddd3 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Fri, 19 Nov 2021 16:06:31 +0100 Subject: [PATCH 178/249] Fixing image override by args on iperf3 Signed-off-by: Vicente Zepeda Mas --- roles/iperf3/templates/client_store.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/iperf3/templates/client_store.yml.j2 b/roles/iperf3/templates/client_store.yml.j2 index f856b2d33..d2df2e8b5 100644 --- a/roles/iperf3/templates/client_store.yml.j2 +++ b/roles/iperf3/templates/client_store.yml.j2 @@ -42,7 +42,7 @@ spec: {% endif %} containers: - name: benchmark - image: "quay.io/cloud-bulldozer/iperf3:latest" + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/iperf3:latest') }} imagePullPolicy: Always command: ["/bin/sh", "-c"] args: From 3bb263c2df34422a6743ed12f69c883dad687c19 Mon Sep 17 00:00:00 2001 From: Marko Karg <31761192+mkarg75@users.noreply.github.com> Date: Wed, 24 Nov 2021 17:14:52 +0100 Subject: [PATCH 179/249] fixing the runtime parameter which needs to be duration (#714) Co-authored-by: Marko Karg --- roles/oslat/files/oslat.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/oslat/files/oslat.sh b/roles/oslat/files/oslat.sh index 53cc33300..3598d5f77 100755 --- a/roles/oslat/files/oslat.sh +++ b/roles/oslat/files/oslat.sh @@ -78,9 +78,9 @@ if [ "${USE_TASKSET}" == "true" ]; then prefix_cmd="taskset --cpu-list ${cyccore}" fi -echo "cmd to run: oslat --runtime ${RUNTIME} --rtprio ${RTPRIO} --cpu-list ${cyccore} --cpu-main-thread ${cpus[0]}" +echo "cmd to run: oslat --duration ${RUNTIME} --rtprio ${RTPRIO} --cpu-list ${cyccore} --cpu-main-thread ${cpus[0]}" -oslat --runtime ${RUNTIME} --rtprio ${RTPRIO} --cpu-list ${cyccore} --cpu-main-thread ${cpus[0]} +oslat --duration ${RUNTIME} --rtprio ${RTPRIO} --cpu-list ${cyccore} --cpu-main-thread ${cpus[0]} if [ "${DISABLE_CPU_BALANCE}" == "true" ]; then enable_balance From 5bc5f2798c8f793c41650d3a7172e3daa77565de Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Mon, 29 Nov 2021 15:36:56 +0100 Subject: [PATCH 180/249] Set a default of 8h for api_load job timeout (#708) Signed-off-by: Vicente Zepeda Mas --- roles/api_load/templates/api_load.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index f0dfdc97d..988cd8601 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -6,7 +6,7 @@ metadata: namespace: '{{ operator_namespace }}' spec: backoffLimit: 0 - activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} + activeDeadlineSeconds: {{ workload_args.job_timeout | default(28800) }} parallelism: {{ workload_args.pod_count | default(1) | int }} template: metadata: From 95d281afdb6c2b2db707ebb76eaa65d0ba66888d Mon Sep 17 00:00:00 2001 From: jeniferh Date: Mon, 13 Sep 2021 18:15:53 -0400 Subject: [PATCH 181/249] Fio server_vm image updates Adjusting server_vm config so that any VM containerDisk image can be used since the workload is run from chroot into a fio container image (i.e. no custom VM images need to be maintained). Also adding a fallback VM option if no PVC/HostDisk is specified, use an emptyDisk (ephemeral storage). Adding nodeSelector support for VMs. --- config/samples/fio/vm-cr.yaml | 5 +- docs/fio_distributed.md | 12 ++-- .../templates/server_vm.yml.j2 | 59 ++++++++++--------- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/config/samples/fio/vm-cr.yaml b/config/samples/fio/vm-cr.yaml index aef25617a..2a39af1a9 100644 --- a/config/samples/fio/vm-cr.yaml +++ b/config/samples/fio/vm-cr.yaml @@ -33,7 +33,10 @@ spec: # vm_cores: 2 # Memory that will be available inside the VM # Default: 5G - # vm_cores: 10G + # vm_memory: 10G + # VM bus type: virtio, sata, or scsi + # Default: virtio + # vm_bus: virtio # test types, see fio documentation jobs: - write diff --git a/docs/fio_distributed.md b/docs/fio_distributed.md index 58e3314a8..7f5576e0b 100644 --- a/docs/fio_distributed.md +++ b/docs/fio_distributed.md @@ -108,12 +108,13 @@ The workload loops are nested as such from the CR options: > the specified annotations on the client pods' metadata. - **kind**: Can either be `pod` or `vm` to determine if the fio workload is run in a Pod or in a VM > Note: For VM workloads, you need to install Openshift Virtualization first -- **vm_image**: Whether to use a pre-defined VM image with pre-installed requirements. Necessary for disconnected installs. - > Note: You can use my fedora image here: quay.io/mulbc/fed-fio \ - > Note: Only applies when kind is set to `vm` +- **vm_image**: VM image to use for the test, benchmark dependencies are provided by a pod container image (Default pod image: quay.io/cloud-bulldozer/fio:latest). Container images should be pre-loaded for disconnected installs. Default: quay.io/kubevirt/fedora-container-disk-images:latest + > Note: Only applies when kind is set to `vm`, current assumption is the firewall is disabled in the VM (default for most cloud images) - **vm_cores**: The number of CPU cores that will be available inside the VM. Default=1 > Note: Only applies when kind is set to `vm` -- **vm_cores**: The amount of Memory that will be available inside the VM in Kubernetes format. Default=5G +- **vm_memory**: The amount of Memory that will be available inside the VM in Kubernetes format. Default=5G + > Note: Only applies when kind is set to `vm` +- **vm_bus**: The VM bus type (virtio, scsi, or sata). Default=virtio > Note: Only applies when kind is set to `vm` - **bs**: (list) blocksize values to use for I/O transactions > Note: We set the `direct=1` fio option in the jobfile configmap. In order to avoid errors, the `bs` values @@ -141,6 +142,9 @@ The workload loops are nested as such from the CR options: - **log_sample_rate**: (optional) Applied to fio options `log_avg_msec` (milliseconds) in the jobfile configmap; see `fio(1)` - **log_hist_msec** (optional) if set, enables histogram logging at this specified interval in milliseconds - **storageclass**: (optional) The K8S StorageClass to use for persistent volume claims (PVC) per server pod + > Note: Currently when the type is set to `vm` only Block-device storageclasses are supported +- **hostpath**: (optional) The local storage path to be used for hostPath (pods) or hostDisk (VMs) + > Note: When the type is set to `vm` make sure the hostDisk feature-gate is enabled and proper permission settings are applied to the path if applicable (i.e. chcon -Rt container_file_t ) - **pvcaccessmode**: (optional) The AccessMode to request with the persistent volume claim (PVC) for the fio server. Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce - **pvcvolumemode**: (optional) The volmeMode to request with the persistent volume claim (PVC) for the fio server. Can be one of Filesystem,Block Default: Filesystem > Note: It is recommended to change this to `Block` for VM tests diff --git a/roles/fio_distributed/templates/server_vm.yml.j2 b/roles/fio_distributed/templates/server_vm.yml.j2 index 1aa1fa0bc..7fe34ee96 100644 --- a/roles/fio_distributed/templates/server_vm.yml.j2 +++ b/roles/fio_distributed/templates/server_vm.yml.j2 @@ -22,14 +22,23 @@ spec: cores: {{ workload_args.vm_cores | default(1) }} devices: disks: - - name: registrydisk - - name: cloudinitdisk + - disk: + bus: {{ workload_args.vm_bus | default('virtio') }} + name: registrydisk + - disk: + bus: {{ workload_args.vm_bus | default('virtio') }} + name: cloudinitdisk {% if workload_args.storageclass is defined or workload_args.hostpath is defined %} - disk: - bus: virtio + bus: {{ workload_args.vm_bus | default('virtio') }} name: data-volume serial: data +{% else %} + - disk: + bus: {{ workload_args.vm_bus | default('virtio') }} + name: emptydisk + serial: data {% endif %} resources: requests: @@ -37,37 +46,23 @@ spec: volumes: - name: registrydisk containerDisk: - image: {{ workload_args.vm_image | default('kubevirt/fedora-cloud-container-disk-demo:latest') }} + image: {{ workload_args.vm_image | default('quay.io/kubevirt/fedora-container-disk-images:latest') }} - name: cloudinitdisk cloudInitNoCloud: userData: |- #cloud-config - password: fedora + password: ripsaw chpasswd: { expire: False } bootcmd: -{% if workload_args.storageclass is defined - or workload_args.hostpath is defined %} - "mkdir -p {{ fio_path }} || true" - - mkfs.ext4 /dev/disk/by-id/virtio-data - - "mount /dev/disk/by-id/virtio-data {{ fio_path }}" -{% endif %} + - "[ -e /dev/disk/by-id/*data ] && disk=$(shopt -s nullglob; basename /dev/disk/by-id/*data) && mkfs.ext4 /dev/disk/by-id/$disk && mount /dev/disk/by-id/$disk {{ fio_path }}" runcmd: -{% if workload_args.vm_image is undefined %} - - dnf install -y fio wget curl python3 python3-pip -{% endif %} - - cd /tmp -{% if workload_args.metadata is defined - and (workload_args.metadata.collection|default(false)) - and (workload_args.metadata.targeted|default(true)) %} - - export ES_SERVER={{ elasticsearch.url }} -{% if workload_args.vm_image is undefined %} - - wget https://raw.githubusercontent.com/cloud-bulldozer/bohica/master/stockpile-wrapper/stockpile-wrapper.py - - pip3 install elasticsearch-dsl openshift kubernetes redis -{% endif %} - # TODO: start stockpile-wrapper script - python3 stockpile-wrapper.py -s {{ elasticsearch.url }} -u {{ uuid }} -n "$my_node_name" -N "$my_pod_name" --redisip {{ bo.resources[0].status.podIP }} --redisport 6379 --force -{% endif %} - - fio --server + - dnf install -y podman + - "img=`podman create {{ workload_args.image | default('quay.io/cloud-bulldozer/fio:latest') }}`" + - fs=`podman mount $img` + - "mkdir -p $fs/{{ fio_path }} || true" + - "mount -o bind /{{ fio_path }} $fs/{{ fio_path }}" + - "chroot $fs bash -c 'cd /{{ fio_path }}; fio --server'" {% if workload_args.storageclass is defined %} - name: data-volume persistentVolumeClaim: @@ -75,7 +70,17 @@ spec: {% elif workload_args.hostpath is defined %} - name: data-volume hostDisk: - path: {{ workload_args.hostpath }} + path: "{{ workload_args.hostpath }}/fio-server-{{item | string}}-{{ trunc_uuid }}" capacity: {{ workload_args.storagesize | default("5Gi") }} type: DiskOrCreate +{% else %} + - name: emptydisk + emptyDisk: + capacity: {{ workload_args.storagesize | default("5Gi") }} {% endif %} +{% if workload_args.nodeselector is defined %} + nodeSelector: +{% for label, value in workload_args.nodeselector.items() %} + {{ label | replace ("_", "-" )}}: {{ value }} +{% endfor %} +{% endif %} From 8b2694a3c9ea9f29b079dda4c46cdee7b7731e62 Mon Sep 17 00:00:00 2001 From: "Joe Talerico (rook)" Date: Wed, 12 Jan 2022 16:10:42 -0500 Subject: [PATCH 182/249] Add missing deps to setup.cfg (#717) Missing requests and dateutil packages for the Ripsaw CLI in a fresh 3.10 venv. Fixes #716 --- cli/setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cli/setup.cfg b/cli/setup.cfg index 5e7fb573f..f223274b3 100644 --- a/cli/setup.cfg +++ b/cli/setup.cfg @@ -29,6 +29,8 @@ install_requires = PyYAML==5.4.1 elasticsearch==7.13.4 GitPython==3.1.18 + python-dateutil + requests python_requires = >=3.8 [options.extras_require] From 2c9ca7913a941c8e01a7ad31693eac4b2574ea34 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 13 Jan 2022 17:24:12 +0100 Subject: [PATCH 183/249] System-metrics doc typo Signed-off-by: Raul Sevilla --- docs/system-metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/system-metrics.md b/docs/system-metrics.md index 22df8f9db..5586ae2da 100644 --- a/docs/system-metrics.md +++ b/docs/system-metrics.md @@ -26,7 +26,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - enabled: true + collection: true prom_token: eyJhbGciOiJSUzI1NiIsImtpZCI6IlljTUxlUHBTY2hvUVJQYUlZWmV5MTE4d3VnRFpjUUh5MWZtdE9hdnlvNFUifQ.eyJpc3MiOiJrdWJlcnopeVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJvcGVuc2hpZnQtbW9uaXRvcmluZyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cy10b2tlbi12NGo3YyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjFkYTdkMTRkLWE2MTktNDZjYS1iZGRlLTMzOTYxOWYxMmM4MiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpvcGVuc2hpZnQtbW9uaXRvcmluZzpwcm9tZXRoZXVzLWs4cyJ9.PJp5pD_CjMG05vVLFdxUDRWGA8C71TNyRsUcHmpMlnZLQWBwxZSDZ-Uh3y6g1O-Yz3nopeCoZLB6lugxxalcT1DhiEC9yNK53Lr6HLqaz8nWUbRPbex0913KcuSsnpeRj7tzlwQ2K3WbtIeyyHpG5vAeff07LDvHUcPsc3B_dyetGnInClBHFVEJRES6f5DbIUidtXZEfYKANJNcssly0qZMZicwvM4a_pRp6ctGB-zzR6Ac4lh3b1JLfl_5TLGuuoYEOAeJPVUF4TjemsNNJ5BlycEkVI377LKNdHf83wua5pn3ItJtKE5gdrG833203p-y0pj-UDJj2bAv0cjUQ metrics_profile: https://raw.githubusercontent.com/cloud-bulldozer/benchmark-operator/master/roles/kube-burner/files/metrics-aggregated.yaml elasticsearch: From 4e3f7eb5c9ee8ed24337fe574fadb8a6772efb5f Mon Sep 17 00:00:00 2001 From: "Joe Talerico (rook)" Date: Wed, 19 Jan 2022 11:23:51 -0500 Subject: [PATCH 184/249] Add requests to manifest for kustomize (#720) Pod request is set to the limit value if unspecified, which can block deployment on small clusters. --- config/manager/manager.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index e87d75501..07bd665c6 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -72,6 +72,8 @@ spec: ports: - containerPort: 6379 resources: + requests: + cpu: "100m" limits: cpu: "2.0" volumeMounts: From 0e91cb79033461adb9639d2fb43498ca065c3d28 Mon Sep 17 00:00:00 2001 From: Marko Karg Date: Fri, 21 Jan 2022 12:07:43 +0100 Subject: [PATCH 185/249] shortening job names for cyclictest and oslat to prevent too long names when deployed via airflow --- roles/cyclictest/tasks/main.yml | 2 +- roles/cyclictest/templates/cyclictestjob.yaml | 2 +- roles/oslat/tasks/main.yml | 2 +- roles/oslat/templates/oslatjob.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/cyclictest/tasks/main.yml b/roles/cyclictest/tasks/main.yml index cbd25ddfb..881967181 100644 --- a/roles/cyclictest/tasks/main.yml +++ b/roles/cyclictest/tasks/main.yml @@ -7,7 +7,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' + name: 'cyclictest-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: cyclictest.sh: "{{ lookup('file', 'cyclictest.sh') }}" diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index 30cc68455..c082cb4a5 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + name: "cyclictest-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: template: diff --git a/roles/oslat/tasks/main.yml b/roles/oslat/tasks/main.yml index 4a8c0df6a..a043c7b97 100644 --- a/roles/oslat/tasks/main.yml +++ b/roles/oslat/tasks/main.yml @@ -5,7 +5,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' + name: 'oslat-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: oslat.sh: "{{ lookup('file', 'oslat.sh') }}" diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index 01b74359c..3e1656dae 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + name: "oslat-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: backoffLimit: 0 From 7deb23de2ded52b351010e6a304067529b88d69a Mon Sep 17 00:00:00 2001 From: Marko Karg Date: Fri, 21 Jan 2022 13:39:08 +0100 Subject: [PATCH 186/249] fixing configmap names --- roles/cyclictest/templates/cyclictestjob.yaml | 2 +- roles/oslat/templates/oslatjob.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index c082cb4a5..9c337dfeb 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -90,7 +90,7 @@ spec: path: /dev/cpu_dma_latency - name: cyclictest-volume configMap: - name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + name: "cyclictest-{{ trunc_uuid }}" defaultMode: 0555 restartPolicy: Never serviceAccountName: benchmark-operator diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index 3e1656dae..c679bfb18 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -91,7 +91,7 @@ spec: path: /dev/cpu_dma_latency - name: oslat-volume configMap: - name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + name: "oslat-{{ trunc_uuid }}" defaultMode: 0555 restartPolicy: Never serviceAccountName: benchmark-operator From f71644b6e265abe76952e55c6b05c6c7cf5a6b14 Mon Sep 17 00:00:00 2001 From: "Joe Talerico (rook)" Date: Tue, 25 Jan 2022 12:22:57 -0500 Subject: [PATCH 187/249] Helm Chart missing label (#724) The helm chart was missing a label, which was causing issues with deploy UPerf. Fixing the template. Signed-off-by: Joe Talerico --- charts/benchmark-operator/templates/operator.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/benchmark-operator/templates/operator.yaml b/charts/benchmark-operator/templates/operator.yaml index 73f1b6641..514eeabd4 100644 --- a/charts/benchmark-operator/templates/operator.yaml +++ b/charts/benchmark-operator/templates/operator.yaml @@ -11,6 +11,7 @@ spec: template: metadata: labels: + control-plane: controller-manager name: {{ include "benchmark-operator.fullname" . }} spec: serviceAccountName: {{ include "benchmark-operator.fullname" . }} From dc375a5bc345f29c31cd92c3be9d6eebd55c56ea Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy Date: Tue, 18 Jan 2022 18:27:05 -0500 Subject: [PATCH 188/249] nodeport and loadbalancer type svc for uperf --- roles/uperf/tasks/start_client.yml | 10 ++- roles/uperf/tasks/start_server.yml | 16 +++- roles/uperf/templates/configmap.yml.j2 | 6 +- roles/uperf/templates/server.yml.j2 | 4 +- roles/uperf/templates/service.yml.j2 | 29 ++++++- roles/uperf/templates/service_metallb.yml.j2 | 90 ++++++++++++++++++++ roles/uperf/templates/workload.yml.j2 | 15 ++++ 7 files changed, 158 insertions(+), 12 deletions(-) create mode 100644 roles/uperf/templates/service_metallb.yml.j2 diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml index 2d1b30809..494fa5d7b 100644 --- a/roles/uperf/tasks/start_client.yml +++ b/roles/uperf/tasks/start_client.yml @@ -20,7 +20,10 @@ definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" vars: resource_item: "{{ server_pods.resources }}" - when: workload_args.serviceip|default(False) == False and server_pods.resources|length > 0 + when: + - ( workload_args.serviceip|default(False) == False and server_pods.resources|length > 0 ) or + ( workload_args.serviceip|default(False) == True and server_pods.resources|length > 0 and + workload_args.servicetype | default("clusterip") == "nodeport" ) # # Each server annotates a "node_idx". Each peer client will @@ -32,8 +35,9 @@ definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" vars: resource_item: "{{ serviceip.resources }}" - when: workload_args.serviceip|default(False) == True and serviceip.resources|length > 0 - + when: + - workload_args.serviceip|default(False) == True and serviceip.resources|length > 0 + - workload_args.servicetype | default("clusterip") != "nodeport" when: resource_kind == "pod" - block: diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml index 8036c3769..88feb2fa9 100644 --- a/roles/uperf/tasks/start_server.yml +++ b/roles/uperf/tasks/start_server.yml @@ -11,7 +11,21 @@ pod_sequence: "{{ pod_hi_idx|int +1 }}" node_sequence: "{{ node_hi_idx|int +1 }}" - when: workload_args.serviceip is defined and workload_args.serviceip + when: + - workload_args.serviceip is defined and workload_args.serviceip + - ( workload_args.servicetype | default("clusterip") == "clusterip" ) or + ( workload_args.servicetype | default("clusterip") == "nodeport" ) + + - name: Create metal LB service for server pods + k8s: + definition: "{{ lookup('template', 'service_metallb.yml.j2') | from_yaml }}" + vars: + pod_sequence: "{{ pod_hi_idx|int +1 }}" + node_sequence: "{{ node_hi_idx|int +1 }}" + + when: + - workload_args.serviceip is defined and workload_args.serviceip + - workload_args.servicetype | default("clusterip") == "metallb" - name: Start Server(s) - total = eligible nodes * density k8s: diff --git a/roles/uperf/templates/configmap.yml.j2 b/roles/uperf/templates/configmap.yml.j2 index f892a65e6..bc1f82562 100644 --- a/roles/uperf/templates/configmap.yml.j2 +++ b/roles/uperf/templates/configmap.yml.j2 @@ -22,7 +22,7 @@ data: {% if ( 'rr' == test ) %} - + @@ -35,7 +35,7 @@ data: {% if ( 'stream' == test or 'bidirec' == test ) %} - + @@ -48,7 +48,7 @@ data: {% if ( 'maerts' == test or 'bidirec' == test ) %} - + diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index f55feabb5..8bda30bd2 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -60,7 +60,7 @@ items: {% endif %} imagePullPolicy: Always command: ["/bin/sh","-c"] - args: ["uperf -s -v -P 20000"] + args: ["uperf -s -v -P 30000"] restartPolicy: OnFailure {% if workload_args.pin is sameas true %} nodeSelector: @@ -73,7 +73,7 @@ items: securityContext: sysctls: - name: net.ipv4.ip_local_port_range - value: 20000 20011 + value: 30000 30011 {% endif %} {% macro metadata() %}{% include "metadata.yml.j2" %}{% endmacro %} {{ metadata()|indent }} diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 2664a4872..8db9a6186 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -28,21 +28,44 @@ items: {% else %} app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} {% endif %} +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + type: NodePort ports: - name: uperf - port: 20000 - targetPort: 20000 + port: 30000 + targetPort: 30000 + nodePort: 30000 protocol: TCP -{% for num in range(20001,20012,1) %} +{% for num in range(30001,30012,1) %} - name: uperf-control-tcp-{{num}} port: {{num}} targetPort: {{num}} + nodePort: {{num}} protocol: TCP - name: uperf-control-udp-{{num}} port: {{num}} targetPort: {{num}} + nodePort: {{num}} protocol: UDP {% endfor %} +{% else %} + type: ClusterIP + ports: + - name: uperf + port: 30000 + targetPort: 30000 + protocol: TCP +{% for num in range(30001,30012,1) %} + - name: uperf-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: TCP + - name: uperf-control-udp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: UDP +{% endfor %} +{% endif %} {% endmacro %} {% for node_idx_item in range(node_sequence|int) %} {% for item in range(pod_sequence|int) %} diff --git a/roles/uperf/templates/service_metallb.yml.j2 b/roles/uperf/templates/service_metallb.yml.j2 new file mode 100644 index 000000000..e19b33c7b --- /dev/null +++ b/roles/uperf/templates/service_metallb.yml.j2 @@ -0,0 +1,90 @@ +--- +apiVersion: v1 +kind: List +metadata: {} +items: +{% macro job_template(item, node_idx_item='') %} + - kind: Service + apiVersion: v1 + metadata: + name: uperf-service-{{ item }}-{{ trunc_uuid }} + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + annotations: + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-service-{{ item }}-{{ trunc_uuid }} + spec: + selector: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf + port: 30000 + targetPort: 30000 + protocol: TCP +{% for num in range(30001,30012,1) %} + - name: uperf-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: TCP +{% endfor %} + - kind: Service + apiVersion: v1 + metadata: + name: uperf-service-{{ item }}-{{ trunc_uuid }}-udp + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}-udp + annotations: + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-service-{{ item }}-{{ trunc_uuid }} + spec: + selector: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf + port: 30000 + targetPort: 30000 + protocol: UDP +{% for num in range(30001,30012,1) %} + - name: uperf-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: UDP +{% endfor %} +{% endmacro %} +{% for node_idx_item in range(node_sequence|int) %} +{% for item in range(pod_sequence|int) %} +{{ job_template(item,node_idx_item) }} +{% endfor %} +{% endfor %} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index aa8b9325f..686e7197d 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -8,7 +8,13 @@ items: apiVersion: batch/v1 metadata: {% if workload_args.serviceip is sameas true %} +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + name: 'uperf-client-{{item.status.hostIP}}-{{ trunc_uuid }}' +{% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} + name: 'uperf-client-{{item.status.loadBalancer.ingress[0].ip}}-{{ trunc_uuid }}' +{% else %} name: 'uperf-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}' +{% endif %} {% else %} name: 'uperf-client-{{item.status.podIP}}-{{ trunc_uuid }}' {% endif %} @@ -100,7 +106,16 @@ items: args: {% if workload_args.serviceip is sameas true %} - "export serviceip=true; +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + export h={{item.status.hostIP}}; + export servicetype={{workload_args.servicetype}}; +{% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} + export h={{item.status.loadBalancer.ingress[0].ip}}; + export servicetype={{workload_args.servicetype}}; +{% else %} export h={{item.spec.clusterIP}}; + export servicetype={{workload_args.servicetype | default("clusterip")}}; +{% endif %} {% else %} {% if workload_args.multus.client is defined %} - "export multus_client={{workload_args.multus.client}}; From a6e94218022bb524236d37e00e1e2fc825121a81 Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy Date: Wed, 19 Jan 2022 09:20:19 -0500 Subject: [PATCH 189/249] updated uperf vm port --- roles/uperf/templates/server_vm.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/uperf/templates/server_vm.yml.j2 b/roles/uperf/templates/server_vm.yml.j2 index ad3f438f3..744a58103 100644 --- a/roles/uperf/templates/server_vm.yml.j2 +++ b/roles/uperf/templates/server_vm.yml.j2 @@ -73,6 +73,6 @@ spec: - dnf install -y uperf redis git - redis-cli -h {{ bo.resources[0].status.podIP }} setnx {{ trunc_uuid }} 0 - redis-cli -h {{ bo.resources[0].status.podIP }} incr {{ trunc_uuid }} - - uperf -s -v -P 20000 + - uperf -s -v -P 30000 name: cloudinitdisk status: {} From 94bdfb2562f2dc55c28d7a67ca3a823a14915a35 Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy Date: Mon, 24 Jan 2022 10:49:44 -0500 Subject: [PATCH 190/249] document update --- docs/uperf.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/docs/uperf.md b/docs/uperf.md index ec37a53c0..f10298a24 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -260,6 +260,56 @@ Once done creating/editing the resource file, you can run it by: # kubectl apply -f config/samples/uperf/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` +### Advanced Service types + +Benchmark operator now also supports different service types, it can create `NodePort` and `LoadBalancer` (only metallb) +type services along with the current default `ClusterIP` type. + +No pre-requisites needed for `NodePort` service, as long as the ports used by uperf(30000 to 30012) are allowed at the node level, +which is the cluster default. + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) +uperf-service-np NodePort 172.30.177.81 30000:31955/TCP,30001:31935/TCP,30002:31942/TCP,30001:31935/UDP,30002:31942/UDP +``` + +For `metallb` type, there are certain pre-requisites, +1. Installation of [MetalLB](https://metallb.universe.tf/) operator and CRD +2. Configuration of [BGP](https://github.com/metallb/metallb-operator#create-a-bgp-peer-object) +3. Configuration of [AddressPool](https://github.com/metallb/metallb-operator#create-an-address-pool-object) for lb service +4. Configuration of extenal router for BGP + +`metallb` type creates 2 services per benchmark CRD (for each protocol, tcp and udp) and they will share the external IP like below + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) +uperf-service-lb LoadBalancer 172.30.177.99 192.168.216.102 30000:30976/TCP,30001:30652/TCP,30002:30099/TCP +uperf-service-lb2 LoadBalancer 172.30.126.71 192.168.216.102 30001:31312/UDP,30002:30776/UDP +``` + +#### CR file inputs + +##### For NodePort +```yaml + ... + name: uperf + serviceip: true + servicetype: "nodeport" + ... +``` + +##### For MetalLB +`metallb` +```yaml + ... + name: uperf + serviceip: true + servicetype: "metallb" + metallb: + addresspool: "addresspool-l3" + service_etp: "Cluster" + ... +``` ## Running Uperf in VMs through kubevirt/cnv [Preview] Note: this is currently in preview mode. From 6fef741a566746460921518d334be89318d1aa2f Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy Date: Mon, 24 Jan 2022 10:53:48 -0500 Subject: [PATCH 191/249] readme correction --- docs/uperf.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/uperf.md b/docs/uperf.md index f10298a24..55c64c7d8 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -279,7 +279,7 @@ For `metallb` type, there are certain pre-requisites, 3. Configuration of [AddressPool](https://github.com/metallb/metallb-operator#create-an-address-pool-object) for lb service 4. Configuration of extenal router for BGP -`metallb` type creates 2 services per benchmark CRD (for each protocol, tcp and udp) and they will share the external IP like below +`metallb` type creates 2 services per benchmark CR (for each protocol, `tcp` and `udp`) and they will share the external IP like below ``` NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) @@ -307,7 +307,7 @@ uperf-service-lb2 LoadBalancer 172.30.126.71 192.168.216.102 30001:31312 servicetype: "metallb" metallb: addresspool: "addresspool-l3" - service_etp: "Cluster" + service_etp: "Cluster" # Either `Cluster` or `Local` ... ``` From e6fab66a9add1b1a079af84b9e1fcee44fe22102 Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy Date: Mon, 24 Jan 2022 10:59:43 -0500 Subject: [PATCH 192/249] test workload for nodeport --- .../valid_uperf_serviceip_nodeport.yaml | 45 +++++++++++++++++++ tests/test_uperf.sh | 1 + 2 files changed, 46 insertions(+) create mode 100644 tests/test_crs/valid_uperf_serviceip_nodeport.yaml diff --git a/tests/test_crs/valid_uperf_serviceip_nodeport.yaml b/tests/test_crs/valid_uperf_serviceip_nodeport.yaml new file mode 100644 index 000000000..dba9ad4b3 --- /dev/null +++ b/tests/test_crs/valid_uperf_serviceip_nodeport.yaml @@ -0,0 +1,45 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-serviceip-nodeport + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: true + servicetype: "nodeport" + pin: false + pin_server: "node-0" + pin_client: "node-1" + multus: + enabled: false + samples: 2 + pair: 1 + test_types: + - stream + - rr + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + - 2 + runtime: 2 + debug: true diff --git a/tests/test_uperf.sh b/tests/test_uperf.sh index 3621268fe..025d2499f 100755 --- a/tests/test_uperf.sh +++ b/tests/test_uperf.sh @@ -44,5 +44,6 @@ function functional_test_uperf { figlet $(basename $0) functional_test_uperf "Uperf without resources definition" tests/test_crs/valid_uperf.yaml functional_test_uperf "Uperf with ServiceIP" tests/test_crs/valid_uperf_serviceip.yaml +functional_test_uperf "Uperf with NodePort ServiceIP" tests/test_crs/valid_uperf_serviceip_nodeport.yaml functional_test_uperf "Uperf with resources definition and hostNetwork" tests/test_crs/valid_uperf_resources.yaml functional_test_uperf "Uperf with networkpolicy" tests/test_crs/valid_uperf_networkpolicy.yaml From cbc6a568e18d2fb2870ba10c5d83e671d1ed998f Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy Date: Mon, 24 Jan 2022 14:26:51 -0500 Subject: [PATCH 193/249] bats test case for nodeport --- e2e/013-uperf.bats | 9 ++++++ e2e/uperf/uperf_serviceip_nodeport.yaml | 39 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 e2e/uperf/uperf_serviceip_nodeport.yaml diff --git a/e2e/013-uperf.bats b/e2e/013-uperf.bats index b219ad6f1..706b3bc19 100755 --- a/e2e/013-uperf.bats +++ b/e2e/013-uperf.bats @@ -43,6 +43,15 @@ ES_INDEX=ripsaw-uperf-results check_es } +@test "uperf-serviceip-nodeport" { + CR=uperf/uperf_serviceip_nodeport.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + @test "uperf-hostnetwork" { CR=uperf/uperf_hostnetwork.yaml CR_NAME=$(get_benchmark_name ${CR}) diff --git a/e2e/uperf/uperf_serviceip_nodeport.yaml b/e2e/uperf/uperf_serviceip_nodeport.yaml new file mode 100644 index 000000000..b887c2d16 --- /dev/null +++ b/e2e/uperf/uperf_serviceip_nodeport.yaml @@ -0,0 +1,39 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-serviceip-nodeport + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: true + servicetype: "nodeport" + multus: + enabled: false + samples: 1 + pair: 1 + test_types: + - stream + protos: + - tcp + - udp + sizes: + - 512 + nthrs: + - 1 + runtime: 2 + debug: true From 466ca91c9cf1a799b1ff90170c20119019d90670 Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Tue, 8 Feb 2022 12:57:41 -0600 Subject: [PATCH 194/249] Update AppStream Repo for CentOS 8 CentOS 8 is now EOL, so moving to stream for now. Signed-off-by: Sai Sindhur Malleni --- image_resources/centos8-appstream.repo | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/image_resources/centos8-appstream.repo b/image_resources/centos8-appstream.repo index b35453861..c2067f100 100644 --- a/image_resources/centos8-appstream.repo +++ b/image_resources/centos8-appstream.repo @@ -1,17 +1,17 @@ [centos8-appstream-x86_64] name=CentOS-8-Appstream-x86_64 -baseurl=http://mirror.centos.org/centos/8/AppStream/x86_64/os/ +baseurl=http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/ enabled=0 gpgcheck=0 [centos8-appstream-aarch64] name=CentOS-8-Appstream-aarch64 -baseurl=http://mirror.centos.org/centos/8/AppStream/aarch64/os/ +baseurl=http://mirror.centos.org/centos/8-stream/AppStream/aarch64/os/ enabled=0 gpgcheck=0 [centos8-appstream-ppc64le] name=CentOS-8-Appstream-ppc64le -baseurl=http://mirror.centos.org/centos/8/AppStream/ppc64le//os/ +baseurl=http://mirror.centos.org/centos/8-stream/AppStream/ppc64le/os/ enabled=0 gpgcheck=0 From 0276849f71582ab6e5a6e5e4adb404c7873e27cb Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Mon, 7 Feb 2022 13:06:10 -0600 Subject: [PATCH 195/249] Add new CNI focussed kubelet tests 1. kubelet-density-cni: This creates deployments of node.js webservers fronted by services and another deployment of pods that run curl against the webserver service for readiness probe. This ensures that the curl pods only go into ready when they are able to reach the webserver srvice. kube-burner accordingly accounts for this delay in the podready metrics. 2. kubelet-density-cni-policy: This is similar to above workload except a deny-all policy is created in the namespace to begin with and later a newotk policy to allow http on 8080 is introduced when the pods/services are created. This also ensures that the curl pods' readiness probe only succeeds when the flows dictated by the network policy are applied. kube-burner podready metrics now account for propagation of this network policy. Signed-off-by: Sai Sindhur Malleni --- docs/kube-burner.md | 17 +++++ e2e/008-kube-burner.bats | 18 ++++++ .../node-density-cni-networkpolicy.yaml | 39 +++++++++++ e2e/kube-burner/node-density-cni.yaml | 39 +++++++++++ roles/kube-burner/files/allow-http.yml | 12 ++++ roles/kube-burner/files/curl-deployment.yml | 40 ++++++++++++ roles/kube-burner/files/deny-all.yml | 7 ++ .../files/webserver-deployment.yml | 30 +++++++++ roles/kube-burner/files/webserver-service.yml | 12 ++++ roles/kube-burner/tasks/main.yml | 34 ++++++++++ .../node-density-cni-networkpolicy.yml.j2 | 64 +++++++++++++++++++ .../templates/node-density-cni.yml.j2 | 47 ++++++++++++++ tests/test_kubeburner.sh | 2 +- 13 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 e2e/kube-burner/node-density-cni-networkpolicy.yaml create mode 100644 e2e/kube-burner/node-density-cni.yaml create mode 100644 roles/kube-burner/files/allow-http.yml create mode 100644 roles/kube-burner/files/curl-deployment.yml create mode 100644 roles/kube-burner/files/deny-all.yml create mode 100644 roles/kube-burner/files/webserver-deployment.yml create mode 100644 roles/kube-burner/files/webserver-service.yml create mode 100644 roles/kube-burner/templates/node-density-cni-networkpolicy.yml.j2 create mode 100644 roles/kube-burner/templates/node-density-cni.yml.j2 diff --git a/docs/kube-burner.md b/docs/kube-burner.md index 2a4ae3000..2ad49490d 100644 --- a/docs/kube-burner.md +++ b/docs/kube-burner.md @@ -49,6 +49,23 @@ Each iteration of this workload creates the following objects: - 1 deployment holding a client application for the previous database - 1 service pointing to the postgresl database +- **node-density-cni**. Creates a **single namespace with a number of applications equals to job_iterations**. This application consists on two deployments (a node.js webserver and a simple client that curls the webserver) and a service that is used by the client to reach the webserver. +Each iteration of this workload creates the following objects: + - 1 deployment holding a node.js webserver + - 1 deployment holding a client application for curling the webserver + - 1 service pointing to the webserver + + The Readiness Probe of the client pod depends on being able to reach the webserver so that the PodReady latencies collected by kube-burner reflect network connectivity. + +- **node-density-cni-policy**. Creates a **single namespace with a number of applications equals to job_iterations**. This application consists on two deployments (a node.js webserver and a simple client that curls the webserver) and a service that is used by the client to reach the webserver. +Each iteration of this workload creates the following objects: + - 1 deployment holding a node.js webserver + - 1 deployment holding a client application for curling the webserver + - 1 service pointing to the webserver + + A NetworkPolicy to deny all connections is created in the namspace first and then NetworkPolicies specifically applying the connection of each client-webserver pair are applied. The Readiness Probe of the client pod depends on being able to reach the webserver so that the PodReady latencies collected by kube-burner reflect network connectivity. + + - **max-namespaces**: This workload is a cluster limits focused test which creates maximum possible namespaces across the cluster. This is a namespaced workload, meaning that kube-burner **will create as many namespaces with these objects as the configured job_iterations**. - 1 deployment holding a postgresql database - 5 deployments consisting of a client application for the previous database diff --git a/e2e/008-kube-burner.bats b/e2e/008-kube-burner.bats index e5f7e7afc..021053251 100755 --- a/e2e/008-kube-burner.bats +++ b/e2e/008-kube-burner.bats @@ -34,6 +34,24 @@ ES_INDEX=ripsaw-kube-burner check_es } +@test "kube-burner-node-density-cni" { + CR=kube-burner/node-density-cni.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + +@test "kube-burner-node-density-cni-networkpolicy" { + CR=kube-burner/node-density-cni-networkpolicy.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 900 + check_es +} + @test "kube-burner-max-services" { CR=kube-burner/max-services.yaml CR_NAME=$(get_benchmark_name ${CR}) diff --git a/e2e/kube-burner/node-density-cni-networkpolicy.yaml b/e2e/kube-burner/node-density-cni-networkpolicy.yaml new file mode 100644 index 000000000..832076170 --- /dev/null +++ b/e2e/kube-burner/node-density-cni-networkpolicy.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-node-density-cni-networkpolicy + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: false + prometheus: + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + workload: node-density-cni-policy + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 2 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/node-density-cni.yaml b/e2e/kube-burner/node-density-cni.yaml new file mode 100644 index 000000000..147ee9e18 --- /dev/null +++ b/e2e/kube-burner/node-density-cni.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-node-density-cni + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: false + prometheus: + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + workload: node-density-cni + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 2 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/roles/kube-burner/files/allow-http.yml b/roles/kube-burner/files/allow-http.yml new file mode 100644 index 000000000..dcda8c641 --- /dev/null +++ b/roles/kube-burner/files/allow-http.yml @@ -0,0 +1,12 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-{{.Replica}}-{{.Iteration}} +spec: + podSelector: + matchLabels: + name: webserver-{{.Replica}}-{{.Iteration}} + ingress: + - ports: + - protocol: TCP + port: 8080 diff --git a/roles/kube-burner/files/curl-deployment.yml b/roles/kube-burner/files/curl-deployment.yml new file mode 100644 index 000000000..41b647746 --- /dev/null +++ b/roles/kube-burner/files/curl-deployment.yml @@ -0,0 +1,40 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: curl-{{.Replica}}-{{.Iteration}} +spec: + template: + metadata: + labels: + name: curl-{{.Replica}}-{{.Iteration}} + spec: + nodeSelector: + {{.nodeSelectorKey}}: {{.nodeSelectorValue}} + containers: + - name: curlapp + image: quay.io/cloud-bulldozer/curl:latest + command: ["sleep", "inf"] + env: + - name: WEBSERVER_HOSTNAME + value: webserver-{{.Replica}}-{{.Iteration}} + - name: WEBSERVER_PORT + value: "8080" + imagePullPolicy: IfNotPresent + securityContext: + privileged: false + readinessProbe: + exec: + command: + - "/bin/sh" + - "-c" + - "curl ${WEBSERVER_HOSTNAME}:${WEBSERVER_PORT}" + periodSeconds: 1 + restartPolicy: Always + replicas: 1 + selector: + matchLabels: + name: curl-{{.Replica}}-{{.Iteration}} + triggers: + - type: ConfigChange + strategy: + type: RollingUpdate diff --git a/roles/kube-burner/files/deny-all.yml b/roles/kube-burner/files/deny-all.yml new file mode 100644 index 000000000..599bf171b --- /dev/null +++ b/roles/kube-burner/files/deny-all.yml @@ -0,0 +1,7 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: deny-by-default +spec: + podSelector: {} + ingress: [] diff --git a/roles/kube-burner/files/webserver-deployment.yml b/roles/kube-burner/files/webserver-deployment.yml new file mode 100644 index 000000000..8b2267e49 --- /dev/null +++ b/roles/kube-burner/files/webserver-deployment.yml @@ -0,0 +1,30 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: webserver-{{.Replica}}-{{.Iteration}} +spec: + template: + metadata: + labels: + name: webserver-{{.Replica}}-{{.Iteration}} + spec: + nodeSelector: + {{.nodeSelectorKey}}: {{.nodeSelectorValue}} + containers: + - name: webserver + image: quay.io/cloud-bulldozer/sampleapp:latest + ports: + - containerPort: 8080 + protocol: TCP + imagePullPolicy: IfNotPresent + securityContext: + privileged: false + restartPolicy: Always + replicas: 1 + selector: + matchLabels: + name: webserver-{{.Replica}}-{{.Iteration}} + triggers: + - type: ConfigChange + strategy: + type: RollingUpdate diff --git a/roles/kube-burner/files/webserver-service.yml b/roles/kube-burner/files/webserver-service.yml new file mode 100644 index 000000000..a569151b8 --- /dev/null +++ b/roles/kube-burner/files/webserver-service.yml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: webserver-{{.Replica}}-{{.Iteration}} +spec: + selector: + name: webserver-{{.Replica}}-{{.Iteration}} + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + type: ClusterIP diff --git a/roles/kube-burner/tasks/main.yml b/roles/kube-burner/tasks/main.yml index 505639e41..e245a9a67 100644 --- a/roles/kube-burner/tasks/main.yml +++ b/roles/kube-burner/tasks/main.yml @@ -68,6 +68,40 @@ postgres-service.yml: "{{ lookup('file', 'postgres-service.yml')}}" when: workload_args.workload == "node-density-heavy" + - name: Create node-density-cni configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'node-density-cni.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" + curl-deployment.yml: "{{ lookup('file', 'curl-deployment.yml')}}" + webserver-deployment.yml: "{{ lookup('file', 'webserver-deployment.yml')}}" + webserver-service.yml: "{{ lookup('file', 'webserver-service.yml')}}" + when: workload_args.workload == "node-density-cni" + + - name: Create node-density-cni-networkpolicy configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'node-density-cni-networkpolicy.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" + curl-deployment.yml: "{{ lookup('file', 'curl-deployment.yml')}}" + webserver-deployment.yml: "{{ lookup('file', 'webserver-deployment.yml')}}" + webserver-service.yml: "{{ lookup('file', 'webserver-service.yml')}}" + deny-all.yml: "{{ lookup('file', 'deny-all.yml')}}" + allow-http.yml: "{{ lookup('file', 'allow-http.yml')}}" + when: workload_args.workload == "node-density-cni-policy" + - name: Create max-namespaces configmaps k8s: definition: diff --git a/roles/kube-burner/templates/node-density-cni-networkpolicy.yml.j2 b/roles/kube-burner/templates/node-density-cni-networkpolicy.yml.j2 new file mode 100644 index 000000000..21f3085fb --- /dev/null +++ b/roles/kube-burner/templates/node-density-cni-networkpolicy.yml.j2 @@ -0,0 +1,64 @@ +--- +global: + writeToFile: false +{% if prometheus is defined and prometheus.prom_url is defined and prometheus.es_url != "" %} + indexerConfig: + enabled: true + esServers: ["{{ prometheus.es_url }}"] + insecureSkipVerify: true + defaultIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} + type: elastic + measurements: + - name: podLatency + esIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} +{% endif %} + + +jobs: + + - name: deny-all-policy + jobIterations: 1 + qps: 1 + burst: 1 + namespacedIterations: false + namespace: {{workload_args.workload}}-{{ uuid }} + cleanup: {{ workload_args.cleanup|default(true) }} + jobPause: 1m + objects: + + - objectTemplate: deny-all.yml + replicas: 1 + + - name: {{ workload_args.workload }} + jobIterations: {{ workload_args.job_iterations }} + qps: {{ workload_args.qps|default(5) }} + burst: {{ workload_args.burst|default(10) }} + namespacedIterations: false + namespace: {{workload_args.workload}}-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + preLoadImages: true + preLoadPeriod: 2m + objects: + + - objectTemplate: webserver-deployment.yml + replicas: 1 + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + + + - objectTemplate: webserver-service.yml + replicas: 1 + + - objectTemplate: allow-http.yml + replicas: 1 + + - objectTemplate: curl-deployment.yml + replicas: 1 + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + diff --git a/roles/kube-burner/templates/node-density-cni.yml.j2 b/roles/kube-burner/templates/node-density-cni.yml.j2 new file mode 100644 index 000000000..aa7a1f738 --- /dev/null +++ b/roles/kube-burner/templates/node-density-cni.yml.j2 @@ -0,0 +1,47 @@ +--- +global: + writeToFile: false +{% if prometheus is defined and prometheus.prom_url is defined and prometheus.es_url != "" %} + indexerConfig: + enabled: true + esServers: ["{{ prometheus.es_url }}"] + insecureSkipVerify: true + defaultIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} + type: elastic + measurements: + - name: podLatency + esIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} +{% endif %} + + +jobs: + - name: {{ workload_args.workload }} + jobIterations: {{ workload_args.job_iterations }} + qps: {{ workload_args.qps|default(5) }} + burst: {{ workload_args.burst|default(10) }} + namespacedIterations: false + namespace: {{workload_args.workload}}-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + cleanup: {{ workload_args.cleanup|default(true) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + preLoadImages: true + preLoadPeriod: 2m + objects: + + - objectTemplate: webserver-deployment.yml + replicas: 1 + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + + - objectTemplate: webserver-service.yml + replicas: 1 + + - objectTemplate: curl-deployment.yml + replicas: 1 + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + diff --git a/tests/test_kubeburner.sh b/tests/test_kubeburner.sh index 88f57883f..c4f6a15a0 100755 --- a/tests/test_kubeburner.sh +++ b/tests/test_kubeburner.sh @@ -54,4 +54,4 @@ functional_test_kubeburner node-density metrics.yaml functional_test_kubeburner node-density-heavy metrics.yaml functional_test_kubeburner max-namespaces metrics-aggregated.yaml functional_test_kubeburner max-services metrics-aggregated.yaml -functional_test_kubeburner concurrent-builds metrics-aggregated.yaml \ No newline at end of file +functional_test_kubeburner concurrent-builds metrics-aggregated.yaml From 2c64d45eb9885170b03613a6f6fc32c06c4a49ca Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy <70236227+mukrishn@users.noreply.github.com> Date: Tue, 22 Feb 2022 13:07:44 -0500 Subject: [PATCH 196/249] corrected indentation with `filter` (#735) `include` within `macro` couldn't pass contexts so `metadata` wasn't set --- roles/uperf/templates/server.yml.j2 | 5 +++-- roles/uperf/templates/workload.yml.j2 | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 8bda30bd2..37e8fc5ee 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -75,8 +75,9 @@ items: - name: net.ipv4.ip_local_port_range value: 30000 30011 {% endif %} -{% macro metadata() %}{% include "metadata.yml.j2" %}{% endmacro %} - {{ metadata()|indent }} +{% filter indent(width=4, first=True) %} +{% include "metadata.yml.j2" %} +{% endfilter %} {% endmacro %} {% for node_idx_item in range(node_sequence|int) %} {% for item in range(pod_sequence|int) %} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 686e7197d..4ac2f5403 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -238,6 +238,7 @@ items: {% endif %} -{% macro metadata() %}{% include "metadata.yml.j2" %}{% endmacro %} - {{ metadata()|indent }} +{% filter indent(width=4, first=True) %} +{% include "metadata.yml.j2" %} +{% endfilter %} {% endfor %} From 0179028a9980f22c35d9239ed15182c53b26b5fb Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy Date: Wed, 2 Mar 2022 19:03:10 -0500 Subject: [PATCH 197/249] included eol to fix the issue --- roles/uperf/templates/workload.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 4ac2f5403..ea5466777 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -241,4 +241,5 @@ items: {% filter indent(width=4, first=True) %} {% include "metadata.yml.j2" %} {% endfilter %} + {% endfor %} From f72e3b07d9b9184a7c69b5d1fecee5949c604b94 Mon Sep 17 00:00:00 2001 From: Qiujie Li Date: Sat, 29 Jan 2022 17:25:00 +0800 Subject: [PATCH 198/249] add proxy to k8s client if it exists in env CentOS 8 is now EOL, so moving to stream for now. Signed-off-by: Sai Sindhur Malleni --- cli/ripsaw/clients/k8s.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/cli/ripsaw/clients/k8s.py b/cli/ripsaw/clients/k8s.py index b42fa43e0..2f3b67746 100755 --- a/cli/ripsaw/clients/k8s.py +++ b/cli/ripsaw/clients/k8s.py @@ -16,10 +16,12 @@ import time +import urllib3 from kubernetes import client, config from ripsaw.util import logging from ripsaw.util.exceptions import BenchmarkFailedError, BenchmarkTimeoutError, PodsNotFoundError +import os logger = logging.get_logger(__name__) @@ -40,11 +42,21 @@ class Cluster: def __init__(self, kubeconfig_path=None): """Initialize object and create clients from specified kubeconfig""" - config.load_kube_config(config_file=kubeconfig_path) - self.api_client = client.ApiClient() - self.core_client = client.CoreV1Api() - self.batch_client = client.BatchV1Api() - self.crd_client = client.CustomObjectsApi() + client_config = client.Configuration() + http_proxy = os.getenv('http_proxy', None) + """Proxy has auth header""" + if http_proxy and "@" in http_proxy: + proxy_auth = http_proxy.split("@")[0].split("//")[1] + proxy_url = http_proxy + if proxy_url: + client_config.proxy = proxy_url + if proxy_auth: + client_config.proxy_headers = urllib3.make_headers(proxy_basic_auth=proxy_auth) + config.load_kube_config(config_file=kubeconfig_path, client_configuration=client_config) + self.api_client = client.ApiClient(client_config) + self.core_client = client.CoreV1Api(self.api_client) + self.batch_client = client.BatchV1Api(self.api_client) + self.crd_client = client.CustomObjectsApi(self.api_client) def get_pods(self, label_selector, namespace): """Get pods matching specified label selector in specified namespace""" From f296081b4963742f286cbf22b2b7c97f0fef6189 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 7 Mar 2022 15:57:12 +0100 Subject: [PATCH 199/249] Update package list before installing Signed-off-by: Raul Sevilla --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 46ba91369..72a1c5753 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: steps: - name: Install dependencies required for multi-arch builds - run: sudo apt-get install qemu-user-static podman fuse-overlayfs + run: sudo apt-get update && sudo apt-get install qemu-user-static podman fuse-overlayfs - name: Check out code uses: actions/checkout@v2 From 5a277c4ae2951e689e3b68e827ba4ec284761ff2 Mon Sep 17 00:00:00 2001 From: ebattat <73884315+ebattat@users.noreply.github.com> Date: Fri, 11 Mar 2022 09:01:00 +0200 Subject: [PATCH 200/249] fix firewall in centos stream8 (#742) --- roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 index d5b29c65b..b90a84cb3 100644 --- a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 @@ -54,8 +54,10 @@ export HOME=/root; or workload_args.client_vm.hostpath is sameas true %} sudo systemctl stop mssql-server; sudo ACCEPT_EULA='Y' MSSQL_PID='Enterprise' MSSQL_SA_PASSWORD='s3curePasswordString' /opt/mssql/bin/mssql-conf setup; -sudo firewall-cmd --zone=public --add-port=1433/tcp --permanent; -sudo firewall-cmd --reload; +# Fixed firewall-cmd failure in cloud-init, for more details https://forums.centos.org/viewtopic.php?f=47&t=52162&p=220915 +sudo firewall-offline-cmd --add-port=1433/tcp; +sudo systemctl enable firewalld; +sudo systemctl start firewalld; sudo systemctl start mssql-server; {% endif %} cd /hammer; From 716edf25e4132ffce2e2c42a8381f772dd8567c8 Mon Sep 17 00:00:00 2001 From: dry923 Date: Tue, 8 Mar 2022 09:40:49 -0500 Subject: [PATCH 201/249] Remove redundant metadata section in client --- roles/uperf/templates/workload.yml.j2 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index ea5466777..16df68555 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -238,8 +238,4 @@ items: {% endif %} -{% filter indent(width=4, first=True) %} -{% include "metadata.yml.j2" %} -{% endfilter %} - {% endfor %} From da344e8921bc4d2410b3750e5716a0c517700907 Mon Sep 17 00:00:00 2001 From: ebattat <73884315+ebattat@users.noreply.github.com> Date: Thu, 17 Mar 2022 09:14:55 +0200 Subject: [PATCH 202/249] update hammerdb vm to centos stream8 (#744) --- config/samples/hammerdb/mariadb/vm-cr.yaml | 2 +- config/samples/hammerdb/mssql/vm-cr.yaml | 2 +- config/samples/hammerdb/postgres/vm-cr.yaml | 2 +- config/samples/hammerdb/vm-cr.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/samples/hammerdb/mariadb/vm-cr.yaml b/config/samples/hammerdb/mariadb/vm-cr.yaml index 03fd85c81..f0841a497 100644 --- a/config/samples/hammerdb/mariadb/vm-cr.yaml +++ b/config/samples/hammerdb/mariadb/vm-cr.yaml @@ -78,7 +78,7 @@ spec: sockets: 1 cores: 2 threads: 1 - image: quay.io/cloud-bulldozer/centos8-mariadb103-container-disk:latest + image: quay.io/cloud-bulldozer/centos-stream8-mariadb103-container-disk:latest requests: memory: 100Mi limits: diff --git a/config/samples/hammerdb/mssql/vm-cr.yaml b/config/samples/hammerdb/mssql/vm-cr.yaml index cb87abcae..65970581a 100644 --- a/config/samples/hammerdb/mssql/vm-cr.yaml +++ b/config/samples/hammerdb/mssql/vm-cr.yaml @@ -77,7 +77,7 @@ spec: sockets: 1 cores: 2 threads: 1 - image: quay.io/cloud-bulldozer/centos8-mssql2019-container-disk:latest + image: quay.io/cloud-bulldozer/centos-stream8-mssql2019-container-disk:latest requests: memory: 100Mi limits: diff --git a/config/samples/hammerdb/postgres/vm-cr.yaml b/config/samples/hammerdb/postgres/vm-cr.yaml index 2d8176d7e..fa136f5c9 100644 --- a/config/samples/hammerdb/postgres/vm-cr.yaml +++ b/config/samples/hammerdb/postgres/vm-cr.yaml @@ -77,7 +77,7 @@ spec: sockets: 1 cores: 2 threads: 1 - image: quay.io/cloud-bulldozer/centos8-postgres10-container-disk:latest + image: quay.io/cloud-bulldozer/centos-stream8-postgres10-container-disk:latest requests: memory: 100Mi limits: diff --git a/config/samples/hammerdb/vm-cr.yaml b/config/samples/hammerdb/vm-cr.yaml index 736f9451c..97e2ec7b0 100644 --- a/config/samples/hammerdb/vm-cr.yaml +++ b/config/samples/hammerdb/vm-cr.yaml @@ -78,7 +78,7 @@ spec: sockets: 1 cores: 2 threads: 1 - image: quay.io/cloud-bulldozer/centos8-mssql2019-container-disk:latest + image: quay.io/cloud-bulldozer/centos-stream8-mssql2019-container-disk:latest limits: memory: 16Gi # at least 16Gi for VM requests: From e36032bf7d925d0cf01b0531493186da6b573cf8 Mon Sep 17 00:00:00 2001 From: Ben England Date: Thu, 24 Mar 2022 18:43:36 -0400 Subject: [PATCH 203/249] Fs drift test captures response time stats (#747) * accept response_times workload parameter * increase test duration, capture response time stats test duration must be > 60 seconds to capture rsp. time stats ask fs-drift to save response time stats * do not embed space in test name * to have nonzero records in ripsaw-fs-drift-rsptimes index * CI actually uses e2e, not tests/test_crs --- e2e/fs_drift/fs_drift.yaml | 3 ++- e2e/fs_drift/fs_drift_hostpath.yaml | 3 ++- roles/fs-drift/templates/other_parameters.yaml.j2 | 3 +++ tests/test_crs/valid_fs_drift.yaml | 10 ++++------ tests/test_crs/valid_fs_drift_hostpath.yaml | 7 ++----- tests/test_fs_drift.sh | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/e2e/fs_drift/fs_drift.yaml b/e2e/fs_drift/fs_drift.yaml index 42e750698..fd168c2f5 100644 --- a/e2e/fs_drift/fs_drift.yaml +++ b/e2e/fs_drift/fs_drift.yaml @@ -26,5 +26,6 @@ spec: threads: 5 max_file_size_kb: 4 max_files: 1000 - duration: 10 + duration: 240 debug: true + response_times: true diff --git a/e2e/fs_drift/fs_drift_hostpath.yaml b/e2e/fs_drift/fs_drift_hostpath.yaml index 77596f9b9..4ccb93a63 100644 --- a/e2e/fs_drift/fs_drift_hostpath.yaml +++ b/e2e/fs_drift/fs_drift_hostpath.yaml @@ -27,5 +27,6 @@ spec: threads: 5 max_file_size_kb: 4 max_files: 1000 - duration: 10 + duration: 240 debug: true + response_times: true diff --git a/roles/fs-drift/templates/other_parameters.yaml.j2 b/roles/fs-drift/templates/other_parameters.yaml.j2 index 69b580d86..d36352be7 100644 --- a/roles/fs-drift/templates/other_parameters.yaml.j2 +++ b/roles/fs-drift/templates/other_parameters.yaml.j2 @@ -47,5 +47,8 @@ gaussian_stddev: {{workload_args.gaussian_stddev}} {% if workload_args.create_stddevs_ahead is defined %} create_stddevs_ahead: {{workload_args.create_stddevs_ahead}} {% endif %} +{% if workload_args.response_times is defined %} +response_times: {{workload_args.response_times}} +{% endif %} diff --git a/tests/test_crs/valid_fs_drift.yaml b/tests/test_crs/valid_fs_drift.yaml index 9ccd180ce..54a73ee18 100644 --- a/tests/test_crs/valid_fs_drift.yaml +++ b/tests/test_crs/valid_fs_drift.yaml @@ -4,18 +4,15 @@ metadata: name: fs-drift-benchmark namespace: benchmark-operator spec: - test_user: homer_simpson - # to separate this test run from everyone else's - clustername: test_ci system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: ES_SERVER + es_url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com prom_token: PROMETHEUS_TOKEN metrics_profile: node-metrics.yml # where elastic search is running elasticsearch: - url: ES_SERVER + url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com index_name: ripsaw-fs-drift metadata: collection: true @@ -26,5 +23,6 @@ spec: threads: 5 max_file_size_kb: 4 max_files: 1000 - duration: 15 + duration: 240 + response_times: Y debug: true diff --git a/tests/test_crs/valid_fs_drift_hostpath.yaml b/tests/test_crs/valid_fs_drift_hostpath.yaml index 91c8f65bb..66267425d 100644 --- a/tests/test_crs/valid_fs_drift_hostpath.yaml +++ b/tests/test_crs/valid_fs_drift_hostpath.yaml @@ -4,16 +4,12 @@ metadata: name: fs-drift-hostpath-benchmark namespace: benchmark-operator spec: - test_user: homer_simpson - # to separate this test run from everyone else's - clustername: test_ci system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ES_SERVER prom_token: PROMETHEUS_TOKEN metrics_profile: node-metrics.yml - # where elastic search is running elasticsearch: url: ES_SERVER index_name: ripsaw-fs-drift @@ -27,5 +23,6 @@ spec: threads: 5 max_file_size_kb: 4 max_files: 1000 - duration: 15 + duration: 240 + response_times: true debug: true diff --git a/tests/test_fs_drift.sh b/tests/test_fs_drift.sh index ac7d26d17..e62af6af7 100755 --- a/tests/test_fs_drift.sh +++ b/tests/test_fs_drift.sh @@ -43,4 +43,4 @@ function functional_test_fs_drift { figlet $(basename $0) functional_test_fs_drift "fs-drift" tests/test_crs/valid_fs_drift.yaml -functional_test_fs_drift "fs-drift hostpath" tests/test_crs/valid_fs_drift_hostpath.yaml +functional_test_fs_drift "fs-drift-hostpath" tests/test_crs/valid_fs_drift_hostpath.yaml From 176af6ee23fa2393007bfd65abb1d541a0f6c053 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Mon, 4 Apr 2022 11:07:26 +0200 Subject: [PATCH 204/249] Improve cli exeption handling Signed-off-by: Raul Sevilla --- cli/ripsaw/models/benchmark.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cli/ripsaw/models/benchmark.py b/cli/ripsaw/models/benchmark.py index e230422d7..e2cd8b3e8 100755 --- a/cli/ripsaw/models/benchmark.py +++ b/cli/ripsaw/models/benchmark.py @@ -62,9 +62,13 @@ def wait(self, desired_state="Complete", timeout=600): """Wait for benchmark to enter desired state with specified timeout in seconds (default: 600)""" if self.metadata == {}: raise BenchmarkNotStartedError(self.name) - self.cluster.wait_for_benchmark( - self.name, self.namespace, desired_state=desired_state, timeout=timeout - ) + try: + self.cluster.wait_for_benchmark( + self.name, self.namespace, desired_state=desired_state, timeout=timeout + ) + except Exception as err: + logger.error(f"Benchmark exception: {err}") + exit(1) self.metadata = self.cluster.get_benchmark_metadata(self.name, self.namespace) return self.metadata From 0003736ab7f97f7e5c1b172f4b7af8c6d3ac8fe0 Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Mon, 21 Mar 2022 14:58:31 -0500 Subject: [PATCH 205/249] Add resource requests to templates for CNI workloads This should help with better scheduling. Signed-off-by: Sai Sindhur Malleni --- roles/kube-burner/files/curl-deployment.yml | 4 ++++ roles/kube-burner/files/webserver-deployment.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/roles/kube-burner/files/curl-deployment.yml b/roles/kube-burner/files/curl-deployment.yml index 41b647746..b29eab4e1 100644 --- a/roles/kube-burner/files/curl-deployment.yml +++ b/roles/kube-burner/files/curl-deployment.yml @@ -14,6 +14,10 @@ spec: - name: curlapp image: quay.io/cloud-bulldozer/curl:latest command: ["sleep", "inf"] + resources: + requests: + memory: "10Mi" + cpu: "10m" env: - name: WEBSERVER_HOSTNAME value: webserver-{{.Replica}}-{{.Iteration}} diff --git a/roles/kube-burner/files/webserver-deployment.yml b/roles/kube-burner/files/webserver-deployment.yml index 8b2267e49..fa97b1e04 100644 --- a/roles/kube-burner/files/webserver-deployment.yml +++ b/roles/kube-burner/files/webserver-deployment.yml @@ -13,6 +13,10 @@ spec: containers: - name: webserver image: quay.io/cloud-bulldozer/sampleapp:latest + resources: + requests: + memory: "10Mi" + cpu: "10m" ports: - containerPort: 8080 protocol: TCP From 5cc61335c70c54468d6d2224c9f17fa7ce1534e6 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Wed, 6 Apr 2022 10:12:59 +0200 Subject: [PATCH 206/249] Updating api-load to match latest release - Removes extra step to upload to elastic since now it's done within ocm-api-load - Adds flags to use elastic search indexing within ocm-api-load - Adding ramping options for tests - Add missing aws-region parameter Signed-off-by: Vicente Zepeda Mas --- e2e/api_load/api_load.yaml | 1 + roles/api_load/templates/api_load.yml | 27 +++++++++++++++++---------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/e2e/api_load/api_load.yaml b/e2e/api_load/api_load.yaml index 48661a35a..eba253938 100644 --- a/e2e/api_load/api_load.yaml +++ b/e2e/api_load/api_load.yaml @@ -24,6 +24,7 @@ spec: aws_access_key: empty aws_access_secret: empty aws_account_id: empty + aws_region: empty cooldown: 10 sleep: 5 test_list: diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index 988cd8601..84913cb4f 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -65,19 +65,30 @@ spec: {% for key, value in workload_args.test_list.items() %} ocm-load-test --test-id={{ uuid }} - --gateway-url {{ workload_args.gateway_url }} + --gateway-url={{ workload_args.gateway_url }} --ocm-token={{ workload_args.ocm_token }} --duration={{ value.duration | default(workload_args.duration) }} --rate={{ value.rate | default(workload_args.rate) }} --output-path={{ workload_args.output_path | default('/tmp/results') }} - --test-names {{ key }} - --aws-access-key {{ workload_args.aws_access_key }} - --aws-access-secret {{ workload_args.aws_access_secret }} - --aws-account-id {{ workload_args.aws_account_id }}; + --test-names={{ key }} + --aws-access-key={{ workload_args.aws_access_key }} + --aws-access-secret={{ workload_args.aws_access_secret }} + --aws-account-id={{ workload_args.aws_account_id }} + --aws-region={{ workload_args.aws_region | default('us-west-2') }} +{% if value.ramp_type is defined %} + --ramp-type={{ value.ramp_type }} + --ramp-duration={{ value.ramp_duration | default(value.duration) }} + --ramp-steps={{ value.ramp_steps }} + --start-rate={{ value.ramp_start_rate }} + --end-rate={{ value.ramp_end_rate }} +{% endif %} + --elastic-server={{ elasticsearch.url }} + --elastic-index={{ elasticsearch.index_name }}; echo "Cooldown for {{ workload_args.cooldown | default(60) }}"; sleep {{ workload_args.cooldown | default(60) }}; {% endfor %} {% endif %} + echo "Updating redis status"; redis-cli -h {{ bo.resources[0].status.podIP }} SET "{{ uuid }}-status" "ready"; volumeMounts: - mountPath: /tmp/results @@ -96,14 +107,12 @@ spec: fieldPath: metadata.name - name: uuid value: "{{ uuid }}" - - name: ES - value: "{{ elasticsearch.url }}" command: ["/bin/sh", "-c"] args: - > echo "Getting redis status"; status=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "{{ uuid }}-status"`; - while [ $status != "ready" ]; do + while [ "$status" != "ready" ]; do sleep {{ workload_args.sleep | default(360) }}; echo "Testing readiness"; status=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "{{ uuid }}-status"`; @@ -111,8 +120,6 @@ spec: {% if workload_args.override is defined %} echo "succeeded" {% else %} - echo "Uploading to ES..."; - python automation.py esbulk --dir {{ workload_args.output_path | default('/tmp/results') }} --index {{ elasticsearch.index_name }}; {% if snappy.url %} echo "Uploading RAW files..."; python automation.py upload --dir {{ workload_args.output_path | default('/tmp/results') }} --server {{ snappy.url }} --user {{ snappy.user }} --password {{ snappy.password }}; From 0fac50d0ca52401a26636792c0fe676cec902925 Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Fri, 29 Apr 2022 16:48:52 -0500 Subject: [PATCH 207/249] Change readiness probe to startup probe for node-density-cni Signed-off-by: Sai Sindhur Malleni --- roles/kube-burner/files/curl-deployment.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/kube-burner/files/curl-deployment.yml b/roles/kube-burner/files/curl-deployment.yml index b29eab4e1..ad415900a 100644 --- a/roles/kube-burner/files/curl-deployment.yml +++ b/roles/kube-burner/files/curl-deployment.yml @@ -26,13 +26,15 @@ spec: imagePullPolicy: IfNotPresent securityContext: privileged: false - readinessProbe: + startupProbe: exec: command: - "/bin/sh" - "-c" - "curl ${WEBSERVER_HOSTNAME}:${WEBSERVER_PORT}" periodSeconds: 1 + timeoutSeconds: 1 + failureThreshold: 600 restartPolicy: Always replicas: 1 selector: From ba6a54ea3afd2cb09dd0354ed8c7e08e7847aee6 Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Mon, 23 May 2022 17:52:45 +0200 Subject: [PATCH 208/249] Adds option to check if ES certs are verified Signed-off-by: Vicente Zepeda Mas --- roles/api_load/templates/api_load.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml index 84913cb4f..905a13fcd 100644 --- a/roles/api_load/templates/api_load.yml +++ b/roles/api_load/templates/api_load.yml @@ -83,7 +83,12 @@ spec: --end-rate={{ value.ramp_end_rate }} {% endif %} --elastic-server={{ elasticsearch.url }} - --elastic-index={{ elasticsearch.index_name }}; + --elastic-index={{ elasticsearch.index_name }} +{% if elasticsearch.verify_cert %} + --elastic-insecure-skip-verify; +{% else %} + ; +{% endif %} echo "Cooldown for {{ workload_args.cooldown | default(60) }}"; sleep {{ workload_args.cooldown | default(60) }}; {% endfor %} From 369ae4489835827a677f5048ab998d6968a0d0d0 Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Wed, 4 May 2022 10:05:38 -0500 Subject: [PATCH 209/249] Enable multiple client/server pairs with NodePort uperf Currently uperf with NodePort servicetype only works with `pairs: 1`. This PR fixes the issue with running multiple pairs. Related PR: https://github.com/cloud-bulldozer/benchmark-wrapper/pull/426 Fixes: #757 Signed-off-by: Sai Sindhur Malleni --- e2e/uperf/uperf_serviceip_nodeport.yaml | 2 +- roles/uperf/tasks/start_client.yml | 17 +++++++++++-- roles/uperf/templates/configmap.yml.j2 | 9 ++++--- roles/uperf/templates/server.yml.j2 | 5 ++-- roles/uperf/templates/service.yml.j2 | 25 ++++++++++--------- roles/uperf/templates/workload.yml.j2 | 6 +++-- .../valid_uperf_serviceip_nodeport.yaml | 3 +-- 7 files changed, 42 insertions(+), 25 deletions(-) diff --git a/e2e/uperf/uperf_serviceip_nodeport.yaml b/e2e/uperf/uperf_serviceip_nodeport.yaml index b887c2d16..71fd86423 100644 --- a/e2e/uperf/uperf_serviceip_nodeport.yaml +++ b/e2e/uperf/uperf_serviceip_nodeport.yaml @@ -25,7 +25,7 @@ spec: multus: enabled: false samples: 1 - pair: 1 + pair: 2 test_types: - stream protos: diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml index 494fa5d7b..40fac6b62 100644 --- a/roles/uperf/tasks/start_client.yml +++ b/roles/uperf/tasks/start_client.yml @@ -9,9 +9,14 @@ - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods +- name: set pod_sequence + set_fact: + pod_sequence: "{{ pod_hi_idx|int }}" + - name: Generate uperf xml files k8s: definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" + with_sequence: start=0 end={{ pod_sequence|int }} - block: ### kind @@ -21,8 +26,16 @@ vars: resource_item: "{{ server_pods.resources }}" when: - - ( workload_args.serviceip|default(False) == False and server_pods.resources|length > 0 ) or - ( workload_args.serviceip|default(False) == True and server_pods.resources|length > 0 and + - ( workload_args.serviceip|default(False) == False and server_pods.resources|length > 0 ) + + + - name: Start Client(s) with nodeport serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + vars: + resource_item: "{{ server_pods.resources }}" + when: + - ( workload_args.serviceip|default(False) == True and server_pods.resources|length > 0 and workload_args.servicetype | default("clusterip") == "nodeport" ) # diff --git a/roles/uperf/templates/configmap.yml.j2 b/roles/uperf/templates/configmap.yml.j2 index bc1f82562..fc8b3b583 100644 --- a/roles/uperf/templates/configmap.yml.j2 +++ b/roles/uperf/templates/configmap.yml.j2 @@ -1,8 +1,9 @@ --- +{% set control_port = 30000 %} apiVersion: v1 kind: ConfigMap metadata: - name: uperf-test-{{ trunc_uuid }} + name: uperf-test-{{ item }}-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' data: {% for test in workload_args.test_types %} @@ -22,7 +23,7 @@ data: {% if ( 'rr' == test ) %} - + @@ -35,7 +36,7 @@ data: {% if ( 'stream' == test or 'bidirec' == test ) %} - + @@ -48,7 +49,7 @@ data: {% if ( 'maerts' == test or 'bidirec' == test ) %} - + diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 37e8fc5ee..14bd3776b 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -1,4 +1,5 @@ --- +{% set control_port = 30000 %} apiVersion: v1 kind: List metadata: {} @@ -60,7 +61,7 @@ items: {% endif %} imagePullPolicy: Always command: ["/bin/sh","-c"] - args: ["uperf -s -v -P 30000"] + args: ["uperf -s -v -P {{ control_port|int + item|int * 100 }}"] restartPolicy: OnFailure {% if workload_args.pin is sameas true %} nodeSelector: @@ -73,7 +74,7 @@ items: securityContext: sysctls: - name: net.ipv4.ip_local_port_range - value: 30000 30011 + value: 30000 32011 {% endif %} {% filter indent(width=4, first=True) %} {% include "metadata.yml.j2" %} diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 8db9a6186..ee0c0f8bf 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -1,4 +1,5 @@ --- +{% set control_port = 30000 %} apiVersion: v1 kind: List metadata: {} @@ -32,20 +33,20 @@ items: type: NodePort ports: - name: uperf - port: 30000 - targetPort: 30000 - nodePort: 30000 + port: {{ control_port|int + item|int * 100 }} + targetPort: {{ control_port|int + item|int * 100 }} + nodePort: {{ control_port|int + item * 100 }} protocol: TCP -{% for num in range(30001,30012,1) %} - - name: uperf-control-tcp-{{num}} - port: {{num}} - targetPort: {{num}} - nodePort: {{num}} +{% for num in range(1,12,1) %} + - name: uperf-control-tcp-{{ control_port|int + item * 100 + num }} + port: {{ control_port|int + item * 100 + num }} + targetPort: {{ control_port|int + item * 100 + num }} + nodePort: {{ control_port|int + item * 100 + num }} protocol: TCP - - name: uperf-control-udp-{{num}} - port: {{num}} - targetPort: {{num}} - nodePort: {{num}} + - name: uperf-control-udp-{{ control_port|int + item * 100 + num }} + port: {{ control_port|int + item * 100 + num }} + targetPort: {{ control_port|int + item * 100 + num }} + nodePort: {{ control_port|int + item * 100 + num }} protocol: UDP {% endfor %} {% else %} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 16df68555..b27e65c3b 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -1,4 +1,5 @@ --- +{% set control_port = 30000 %} apiVersion: v1 kind: List metadata: {} @@ -9,7 +10,7 @@ items: metadata: {% if workload_args.serviceip is sameas true %} {% if workload_args.servicetype | default("clusterip") == "nodeport" %} - name: 'uperf-client-{{item.status.hostIP}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.status.hostIP}}-{{ loop.index - 1 }}-{{ trunc_uuid }}' {% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} name: 'uperf-client-{{item.status.loadBalancer.ingress[0].ip}}-{{ trunc_uuid }}' {% else %} @@ -124,6 +125,7 @@ items: - "export h={{item.status.podIP}}; {% endif %} {% endif %} + export port={{ control_port + (loop.index - 1) * 100 }} {% if (workload_args.colocate is defined) %} export colocate={{ workload_args.colocate}}; {% endif %} @@ -217,7 +219,7 @@ items: volumes: - name: config-volume configMap: - name: uperf-test-{{ trunc_uuid }} + name: uperf-test-{{ loop.index - 1 }}-{{ trunc_uuid }} restartPolicy: Never {% if workload_args.pin is sameas false %} {% if workload_args.colocate is sameas true %} diff --git a/tests/test_crs/valid_uperf_serviceip_nodeport.yaml b/tests/test_crs/valid_uperf_serviceip_nodeport.yaml index dba9ad4b3..9ddb1fe73 100644 --- a/tests/test_crs/valid_uperf_serviceip_nodeport.yaml +++ b/tests/test_crs/valid_uperf_serviceip_nodeport.yaml @@ -28,7 +28,7 @@ spec: multus: enabled: false samples: 2 - pair: 1 + pair: 2 test_types: - stream - rr @@ -40,6 +40,5 @@ spec: - 512 nthrs: - 1 - - 2 runtime: 2 debug: true From a728e682e29fa3a9bd1e32b904e553fcda33f8aa Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Thu, 26 May 2022 10:29:28 -0500 Subject: [PATCH 210/249] Update ConfigMap name for CNV Fixes: #761 Signed-off-by: Sai Sindhur Malleni --- roles/uperf/tasks/start_client.yml | 2 +- roles/uperf/templates/workload_vm.yml.j2 | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml index 40fac6b62..c567c69e5 100644 --- a/roles/uperf/tasks/start_client.yml +++ b/roles/uperf/tasks/start_client.yml @@ -74,7 +74,7 @@ - name: Start Client(s) k8s: definition: "{{ lookup('template', 'workload_vm.yml.j2') | from_yaml }}" - with_items: "{{ server_vms.resources }}" + with_indexed_items: "{{ server_vms.resources }}" when: server_vms.resources|length > 0 when: resource_kind == "vm" diff --git a/roles/uperf/templates/workload_vm.yml.j2 b/roles/uperf/templates/workload_vm.yml.j2 index 7f548e2b6..5cc88d12a 100644 --- a/roles/uperf/templates/workload_vm.yml.j2 +++ b/roles/uperf/templates/workload_vm.yml.j2 @@ -2,7 +2,7 @@ apiVersion: kubevirt.io/v1alpha3 kind: VirtualMachineInstance metadata: - name: 'uperf-client-{{item.status.interfaces[0].ipAddress}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.1.status.interfaces[0].ipAddress}}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' labels: benchmark-uuid: {{ uuid }} @@ -90,9 +90,9 @@ spec: - bash /tmp/uperf_script/run_script.sh name: cloudinitdisk - configMap: - name: uperf-test-{{ trunc_uuid }} + name: uperf-test-{{ item.0 }}-{{ trunc_uuid }} name: app-config-disk - configMap: - name: uperf-run-script-{{item.status.interfaces[0].ipAddress}}-{{ trunc_uuid }} + name: uperf-run-script-{{item.1.status.interfaces[0].ipAddress}}-{{ trunc_uuid }} name: run-config-disk status: {} From 09a14b5fede0cf6ca0461209f31eea31682fa4c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Tue, 31 May 2022 10:34:08 +0200 Subject: [PATCH 211/249] Reduce test scope (#760) * Reduce FIO test scope Signed-off-by: Raul Sevilla * Disable metadata collection from some benchmarks with more than test Signed-off-by: Raul Sevilla * Collect prom metrics only in one test Signed-off-by: Raul Sevilla * Incrase number of bats concurrent jobs and increase test timeout Signed-off-by: Raul Sevilla * Disable system metrics collection Signed-off-by: Raul Sevilla --- Makefile | 2 +- e2e/001-scale_openshift.bats | 4 +- e2e/002-sysbench.bats | 2 +- e2e/003-byowl.bats | 4 +- e2e/004-backpack.bats | 4 +- e2e/005-stressng.bats | 2 +- e2e/006-vegeta.bats | 4 +- e2e/007-smallfile.bats | 4 +- e2e/008-kube-burner.bats | 25 +++++------- e2e/009-iperf3.bats | 2 +- e2e/010-fio.bats | 8 ++-- e2e/011-ycsb.bats | 2 +- e2e/012-flent.bats | 4 +- e2e/013-uperf.bats | 12 +++--- e2e/014-pgbench.bats | 2 +- e2e/015-image_pull.bats | 2 +- e2e/016-hammerdb.bats | 2 +- e2e/017-fs_drift.bats | 4 +- e2e/018-log-generator.bats | 2 +- e2e/byowl/byowl-targeted.yaml | 2 +- e2e/fio/fio_bsrange.yaml | 40 +------------------ e2e/fio/fio_hostpath.yaml | 39 +----------------- e2e/fio/fio_ocs_cache_drop.yaml | 40 ++----------------- e2e/flent/flent_resources.yaml | 4 +- e2e/fs_drift/fs_drift_hostpath.yaml | 4 +- e2e/kube-burner/concurrent-builds.yaml | 4 -- e2e/kube-burner/configmap.yaml | 6 +-- e2e/kube-burner/max-namespaces.yaml | 4 -- e2e/kube-burner/max-services.yaml | 4 -- .../node-density-cni-networkpolicy.yaml | 4 -- e2e/kube-burner/node-density-cni.yaml | 4 -- e2e/kube-burner/node-density-heavy.yaml | 4 -- e2e/kube-burner/node-density.yaml | 4 -- e2e/scale-openshift/scale_down.yaml | 2 +- e2e/smallfile/smallfile_hostpath.yaml | 4 +- e2e/uperf/uperf_hostnetwork.yaml | 4 +- e2e/uperf/uperf_networkpolicy.yaml | 4 +- e2e/uperf/uperf_resources.yaml | 4 +- e2e/uperf/uperf_serviceip.yaml | 4 +- e2e/uperf/uperf_serviceip_nodeport.yaml | 4 +- e2e/vegeta/vegeta_hostnetwork.yaml | 4 +- 41 files changed, 71 insertions(+), 213 deletions(-) diff --git a/Makefile b/Makefile index 2fca5571e..ca4b52791 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ endif # E2E testing E2E_DIR = e2e E2E_RESULTS = test-results.xml -BATS_FLAGS = -j 5 -F pretty --report-formatter junit -T +BATS_FLAGS = -j 6 -F pretty --report-formatter junit -T ifdef BATS_TESTS FILTERED_TESTS := -f "$(BATS_TESTS)" endif diff --git a/e2e/001-scale_openshift.bats b/e2e/001-scale_openshift.bats index bc3eaf1b8..28da05b7c 100755 --- a/e2e/001-scale_openshift.bats +++ b/e2e/001-scale_openshift.bats @@ -13,7 +13,7 @@ ES_INDEX=openshift-cluster-timings CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 # Make node no schedulable as soon as benchmark finishes kubectl cordon $(kubectl get node --sort-by='{.metadata.creationTimestamp}' -o name | tail -1) # Annotate machine to ensure it's the one deleted in the scale-down test @@ -27,7 +27,7 @@ ES_INDEX=openshift-cluster-timings CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 || die "Timeout waiting for benchmark/${CR_NAME} to complete" + check_benchmark 1200 || die "Timeout waiting for benchmark/${CR_NAME} to complete" check_es } diff --git a/e2e/002-sysbench.bats b/e2e/002-sysbench.bats index d8688f655..3c2fef6f9 100755 --- a/e2e/002-sysbench.bats +++ b/e2e/002-sysbench.bats @@ -12,7 +12,7 @@ ES_INDEX=ripsaw-sysbench-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 } setup_file() { diff --git a/e2e/003-byowl.bats b/e2e/003-byowl.bats index b92b0b9b2..748283cf0 100755 --- a/e2e/003-byowl.bats +++ b/e2e/003-byowl.bats @@ -12,7 +12,7 @@ load helpers.bash CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 } @test "byowl-not-targeted" { @@ -20,7 +20,7 @@ load helpers.bash CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 } setup_file() { diff --git a/e2e/004-backpack.bats b/e2e/004-backpack.bats index 68c519d6b..338a076ee 100755 --- a/e2e/004-backpack.bats +++ b/e2e/004-backpack.bats @@ -12,7 +12,7 @@ indexes=(cpu_vulnerabilities-metadata cpuinfo-metadata dmidecode-metadata k8s_no CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -21,7 +21,7 @@ indexes=(cpu_vulnerabilities-metadata cpuinfo-metadata dmidecode-metadata k8s_no CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/005-stressng.bats b/e2e/005-stressng.bats index cc1a85dd9..a4ca9893b 100755 --- a/e2e/005-stressng.bats +++ b/e2e/005-stressng.bats @@ -12,7 +12,7 @@ ES_INDEX=ripsaw-stressng-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/006-vegeta.bats b/e2e/006-vegeta.bats index 85cd1c560..92188dc72 100755 --- a/e2e/006-vegeta.bats +++ b/e2e/006-vegeta.bats @@ -12,7 +12,7 @@ ES_INDEX=ripsaw-vegeta-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -21,7 +21,7 @@ ES_INDEX=ripsaw-vegeta-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/007-smallfile.bats b/e2e/007-smallfile.bats index 00fc84e83..652e04284 100755 --- a/e2e/007-smallfile.bats +++ b/e2e/007-smallfile.bats @@ -12,7 +12,7 @@ indexes=(ripsaw-smallfile-results ripsaw-smallfile-rsptimes) CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -21,7 +21,7 @@ indexes=(ripsaw-smallfile-results ripsaw-smallfile-rsptimes) CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/008-kube-burner.bats b/e2e/008-kube-burner.bats index 021053251..19c90aa79 100755 --- a/e2e/008-kube-burner.bats +++ b/e2e/008-kube-burner.bats @@ -12,7 +12,7 @@ ES_INDEX=ripsaw-kube-burner CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -21,8 +21,7 @@ ES_INDEX=ripsaw-kube-burner CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 - check_es + check_benchmark 1200 } @test "kube-burner-node-density-heavy" { @@ -30,8 +29,7 @@ ES_INDEX=ripsaw-kube-burner CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 - check_es + check_benchmark 1200 } @test "kube-burner-node-density-cni" { @@ -39,8 +37,7 @@ ES_INDEX=ripsaw-kube-burner CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 - check_es + check_benchmark 1200 } @test "kube-burner-node-density-cni-networkpolicy" { @@ -48,8 +45,7 @@ ES_INDEX=ripsaw-kube-burner CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 - check_es + check_benchmark 1200 } @test "kube-burner-max-services" { @@ -57,8 +53,7 @@ ES_INDEX=ripsaw-kube-burner CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 - check_es + check_benchmark 1200 } @test "kube-burner-max-namespaces" { @@ -66,8 +61,7 @@ ES_INDEX=ripsaw-kube-burner CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 - check_es + check_benchmark 1200 } @test "kube-burner-concurrent-builds" { @@ -75,8 +69,7 @@ ES_INDEX=ripsaw-kube-burner CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 - check_es + check_benchmark 1200 } @test "kube-burner-configmap" { @@ -84,7 +77,7 @@ ES_INDEX=ripsaw-kube-burner CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/009-iperf3.bats b/e2e/009-iperf3.bats index 23efd9a4e..ed0786ce5 100755 --- a/e2e/009-iperf3.bats +++ b/e2e/009-iperf3.bats @@ -11,7 +11,7 @@ load helpers.bash CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 } setup_file() { diff --git a/e2e/010-fio.bats b/e2e/010-fio.bats index cae59ab0d..c676094e0 100755 --- a/e2e/010-fio.bats +++ b/e2e/010-fio.bats @@ -12,7 +12,7 @@ indexes=(ripsaw-fio-results ripsaw-fio-log ripsaw-fio-analyzed-result) CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -21,7 +21,7 @@ indexes=(ripsaw-fio-results ripsaw-fio-log ripsaw-fio-analyzed-result) CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -30,7 +30,7 @@ indexes=(ripsaw-fio-results ripsaw-fio-log ripsaw-fio-analyzed-result) CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -50,7 +50,7 @@ indexes=(ripsaw-fio-results ripsaw-fio-log ripsaw-fio-analyzed-result) sleep 5 envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es fi } diff --git a/e2e/011-ycsb.bats b/e2e/011-ycsb.bats index d720de652..2baf8d271 100755 --- a/e2e/011-ycsb.bats +++ b/e2e/011-ycsb.bats @@ -12,7 +12,7 @@ indexes=(ripsaw-ycsb-summary ripsaw-ycsb-results) CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/012-flent.bats b/e2e/012-flent.bats index 8f6920dff..f25da6daf 100755 --- a/e2e/012-flent.bats +++ b/e2e/012-flent.bats @@ -12,7 +12,7 @@ ES_INDEX=ripsaw-flent-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -22,7 +22,7 @@ ES_INDEX=ripsaw-flent-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/013-uperf.bats b/e2e/013-uperf.bats index 706b3bc19..525c88a61 100755 --- a/e2e/013-uperf.bats +++ b/e2e/013-uperf.bats @@ -12,7 +12,7 @@ ES_INDEX=ripsaw-uperf-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -21,7 +21,7 @@ ES_INDEX=ripsaw-uperf-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -30,7 +30,7 @@ ES_INDEX=ripsaw-uperf-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -39,7 +39,7 @@ ES_INDEX=ripsaw-uperf-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -48,7 +48,7 @@ ES_INDEX=ripsaw-uperf-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -57,7 +57,7 @@ ES_INDEX=ripsaw-uperf-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/014-pgbench.bats b/e2e/014-pgbench.bats index a78260184..d57c9b035 100755 --- a/e2e/014-pgbench.bats +++ b/e2e/014-pgbench.bats @@ -12,7 +12,7 @@ indexes=(ripsaw-pgbench-summary ripsaw-pgbench-raw) CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/015-image_pull.bats b/e2e/015-image_pull.bats index 696229e25..8be95e022 100755 --- a/e2e/015-image_pull.bats +++ b/e2e/015-image_pull.bats @@ -12,7 +12,7 @@ ES_INDEX=image-pull-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/016-hammerdb.bats b/e2e/016-hammerdb.bats index 7b46b0e91..44cfb7ace 100755 --- a/e2e/016-hammerdb.bats +++ b/e2e/016-hammerdb.bats @@ -12,7 +12,7 @@ ES_INDEX=ripsaw-hammerdb-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/017-fs_drift.bats b/e2e/017-fs_drift.bats index 443047629..6970d0180 100755 --- a/e2e/017-fs_drift.bats +++ b/e2e/017-fs_drift.bats @@ -12,7 +12,7 @@ indexes=(ripsaw-fs-drift-results ripsaw-fs-drift-rsptimes ripsaw-fs-drift-rates- CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } @@ -22,7 +22,7 @@ indexes=(ripsaw-fs-drift-results ripsaw-fs-drift-rsptimes ripsaw-fs-drift-rates- CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/018-log-generator.bats b/e2e/018-log-generator.bats index 1a4059407..072f0373c 100755 --- a/e2e/018-log-generator.bats +++ b/e2e/018-log-generator.bats @@ -12,7 +12,7 @@ ES_INDEX=log-generator-results CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" - check_benchmark 900 + check_benchmark 1200 check_es } diff --git a/e2e/byowl/byowl-targeted.yaml b/e2e/byowl/byowl-targeted.yaml index f17654a93..655eb0116 100644 --- a/e2e/byowl/byowl-targeted.yaml +++ b/e2e/byowl/byowl-targeted.yaml @@ -5,7 +5,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} diff --git a/e2e/fio/fio_bsrange.yaml b/e2e/fio/fio_bsrange.yaml index e77fd32d4..77c5fbf4a 100644 --- a/e2e/fio/fio_bsrange.yaml +++ b/e2e/fio/fio_bsrange.yaml @@ -5,7 +5,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -14,7 +14,7 @@ spec: url: ${ES_SERVER} index_name: ripsaw-fio metadata: - collection: true + collection: false cleanup: false workload: name: "fio_distributed" @@ -23,10 +23,6 @@ spec: servers: 2 jobs: - write - - read - - randwrite - - randread - - randrw bsrange: - 4KiB-16KiB numjobs: @@ -52,35 +48,3 @@ spec: - create_on_open=1 - runtime={{ workload_args.write_runtime }} - ramp_time={{ workload_args.write_ramp_time }} - - jobname_match: read - params: - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: rw - params: - - rwmixread=50 - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: readwrite - params: - - rwmixread=50 - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: randread - params: - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: randwrite - params: - - time_based=1 - - runtime={{ workload_args.write_runtime }} - - ramp_time={{ workload_args.write_ramp_time }} - - jobname_match: randrw - params: - - time_based=1 - - runtime={{ workload_args.write_runtime }} - - ramp_time={{ workload_args.write_ramp_time }} diff --git a/e2e/fio/fio_hostpath.yaml b/e2e/fio/fio_hostpath.yaml index 9fb5a2885..71ec9b2da 100644 --- a/e2e/fio/fio_hostpath.yaml +++ b/e2e/fio/fio_hostpath.yaml @@ -5,7 +5,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -14,7 +14,7 @@ spec: url: ${ES_SERVER} index_name: ripsaw-fio metadata: - collection: true + collection: false cleanup: false hostpath: /mnt/vda1/workload_args/ workload: @@ -25,8 +25,6 @@ spec: prefill: true jobs: - write - - randread - - randrw bs: - 4KiB numjobs: @@ -49,36 +47,3 @@ spec: - create_on_open=1 - runtime={{ workload_args.write_runtime }} - ramp_time={{ workload_args.write_ramp_time }} - - jobname_match: read - params: - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: rw - params: - - rwmixread=50 - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: readwrite - params: - - rwmixread=50 - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: randread - params: - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: randwrite - params: - - time_based=1 - - runtime={{ workload_args.write_runtime }} - - ramp_time={{ workload_args.write_ramp_time }} - - jobname_match: randrw - params: - - time_based=1 - - runtime={{ workload_args.write_runtime }} - - ramp_time={{ workload_args.write_ramp_time }} - diff --git a/e2e/fio/fio_ocs_cache_drop.yaml b/e2e/fio/fio_ocs_cache_drop.yaml index 7e3746a7a..6b07f116b 100644 --- a/e2e/fio/fio_ocs_cache_drop.yaml +++ b/e2e/fio/fio_ocs_cache_drop.yaml @@ -5,7 +5,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -14,7 +14,7 @@ spec: url: ${ES_SERVER} index_name: ripsaw-fio metadata: - collection: true + collection: false cleanup: false workload: name: "fio_distributed" @@ -24,9 +24,7 @@ spec: samples: 1 servers: 2 jobs: - - randwrite - - randread - - randrw + - write bs: - 4KiB numjobs: @@ -51,35 +49,3 @@ spec: - create_on_open=1 - runtime={{ workload_args.write_runtime }} - ramp_time={{ workload_args.write_ramp_time }} - - jobname_match: read - params: - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: rw - params: - - rwmixread=50 - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: readwrite - params: - - rwmixread=50 - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: randread - params: - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: randwrite - params: - - time_based=1 - - runtime={{ workload_args.write_runtime }} - - ramp_time={{ workload_args.write_ramp_time }} - - jobname_match: randrw - params: - - time_based=1 - - runtime={{ workload_args.write_runtime }} - - ramp_time={{ workload_args.write_ramp_time }} diff --git a/e2e/flent/flent_resources.yaml b/e2e/flent/flent_resources.yaml index 8fc7253a5..725b50e38 100644 --- a/e2e/flent/flent_resources.yaml +++ b/e2e/flent/flent_resources.yaml @@ -5,7 +5,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -13,6 +13,8 @@ spec: elasticsearch: url: ${ES_SERVER} index_name: ripsaw-flent + metadata: + collection: false workload: # cleanup: true name: flent diff --git a/e2e/fs_drift/fs_drift_hostpath.yaml b/e2e/fs_drift/fs_drift_hostpath.yaml index 4ccb93a63..ea5bb6d43 100644 --- a/e2e/fs_drift/fs_drift_hostpath.yaml +++ b/e2e/fs_drift/fs_drift_hostpath.yaml @@ -8,7 +8,7 @@ spec: # to separate this test run from everyone else's clustername: test_ci system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -18,7 +18,7 @@ spec: url: ${ES_SERVER} index_name: ripsaw-fs-drift metadata: - collection: true + collection: false hostpath: /mnt/vda1/fs_drift workload: name: fs-drift diff --git a/e2e/kube-burner/concurrent-builds.yaml b/e2e/kube-burner/concurrent-builds.yaml index b56c51f82..c0a850eac 100644 --- a/e2e/kube-burner/concurrent-builds.yaml +++ b/e2e/kube-burner/concurrent-builds.yaml @@ -9,10 +9,6 @@ spec: url: ${ES_SERVER} metadata: collection: false - prometheus: - es_url: ${ES_SERVER} - prom_token: ${PROMETHEUS_TOKEN} - prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 workload: name: kube-burner args: diff --git a/e2e/kube-burner/configmap.yaml b/e2e/kube-burner/configmap.yaml index e6ee8b971..f74a6587f 100644 --- a/e2e/kube-burner/configmap.yaml +++ b/e2e/kube-burner/configmap.yaml @@ -8,11 +8,7 @@ spec: elasticsearch: url: ${ES_SERVER} metadata: - collection: true - prometheus: - es_url: ${ES_SERVER} - prom_token: ${PROMETHEUS_TOKEN} - prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + collection: false workload: name: kube-burner args: diff --git a/e2e/kube-burner/max-namespaces.yaml b/e2e/kube-burner/max-namespaces.yaml index db17594d1..08283dd26 100644 --- a/e2e/kube-burner/max-namespaces.yaml +++ b/e2e/kube-burner/max-namespaces.yaml @@ -9,10 +9,6 @@ spec: url: ${ES_SERVER} metadata: collection: false - prometheus: - es_url: ${ES_SERVER} - prom_token: ${PROMETHEUS_TOKEN} - prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 workload: name: kube-burner args: diff --git a/e2e/kube-burner/max-services.yaml b/e2e/kube-burner/max-services.yaml index dc737d5ad..49b94b950 100644 --- a/e2e/kube-burner/max-services.yaml +++ b/e2e/kube-burner/max-services.yaml @@ -9,10 +9,6 @@ spec: url: ${ES_SERVER} metadata: collection: false - prometheus: - es_url: ${ES_SERVER} - prom_token: ${PROMETHEUS_TOKEN} - prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 workload: name: kube-burner args: diff --git a/e2e/kube-burner/node-density-cni-networkpolicy.yaml b/e2e/kube-burner/node-density-cni-networkpolicy.yaml index 832076170..3825bbe82 100644 --- a/e2e/kube-burner/node-density-cni-networkpolicy.yaml +++ b/e2e/kube-burner/node-density-cni-networkpolicy.yaml @@ -9,10 +9,6 @@ spec: url: ${ES_SERVER} metadata: collection: false - prometheus: - es_url: ${ES_SERVER} - prom_token: ${PROMETHEUS_TOKEN} - prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 workload: name: kube-burner args: diff --git a/e2e/kube-burner/node-density-cni.yaml b/e2e/kube-burner/node-density-cni.yaml index 147ee9e18..3778945fc 100644 --- a/e2e/kube-burner/node-density-cni.yaml +++ b/e2e/kube-burner/node-density-cni.yaml @@ -9,10 +9,6 @@ spec: url: ${ES_SERVER} metadata: collection: false - prometheus: - es_url: ${ES_SERVER} - prom_token: ${PROMETHEUS_TOKEN} - prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 workload: name: kube-burner args: diff --git a/e2e/kube-burner/node-density-heavy.yaml b/e2e/kube-burner/node-density-heavy.yaml index 10d00b9b4..3faedb7f1 100644 --- a/e2e/kube-burner/node-density-heavy.yaml +++ b/e2e/kube-burner/node-density-heavy.yaml @@ -9,10 +9,6 @@ spec: url: ${ES_SERVER} metadata: collection: false - prometheus: - es_url: ${ES_SERVER} - prom_token: ${PROMETHEUS_TOKEN} - prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 workload: name: kube-burner args: diff --git a/e2e/kube-burner/node-density.yaml b/e2e/kube-burner/node-density.yaml index 4f00d84fd..ac8be5ef4 100644 --- a/e2e/kube-burner/node-density.yaml +++ b/e2e/kube-burner/node-density.yaml @@ -9,10 +9,6 @@ spec: url: ${ES_SERVER} metadata: collection: false - prometheus: - es_url: ${ES_SERVER} - prom_token: ${PROMETHEUS_TOKEN} - prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 workload: name: kube-burner args: diff --git a/e2e/scale-openshift/scale_down.yaml b/e2e/scale-openshift/scale_down.yaml index 8dfebbece..b950e2f05 100644 --- a/e2e/scale-openshift/scale_down.yaml +++ b/e2e/scale-openshift/scale_down.yaml @@ -7,7 +7,7 @@ spec: metadata: collection: true system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} diff --git a/e2e/smallfile/smallfile_hostpath.yaml b/e2e/smallfile/smallfile_hostpath.yaml index 573714502..465adb6d5 100644 --- a/e2e/smallfile/smallfile_hostpath.yaml +++ b/e2e/smallfile/smallfile_hostpath.yaml @@ -9,7 +9,7 @@ spec: # to separate this test run from everyone else's clustername: test_ci system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -18,7 +18,7 @@ spec: url: ${ES_SERVER} index_name: ripsaw-smallfile metadata: - collection: true + collection: false hostpath: /mnt/vda1/smallfile workload: name: smallfile diff --git a/e2e/uperf/uperf_hostnetwork.yaml b/e2e/uperf/uperf_hostnetwork.yaml index 2e0c1a35a..4c655639b 100644 --- a/e2e/uperf/uperf_hostnetwork.yaml +++ b/e2e/uperf/uperf_hostnetwork.yaml @@ -5,7 +5,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -14,7 +14,7 @@ spec: url: ${ES_SERVER} index_name: ripsaw-uperf metadata: - collection: true + collection: false cleanup: false workload: name: uperf diff --git a/e2e/uperf/uperf_networkpolicy.yaml b/e2e/uperf/uperf_networkpolicy.yaml index 3470bc5c0..1f5bfd4df 100644 --- a/e2e/uperf/uperf_networkpolicy.yaml +++ b/e2e/uperf/uperf_networkpolicy.yaml @@ -5,7 +5,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -14,7 +14,7 @@ spec: url: ${ES_SERVER} index_name: ripsaw-uperf metadata: - collection: true + collection: false cleanup: false workload: name: uperf diff --git a/e2e/uperf/uperf_resources.yaml b/e2e/uperf/uperf_resources.yaml index 46c7d43eb..288d2bb3a 100644 --- a/e2e/uperf/uperf_resources.yaml +++ b/e2e/uperf/uperf_resources.yaml @@ -5,7 +5,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -14,7 +14,7 @@ spec: url: ${ES_SERVER} index_name: ripsaw-uperf metadata: - collection: true + collection: false cleanup: false workload: name: uperf diff --git a/e2e/uperf/uperf_serviceip.yaml b/e2e/uperf/uperf_serviceip.yaml index 5a1fda9af..b12a1eef4 100644 --- a/e2e/uperf/uperf_serviceip.yaml +++ b/e2e/uperf/uperf_serviceip.yaml @@ -5,7 +5,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -14,7 +14,7 @@ spec: url: ${ES_SERVER} index_name: ripsaw-uperf metadata: - collection: true + collection: false cleanup: false workload: name: uperf diff --git a/e2e/uperf/uperf_serviceip_nodeport.yaml b/e2e/uperf/uperf_serviceip_nodeport.yaml index 71fd86423..e0175d9b2 100644 --- a/e2e/uperf/uperf_serviceip_nodeport.yaml +++ b/e2e/uperf/uperf_serviceip_nodeport.yaml @@ -5,7 +5,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -14,7 +14,7 @@ spec: url: ${ES_SERVER} index_name: ripsaw-uperf metadata: - collection: true + collection: false cleanup: false workload: name: uperf diff --git a/e2e/vegeta/vegeta_hostnetwork.yaml b/e2e/vegeta/vegeta_hostnetwork.yaml index 3eecec5c5..771a11b4b 100644 --- a/e2e/vegeta/vegeta_hostnetwork.yaml +++ b/e2e/vegeta/vegeta_hostnetwork.yaml @@ -6,7 +6,7 @@ metadata: namespace: benchmark-operator spec: system_metrics: - collection: true + collection: false prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 es_url: ${ES_SERVER} prom_token: ${PROMETHEUS_TOKEN} @@ -15,7 +15,7 @@ spec: url: ${ES_SERVER} index_name: ripsaw-vegeta metadata: - collection: true + collection: false workload: name: vegeta args: From 639da33ad59894d0031237d8eb9f800005ddc261 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Fri, 3 Jun 2022 10:35:30 +0200 Subject: [PATCH 212/249] Fix tests race condition Signed-off-by: Raul Sevilla --- cli/tests/clients/test_k8s.py | 1 + cli/tests/resources/job.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/cli/tests/clients/test_k8s.py b/cli/tests/clients/test_k8s.py index ddc5211aa..f20faa31b 100755 --- a/cli/tests/clients/test_k8s.py +++ b/cli/tests/clients/test_k8s.py @@ -33,6 +33,7 @@ def test_get_pod_logs(self, cluster): def test_get_jobs(self, cluster, test_job): label_selector = "app=test-job" jobs = cluster.get_jobs(label_selector=label_selector, namespace="default") + cluster.wait_for_pods("job-name=busybox", "default") assert len(jobs.items) == 1 assert jobs.items[0].metadata.name == "busybox" diff --git a/cli/tests/resources/job.yaml b/cli/tests/resources/job.yaml index 976700ca1..6f7f611ef 100755 --- a/cli/tests/resources/job.yaml +++ b/cli/tests/resources/job.yaml @@ -16,3 +16,4 @@ spec: imagePullPolicy: IfNotPresent name: busybox restartPolicy: Never + terminationGracePeriodSeconds: 0 From 53e5a00f310bea1fb2dd2cae59fa94c111b4481f Mon Sep 17 00:00:00 2001 From: Sharada Vetsa Date: Fri, 3 Jun 2022 05:05:23 -0400 Subject: [PATCH 213/249] Point redis image to multi-arch image on docker.io to support arm architecture (#752) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Point to redis image to multi-arch image on quay.io Update mount point for redis to reflect mount point needed for multiarch image to test arm and amd64. Co-authored-by: Raúl Sevilla --- config/manager/manager.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 07bd665c6..d6ec0fe76 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -63,7 +63,7 @@ spec: initialDelaySeconds: 5 periodSeconds: 10 - name: redis-master - image: bitnami/redis:latest + image: quay.io/cloud-bulldozer/redis:latest env: - name: MASTER value: "true" @@ -77,7 +77,7 @@ spec: limits: cpu: "2.0" volumeMounts: - - mountPath: /redis-master-data + - mountPath: /data name: data serviceAccountName: operator terminationGracePeriodSeconds: 10 From c6a0c5ba5a6bf09238f2ebfa579f7c83f5334ff2 Mon Sep 17 00:00:00 2001 From: Vishnu Challa Date: Wed, 1 Jun 2022 22:45:32 -0400 Subject: [PATCH 214/249] Removing unused client_store template from iperf template assets --- roles/iperf3/templates/client_store.yml.j2 | 72 ---------------------- 1 file changed, 72 deletions(-) delete mode 100644 roles/iperf3/templates/client_store.yml.j2 diff --git a/roles/iperf3/templates/client_store.yml.j2 b/roles/iperf3/templates/client_store.yml.j2 deleted file mode 100644 index d2df2e8b5..000000000 --- a/roles/iperf3/templates/client_store.yml.j2 +++ /dev/null @@ -1,72 +0,0 @@ ---- -kind: Job -apiVersion: batch/v1 -metadata: - name: 'iperf3-client-{{ item.status.podIP }}-{{ trunc_uuid }}' - namespace: '{{ operator_namespace }}' -spec: - ttlSecondsAfterFinished: 600 - template: - metadata: - labels: - app: iperf3-bench-client-{{ trunc_uuid }} - benchmark-uuid: {{ uuid }} -{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} - annotations: -{% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" -{% endfor %} -{% for annotation, value in workload_args.client_annotations.items() %} - "{{annotation}}": "{{value}}" -{% endfor %} -{% endif %} - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - iperf3-bench-server-{{ trunc_uuid }} - topologyKey: kubernetes.io/hostname -{% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" -{% endif %} -{% if workload_args.hostnetwork is sameas true %} - hostNetwork: true - serviceAccountName: benchmark-operator -{% endif %} - containers: - - name: benchmark - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/iperf3:latest') }} - imagePullPolicy: Always - command: ["/bin/sh", "-c"] - args: - - "mkdir -p {{results.path}}/iperf3-{{ uuid }}; - iperf3 -c {{ item.status.podIP }} -J - {% if workload_args.port is defined %} -p {{ workload_args.port }} {% endif %} - {% if workload_args.transmit_type is defined and workload_args.transmit_value is defined %} --{{ workload_args.transmit_type }} {{ workload_args.transmit_value }} {% endif %} - {% if workload_args.length_buffer is defined %} -l {{ workload_args.length_buffer }} {% endif %} - {% if workload_args.streams is defined %} -P {{ workload_args.streams }} {% endif %} - {% if workload_args.window_size is defined %} -w {{ workload_args.window_size }} {% endif %} - {% if workload_args.mss is defined %} -M {{ workload_args.mss }} {% endif %} - {% if workload_args.ip_tos is defined %} -S {{ workload_args.ip_tos }} {% endif %} - {% if workload_args.omit_start is defined %} -O {{ workload_args.omit_start }} {% endif %} - {% if workload_args.extra_options_client is defined %} {{ workload_args.extra_options_client }} {% endif %} - >> {{results.path}}/iperf3-{{ uuid }}/iperf_result.json" - volumeMounts: - - name: result-data - mountPath: "{{results.path}}" - volumes: - - name: result-data - persistentVolumeClaim: - claimName: result-volume -{% if workload_args.pin_client is defined %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' -{% endif %} - restartPolicy: Never From 61f325e50487eeb7a05e1ed1f258dffaf45ee126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Mon, 6 Jun 2022 10:55:49 +0200 Subject: [PATCH 215/249] Run tests for modified files (#764) * Rename test files to match with role names Signed-off-by: Raul Sevilla * Execute tests only for changed files Signed-off-by: Raul Sevilla * Add kustomization.yaml to test_map Signed-off-by: Raul Sevilla * Fetch all history Signed-off-by: Raul Sevilla * Always run Junit report Signed-off-by: Raul Sevilla * Rename fs-drift CRs Signed-off-by: Raul Sevilla --- .github/workflows/e2e.yml | 22 ++++++----- e2e/001-scale_openshift.bats | 10 ++--- e2e/{017-fs_drift.bats => 017-fs-drift.bats} | 8 ++-- ...-generator.bats => 018-log_generator.bats} | 5 ++- e2e/019-api_load.bats | 2 +- .../fs_drift.yaml => fs-drift/fs-drift.yaml} | 0 .../fs-drift_hostpath.yaml} | 0 e2e/fs-drift/fs_drift.yaml | 31 +++++++++++++++ e2e/fs-drift/fs_drift_hostpath.yaml | 32 +++++++++++++++ .../log_generator.yaml | 0 e2e/run_tests_ci.sh | 39 +++++++++++++++++++ .../scale_down.yaml | 0 .../scale_up.yaml | 0 e2e/test_map | 21 ++++++++++ 14 files changed, 148 insertions(+), 22 deletions(-) rename e2e/{017-fs_drift.bats => 017-fs-drift.bats} (80%) rename e2e/{018-log-generator.bats => 018-log_generator.bats} (80%) rename e2e/{fs_drift/fs_drift.yaml => fs-drift/fs-drift.yaml} (100%) rename e2e/{fs_drift/fs_drift_hostpath.yaml => fs-drift/fs-drift_hostpath.yaml} (100%) create mode 100644 e2e/fs-drift/fs_drift.yaml create mode 100644 e2e/fs-drift/fs_drift_hostpath.yaml rename e2e/{log-generator => log_generator}/log_generator.yaml (100%) create mode 100755 e2e/run_tests_ci.sh rename e2e/{scale-openshift => scale_openshift}/scale_down.yaml (100%) rename e2e/{scale-openshift => scale_openshift}/scale_up.yaml (100%) create mode 100644 e2e/test_map diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9a7c0b6a0..2e7ed3299 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -31,25 +31,27 @@ jobs: if: github.event_name != 'workflow_dispatch' - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} if: github.event_name == 'pull_request_target' - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 + with: + fetch-depth: 0 if: "github.event_name == 'workflow_dispatch' || github.event_name == 'push'" - name: Install bats run: make install-bats - name: Authenticate against OCP cluster - uses: redhat-actions/oc-login@v1 - with: - openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }} - openshift_username: ${{ secrets.OPENSHIFT_USER }} - openshift_password: ${{ secrets.OPENSHIFT_PASSWORD }} - insecure_skip_tls_verify: true + run: oc login -u ${OPENSHIFT_USER} -p ${OPENSHIFT_PASSWORD} ${OPENSHIFT_SERVER} --insecure-skip-tls-verify=true + env: + OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }} + OPENSHIFT_USER: ${{ secrets.OPENSHIFT_USER }} + OPENSHIFT_PASSWORD: ${{ secrets.OPENSHIFT_PASSWORD }} - name: Login in quay run: podman login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN} @@ -64,7 +66,7 @@ jobs: run: kubectl rollout status -n benchmark-operator --timeout=5m deploy/benchmark-controller-manager - name: Execute e2e tests - run: make e2e-tests + run: e2e/run_tests_ci.sh env: TERM: linux @@ -81,6 +83,6 @@ jobs: - name: Publish Test Report uses: mikepenz/action-junit-report@v2.4.3 - if: failure() # always run even if any of the previous steps fails + if: always() with: report_paths: e2e/report.xml diff --git a/e2e/001-scale_openshift.bats b/e2e/001-scale_openshift.bats index 28da05b7c..a252ba2dd 100755 --- a/e2e/001-scale_openshift.bats +++ b/e2e/001-scale_openshift.bats @@ -8,22 +8,22 @@ load helpers.bash ES_INDEX=openshift-cluster-timings -@test "scale-up" { - CR=scale-openshift/scale_up.yaml +@test "scale_openshift-up" { + CR=scale_openshift/scale_up.yaml CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" check_benchmark 1200 # Make node no schedulable as soon as benchmark finishes kubectl cordon $(kubectl get node --sort-by='{.metadata.creationTimestamp}' -o name | tail -1) - # Annotate machine to ensure it's the one deleted in the scale-down test + # Annotate machine to ensure it's the one deleted in the scale_openshift-down test # Reference: https://github.com/openshift/machine-api-operator/blob/master/FAQ.md#what-decides-which-machines-to-destroy-when-a-machineset-is-scaled-down kubectl -n openshift-machine-api annotate $(kubectl get machine -n openshift-machine-api --sort-by='{.metadata.creationTimestamp}' -o name | tail -1) machine.openshift.io/cluster-api-delete-machine=true --overwrite check_es } -@test "scale-down" { - CR=scale-openshift/scale_down.yaml +@test "scale_openshift-down" { + CR=scale_openshift/scale_down.yaml CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" diff --git a/e2e/017-fs_drift.bats b/e2e/017-fs-drift.bats similarity index 80% rename from e2e/017-fs_drift.bats rename to e2e/017-fs-drift.bats index 6970d0180..21a3302de 100755 --- a/e2e/017-fs_drift.bats +++ b/e2e/017-fs-drift.bats @@ -7,8 +7,8 @@ load helpers.bash indexes=(ripsaw-fs-drift-results ripsaw-fs-drift-rsptimes ripsaw-fs-drift-rates-over-time) -@test "fs_drift-standard" { - CR=fs_drift/fs_drift.yaml +@test "fs-drift-standard" { + CR=fs-drift/fs-drift.yaml CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" @@ -17,8 +17,8 @@ indexes=(ripsaw-fs-drift-results ripsaw-fs-drift-rsptimes ripsaw-fs-drift-rates- } -@test "fs_drift-hostpath" { - CR=fs_drift/fs_drift_hostpath.yaml +@test "fs-drift-hostpath" { + CR=fs-drift/fs-drift_hostpath.yaml CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" diff --git a/e2e/018-log-generator.bats b/e2e/018-log_generator.bats similarity index 80% rename from e2e/018-log-generator.bats rename to e2e/018-log_generator.bats index 072f0373c..425564d37 100755 --- a/e2e/018-log-generator.bats +++ b/e2e/018-log_generator.bats @@ -7,8 +7,8 @@ load helpers.bash ES_INDEX=log-generator-results -@test "log-generator-standard" { - CR=log-generator/log_generator.yaml +@test "log_generator-standard" { + CR=log_generator/log_generator.yaml CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - get_uuid "${CR_NAME}" @@ -23,3 +23,4 @@ setup_file() { teardown() { basic_teardown } + diff --git a/e2e/019-api_load.bats b/e2e/019-api_load.bats index 6dd4a602b..8595c1d5d 100755 --- a/e2e/019-api_load.bats +++ b/e2e/019-api_load.bats @@ -5,7 +5,7 @@ load helpers.bash -@test "api-load-standard" { +@test "api_load-standard" { CR=api_load/api_load.yaml CR_NAME=$(get_benchmark_name ${CR}) envsubst < ${CR} | kubectl apply -f - diff --git a/e2e/fs_drift/fs_drift.yaml b/e2e/fs-drift/fs-drift.yaml similarity index 100% rename from e2e/fs_drift/fs_drift.yaml rename to e2e/fs-drift/fs-drift.yaml diff --git a/e2e/fs_drift/fs_drift_hostpath.yaml b/e2e/fs-drift/fs-drift_hostpath.yaml similarity index 100% rename from e2e/fs_drift/fs_drift_hostpath.yaml rename to e2e/fs-drift/fs-drift_hostpath.yaml diff --git a/e2e/fs-drift/fs_drift.yaml b/e2e/fs-drift/fs_drift.yaml new file mode 100644 index 000000000..fd168c2f5 --- /dev/null +++ b/e2e/fs-drift/fs_drift.yaml @@ -0,0 +1,31 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fs-drift-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + # where elastic search is running + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-fs-drift + metadata: + collection: true + workload: + name: fs-drift + args: + worker_pods: 1 + threads: 5 + max_file_size_kb: 4 + max_files: 1000 + duration: 240 + debug: true + response_times: true diff --git a/e2e/fs-drift/fs_drift_hostpath.yaml b/e2e/fs-drift/fs_drift_hostpath.yaml new file mode 100644 index 000000000..ea5bb6d43 --- /dev/null +++ b/e2e/fs-drift/fs_drift_hostpath.yaml @@ -0,0 +1,32 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fs-drift-hostpath-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + # where elastic search is running + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-fs-drift + metadata: + collection: false + hostpath: /mnt/vda1/fs_drift + workload: + name: fs-drift + args: + worker_pods: 1 + threads: 5 + max_file_size_kb: 4 + max_files: 1000 + duration: 240 + debug: true + response_times: true diff --git a/e2e/log-generator/log_generator.yaml b/e2e/log_generator/log_generator.yaml similarity index 100% rename from e2e/log-generator/log_generator.yaml rename to e2e/log_generator/log_generator.yaml diff --git a/e2e/run_tests_ci.sh b/e2e/run_tests_ci.sh new file mode 100755 index 000000000..8f63b8cfb --- /dev/null +++ b/e2e/run_tests_ci.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +select_tests(){ + for f in $(git diff --name-only origin/master); do + found=false + while read -r t; do + regex=$(echo ${t} | cut -d : -f 1) + test_filter=$(echo ${t} | cut -d : -f 2) + # File found in test_map + if echo $f | grep -qE ${regex}; then + found=true + # Check if test was previously added + if ! echo "${BATS_TESTS}" | grep -q "${test_filter}"; then + if [[ ${BATS_TESTS} != "" ]]; then + BATS_TESTS+="|" + fi + BATS_TESTS+="${test_filter}" + fi + break + fi + done < e2e/test_map + # If one of the modified files is not present in the test_map, we exit the loop and + # run all tests + if [[ ${found} == "false" ]]; then + echo "File ${f} not found in test_map, running all tests" + break + fi + done + + # No exporting BATS_TESTS means, run all tests + if [[ ${found} == "true" ]]; then + export BATS_TESTS + fi +} + +if [[ -z ${BATS_TESTS} ]]; then + select_tests +fi +make e2e-tests diff --git a/e2e/scale-openshift/scale_down.yaml b/e2e/scale_openshift/scale_down.yaml similarity index 100% rename from e2e/scale-openshift/scale_down.yaml rename to e2e/scale_openshift/scale_down.yaml diff --git a/e2e/scale-openshift/scale_up.yaml b/e2e/scale_openshift/scale_up.yaml similarity index 100% rename from e2e/scale-openshift/scale_up.yaml rename to e2e/scale_openshift/scale_up.yaml diff --git a/e2e/test_map b/e2e/test_map new file mode 100644 index 000000000..e45350257 --- /dev/null +++ b/e2e/test_map @@ -0,0 +1,21 @@ +kustomization\.yaml|Makefile|\.md$|servicemesh|benchmark_state|LICENSE|PROJECT|^charts:skip +api_load:api-load +backpackbackpack +byowl:byowl +flent:flent +fs-drift:fs-drift +hammerdb:hammerdb +image_pull:image_pull +iperf3:iperf3 +fio/|fio_distributed/|ceph_osd_cache_drop|kernel_cache_drop:fio +kube-burner:kube-burner +log_generator:log_generator +pgbench:pgbench +scale_openshift:scale_openshift +smallfile:smallfile +stressng:stressng +sysbench:sysbench +testpmd:testpmd +uperf:uperf +vegeta:vegeta +ycsb:ycsb From f4fe8fbc4125083336da2dd0bbd4fc44db9e6123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Tue, 7 Jun 2022 10:08:02 +0200 Subject: [PATCH 216/249] Use cluster-admin ClusterRoleBinding (#763) * Use cluster-admin ClusterRoleBinding Signed-off-by: Raul Sevilla * Remove test not present in benchmark-wrapper Signed-off-by: Raul Sevilla * The new redis image runs with root user, removing runAsNonRoot due to permissions issues Signed-off-by: Raul Sevilla * Set restartPolicy to Never Signed-off-by: Raul Sevilla --- charts/benchmark-operator/README.md | 1 - ...e-binding.yaml => clusterrolebinding.yaml} | 8 +- .../templates/kubevirt-cluster-role.yaml | 76 ------ charts/benchmark-operator/templates/psp.yaml | 25 -- .../templates/role-binding.yaml | 13 - charts/benchmark-operator/templates/role.yaml | 103 ------- charts/benchmark-operator/values.yaml | 7 - .../ripsaw.cloudbulldozer.io_benchmarks.yaml | 3 - config/manager/manager.yaml | 4 +- .../rbac/auth_proxy_client_clusterrole.yaml | 9 - config/rbac/auth_proxy_role.yaml | 17 -- config/rbac/auth_proxy_role_binding.yaml | 12 - config/rbac/auth_proxy_service.yaml | 14 - config/rbac/benchmark_editor_role.yaml | 24 -- config/rbac/benchmark_viewer_role.yaml | 20 -- .../rbac/clusterrolebinding.yaml | 8 +- config/rbac/kustomization.yaml | 18 +- config/rbac/leader_election_role.yaml | 37 --- config/rbac/leader_election_role_binding.yaml | 12 - config/rbac/pod_security_policy.yaml | 25 -- config/rbac/role.yaml | 257 ------------------ config/rbac/role_binding.yaml | 25 -- config/rbac/service_account.yaml | 3 +- docs/metadata.md | 68 ----- docs/scale_openshift.md | 46 ---- e2e/001-scale_openshift.bats | 5 - e2e/004-backpack.bats | 6 - e2e/008-kube-burner.bats | 2 - e2e/backpack/backpack-daemonset.yaml | 1 - e2e/backpack/backpack-init.yaml | 1 - e2e/run_tests_ci.sh | 7 +- e2e/scale_openshift/scale_down.yaml | 1 - e2e/scale_openshift/scale_up.yaml | 1 - playbooks/templates/metadata.yml.j2 | 3 - playbooks/templates/metadata_pod.yml.j2 | 3 - resources/backpack_role.yaml | 49 ---- resources/kube-burner-role.yml | 19 -- resources/scale_role.yaml | 34 --- roles/backpack/templates/backpack.yml | 2 +- roles/byowl/templates/workload.yml | 1 + .../kube-burner/templates/kube-burner.yml.j2 | 2 +- roles/scale_openshift/templates/scale.yml | 2 +- roles/uperf/templates/server.yml.j2 | 2 +- tests/test_api_load.sh | 35 --- tests/test_backpack.sh | 51 ---- tests/test_crs/valid_scale_down.yaml | 1 - tests/test_crs/valid_scale_up.yaml | 1 - tests/test_kubeburner.sh | 57 ---- tests/test_list | 3 - tests/test_scale_openshift.sh | 2 - 50 files changed, 20 insertions(+), 1106 deletions(-) rename charts/benchmark-operator/templates/{self-provisioner-cluster-role-binding.yaml => clusterrolebinding.yaml} (79%) delete mode 100644 charts/benchmark-operator/templates/kubevirt-cluster-role.yaml delete mode 100644 charts/benchmark-operator/templates/psp.yaml delete mode 100644 charts/benchmark-operator/templates/role-binding.yaml delete mode 100644 charts/benchmark-operator/templates/role.yaml delete mode 100644 config/rbac/auth_proxy_client_clusterrole.yaml delete mode 100644 config/rbac/auth_proxy_role.yaml delete mode 100644 config/rbac/auth_proxy_role_binding.yaml delete mode 100644 config/rbac/auth_proxy_service.yaml delete mode 100644 config/rbac/benchmark_editor_role.yaml delete mode 100644 config/rbac/benchmark_viewer_role.yaml rename resources/self_provisioner_binding.yaml => config/rbac/clusterrolebinding.yaml (73%) delete mode 100644 config/rbac/leader_election_role.yaml delete mode 100644 config/rbac/leader_election_role_binding.yaml delete mode 100644 config/rbac/pod_security_policy.yaml delete mode 100644 config/rbac/role.yaml delete mode 100644 config/rbac/role_binding.yaml delete mode 100644 resources/backpack_role.yaml delete mode 100644 resources/kube-burner-role.yml delete mode 100644 resources/scale_role.yaml delete mode 100755 tests/test_api_load.sh delete mode 100755 tests/test_backpack.sh delete mode 100755 tests/test_kubeburner.sh diff --git a/charts/benchmark-operator/README.md b/charts/benchmark-operator/README.md index da778f107..0f2d4a83a 100644 --- a/charts/benchmark-operator/README.md +++ b/charts/benchmark-operator/README.md @@ -21,7 +21,6 @@ A Helm chart for Kubernetes | operator.replicaCount | int | `1` | how many replicas for the operator deployment | | operator.resources.limits.cpu | string | `"0.1"` | | | operator.tolerations[0] | object | `{"effect":"NoSchedule","key":"role","value":"workload"}` | schedule on a workload node even if it's tainted | -| rbac | object | `{"clusterRoles":{"kubevirt":false,"selfProvisioner":false}}` | map for custom cluster roles for certain benchmarks like kubevirt | ---------------------------------------------- Autogenerated from chart metadata using [helm-docs v1.4.0](https://github.com/norwoodj/helm-docs/releases/v1.4.0) diff --git a/charts/benchmark-operator/templates/self-provisioner-cluster-role-binding.yaml b/charts/benchmark-operator/templates/clusterrolebinding.yaml similarity index 79% rename from charts/benchmark-operator/templates/self-provisioner-cluster-role-binding.yaml rename to charts/benchmark-operator/templates/clusterrolebinding.yaml index 40b9faaef..8a4b72544 100644 --- a/charts/benchmark-operator/templates/self-provisioner-cluster-role-binding.yaml +++ b/charts/benchmark-operator/templates/clusterrolebinding.yaml @@ -1,14 +1,14 @@ -{{ if .Values.rbac.clusterRoles.selfProvisioner }} -apiVersion: rbac.authorization.k8s.io/v1 +--- kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "benchmark-operator.fullname" . }} + namespace: {{ .Release.Namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: self-provisioner + name: cluster-admin subjects: - kind: ServiceAccount name: {{ include "benchmark-operator.fullname" . }} namespace: {{ .Release.Namespace }} -{{ end }} \ No newline at end of file diff --git a/charts/benchmark-operator/templates/kubevirt-cluster-role.yaml b/charts/benchmark-operator/templates/kubevirt-cluster-role.yaml deleted file mode 100644 index 02e33f6ff..000000000 --- a/charts/benchmark-operator/templates/kubevirt-cluster-role.yaml +++ /dev/null @@ -1,76 +0,0 @@ -{{ if .Values.rbac.clusterRoles.kubevirt }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "benchmark-operator.fullname" . }}-kubevirt -rules: -- apiGroups: - - rbac.authorization.k8s.io - resources: - - roles - - rolebindings - - clusterroles - - clusterrolebindings - - daemonsets - verbs: - - '*' -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - get - - list -- apiGroups: - - rbac.authorization.k8s.io - resources: - - nodes - verbs: - - get - - list -- apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachineinstances/console - - virtualmachineinstances/vnc - verbs: - - get -- apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - - virtualmachines - - virtualmachineinstancepresets - - virtualmachineinstancereplicasets - verbs: - - get - - delete - - create - - update - - patch - - list - - watch - - deletecollection -- apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubevirt-config - verbs: - - update - - get - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "benchmark-operator.fullname" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "benchmark-operator.fullname" . }}-kubevirt -subjects: -- kind: ServiceAccount - name: {{ include "benchmark-operator.fullname" . }} - namespace: {{ .Release.Namespace }} -{{ end }} \ No newline at end of file diff --git a/charts/benchmark-operator/templates/psp.yaml b/charts/benchmark-operator/templates/psp.yaml deleted file mode 100644 index a8e05fb49..000000000 --- a/charts/benchmark-operator/templates/psp.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: privileged -spec: - privileged: true - allowPrivilegeEscalation: true - allowedCapabilities: - - '*' - volumes: - - '*' - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - hostIPC: true - hostPID: true - runAsUser: - rule: 'RunAsAny' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' diff --git a/charts/benchmark-operator/templates/role-binding.yaml b/charts/benchmark-operator/templates/role-binding.yaml deleted file mode 100644 index 638a77f58..000000000 --- a/charts/benchmark-operator/templates/role-binding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "benchmark-operator.fullname" . }} - namespace: {{ .Release.Namespace }} -subjects: -- kind: ServiceAccount - name: {{ include "benchmark-operator.fullname" . }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: Role - name: {{ include "benchmark-operator.fullname" . }} - apiGroup: rbac.authorization.k8s.io diff --git a/charts/benchmark-operator/templates/role.yaml b/charts/benchmark-operator/templates/role.yaml deleted file mode 100644 index 2e9711eb3..000000000 --- a/charts/benchmark-operator/templates/role.yaml +++ /dev/null @@ -1,103 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "benchmark-operator.fullname" . }} - namespace: {{ .Release.Namespace }} -rules: -- apiGroups: - - "" - resources: - - pods - - daemonsets - - services - - endpoints - - persistentvolumeclaims - - virtualmachineinstances - - events - - configmaps - - secrets - - pods/log - verbs: - - '*' -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - apps - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - - deployments/finalizers - verbs: - - '*' -- apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - create -- apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - verbs: - - '*' -- apiGroups: - - ripsaw.cloudbulldozer.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - batch - - extensions - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use - resourceNames: - - privileged -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use -- apiGroups: - - hyperfoil.io - resources: - - hyperfoils - verbs: - - '*' -- apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch - - create - - update - - patch - - delete diff --git a/charts/benchmark-operator/values.yaml b/charts/benchmark-operator/values.yaml index bd315e3dc..7a4f47ecf 100644 --- a/charts/benchmark-operator/values.yaml +++ b/charts/benchmark-operator/values.yaml @@ -5,13 +5,6 @@ nameOverride: "" # -- (string) overrides naming of all resources fullnameOverride: "benchmark-operator" - -# -- map for custom cluster roles for certain benchmarks like kubevirt -rbac: - clusterRoles: - kubevirt: false - selfProvisioner: false - operator: # -- how many replicas for the operator deployment diff --git a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml index b4bb9efda..27df9b3c0 100644 --- a/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml +++ b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml @@ -135,9 +135,6 @@ spec: privileged: default: false type: boolean - serviceaccount: - default: default - type: string stockpileSkipTags: default: [] items: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d6ec0fe76..70dcdb2b1 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -22,8 +22,6 @@ spec: labels: control-plane: controller-manager spec: - securityContext: - runAsNonRoot: true containers: - name: manager args: @@ -79,7 +77,7 @@ spec: volumeMounts: - mountPath: /data name: data - serviceAccountName: operator + serviceAccountName: benchmark-operator terminationGracePeriodSeconds: 10 volumes: - name: data diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index 51a75db47..000000000 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 80e1857c5..000000000 --- a/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index ec7acc0a1..000000000 --- a/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml deleted file mode 100644 index 6cf656be1..000000000 --- a/config/rbac/auth_proxy_service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: controller-manager - name: controller-manager-metrics-service - namespace: system -spec: - ports: - - name: https - port: 8443 - targetPort: https - selector: - control-plane: controller-manager diff --git a/config/rbac/benchmark_editor_role.yaml b/config/rbac/benchmark_editor_role.yaml deleted file mode 100644 index 7d50193d5..000000000 --- a/config/rbac/benchmark_editor_role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# permissions for end users to edit benchmarks. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: benchmark-editor-role -rules: -- apiGroups: - - ripsaw.cloudbulldozer.io - resources: - - benchmarks - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - ripsaw.cloudbulldozer.io - resources: - - benchmarks/status - verbs: - - get diff --git a/config/rbac/benchmark_viewer_role.yaml b/config/rbac/benchmark_viewer_role.yaml deleted file mode 100644 index 9601801a7..000000000 --- a/config/rbac/benchmark_viewer_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# permissions for end users to view benchmarks. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: benchmark-viewer-role -rules: -- apiGroups: - - ripsaw.cloudbulldozer.io - resources: - - benchmarks - verbs: - - get - - list - - watch -- apiGroups: - - ripsaw.cloudbulldozer.io - resources: - - benchmarks/status - verbs: - - get diff --git a/resources/self_provisioner_binding.yaml b/config/rbac/clusterrolebinding.yaml similarity index 73% rename from resources/self_provisioner_binding.yaml rename to config/rbac/clusterrolebinding.yaml index fbc43c503..2996ee665 100644 --- a/resources/self_provisioner_binding.yaml +++ b/config/rbac/clusterrolebinding.yaml @@ -1,11 +1,13 @@ -apiVersion: rbac.authorization.k8s.io/v1 +--- kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: benchmark-operator-self-provisioner + name: operator-binding + namespace: benchmark-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: self-provisioner + name: cluster-admin subjects: - kind: ServiceAccount name: benchmark-operator diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index caef677fc..af5e06965 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -1,19 +1,3 @@ resources: -# All RBAC will be applied under this service account in -# the deployment namespace. You may comment out this resource -# if your manager will use a service account that exists at -# runtime. Be sure to update RoleBinding and ClusterRoleBinding -# subjects if changing service account names. - service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml -- pod_security_policy.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +- clusterrolebinding.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml deleted file mode 100644 index 4190ec805..000000000 --- a/config/rbac/leader_election_role.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# permissions to do leader election. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: leader-election-role -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml deleted file mode 100644 index 1d1321ed4..000000000 --- a/config/rbac/leader_election_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: leader-election-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: leader-election-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/pod_security_policy.yaml b/config/rbac/pod_security_policy.yaml deleted file mode 100644 index a8e05fb49..000000000 --- a/config/rbac/pod_security_policy.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: privileged -spec: - privileged: true - allowPrivilegeEscalation: true - allowedCapabilities: - - '*' - volumes: - - '*' - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - hostIPC: true - hostPID: true - runAsUser: - rule: 'RunAsAny' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml deleted file mode 100644 index 577062564..000000000 --- a/config/rbac/role.yaml +++ /dev/null @@ -1,257 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: manager-role -rules: - ## - ## Base operator rules - ## - - apiGroups: - - "" - resources: - - secrets - - pods - - pods/exec - - pods/log - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - apps - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - ## - ## Rules Ported from old namespaced role - ## - - apiGroups: - - "" - resources: - - pods - - daemonsets - - services - - endpoints - - persistentvolumeclaims - - virtualmachineinstances - - events - - configmaps - - secrets - - pods/log - verbs: - - '*' - - apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - apiGroups: - - apps - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - - deployments/finalizers - verbs: - - '*' - - apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - create - - apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - verbs: - - '*' - - apiGroups: - - ripsaw.cloudbulldozer.io - resources: - - '*' - verbs: - - '*' - - apiGroups: - - batch - - extensions - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use - resourceNames: - - privileged - - apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use - - apiGroups: - - hyperfoil.io - resources: - - hyperfoils - verbs: - - '*' - - apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - k8s.cni.cncf.io - resources: - - network-attachment-definitions - verbs: - - get - - list - - watch - - ## - ## Rules from old cluster role - ## - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - patch - - ## - ## Rules from kube-virt - ## - - apiGroups: - - rbac.authorization.k8s.io - resources: - - roles - - rolebindings - - clusterroles - - clusterrolebindings - - daemonsets - verbs: - - '*' - - apiGroups: - - apps - resources: - - daemonsets - verbs: - - get - - list - - apiGroups: - - rbac.authorization.k8s.io - resources: - - nodes - verbs: - - get - - list - - apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachineinstances/console - - virtualmachineinstances/vnc - verbs: - - get - - apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - - virtualmachines - - virtualmachineinstancepresets - - virtualmachineinstancereplicasets - verbs: - - get - - delete - - create - - update - - patch - - list - - watch - - deletecollection - - apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubevirt-config - verbs: - - update - - get - - patch - - ## - ## Rules from kernel-cache-drop - ## - - apiGroups: - - '' - resources: - - nodes - - pods - verbs: - - get - - list - - apiGroups: - - apps - resources: - - daemonsets - verbs: - - get - - list - - ## - ## Rules for ripsaw.cloudbulldozer.io/v1alpha1, Kind: Benchmark - ## - - apiGroups: - - ripsaw.cloudbulldozer.io - resources: - - benchmarks - - benchmarks/status - - benchmarks/finalizers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -#+kubebuilder:scaffold:rules diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml deleted file mode 100644 index 2ffba9b95..000000000 --- a/config/rbac/role_binding.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: manager-role -subjects: -- kind: ServiceAccount - name: operator - namespace: system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manager-self-provisioner -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: self-provisioner -subjects: -- kind: ServiceAccount - name: operator - namespace: system \ No newline at end of file diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index 38423bc86..36e029d5b 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -1,5 +1,6 @@ +--- apiVersion: v1 kind: ServiceAccount metadata: name: operator - namespace: system + namespace: benchmark-operator diff --git a/docs/metadata.md b/docs/metadata.md index 3388828fc..79871c1e2 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -36,9 +36,6 @@ the workload template with an init container section that looks like: ```jinja {% if metadata.collection is sameas true and metadata.targeted is sameas true %} -{% if metadata.serviceaccount != "default" %} - serviceAccountName: {{ metadata.serviceaccount }} -{% endif %} initContainers: - name: backpack-{{ trunc_uuid }} image: {{ metadata.image }} @@ -261,71 +258,6 @@ metadata: ssl: true ``` -### Additional k8s Information - -There are multiple kubernetes (k8s) based modules that are run in stockpile. -These modules are not accessable by default as the default service account -that runs the pod does not have the required privileges to view them. - -To allow the pods view permissions on the cluster you can apply -the following yaml to create a service account with the appropriate -privileges. - -You will need to change the namespace to fit with your environment -Note: You can also find this yaml in [backpack_role.yaml](../resources/backpack_role.yaml) - -```yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: backpack_role -rules: -- apiGroups: - - "*" - resources: - - "*" - verbs: - - get - - list - - watch ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: backpack-view - namespace: benchmark-operator ---- -apiVersion: v1 -kind: Secret -metadata: - name: backpack-view - namespace: benchmark-operator - annotations: - kubernetes.io/service-account.name: backpack-view -type: kubernetes.io/service-account-token ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: backpack-view - namespace: benchmark-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: backpack_role -subjects: -- kind: ServiceAccount - name: backpack-view - namespace: benchmark-operator -``` - -Once the `backpack-view` service account is created you can modify the default backpack service account setting: - -```yaml -metadata: - serviceaccount: backpack-view -``` - ### Stockpile tags Backpack leverages [stockpile](https://github.com/cloud-bulldozer/stockpile/) to collect metadata. Stockpile is basically a set of Ansible roles, these roles have different tags. It's possible to pass custom tags to the inner `ansible-playbook` command through the parameters `stockpileSkipTags` and `stockpileTags` from the metadata section. These parameters are translated to the Ansible's flags _--tags_ and _--skip-tags_ respectively. diff --git a/docs/scale_openshift.md b/docs/scale_openshift.md index a24d8aa68..b048162ea 100644 --- a/docs/scale_openshift.md +++ b/docs/scale_openshift.md @@ -12,8 +12,6 @@ The scale workload takes the following required variables: `scale` the target number of workers to scale to. -`serviceaccount` the service account to run as. Note the "default" account generally does not have enough privileges - Optional variables: `poll_interval` how long, in seconds, to wait between polls to see if the scaling is complete. default: 5 @@ -62,7 +60,6 @@ spec: name: scale_openshift args: scale: 25 - serviceaccount: scaler poll_interval: 2 post_sleep: 300 label: @@ -85,46 +82,3 @@ spec: *NOTE:* If the cluster is already at the desired scale the timings will still be captured and uploaded to Elasticsearch (if provided). The overall time will simply be the time it took the scale process to confirm that the cluster is at the correct scale. - -## Service Account and Permissions - -The default service account generally does not have the required visibility/permissions to allow the Scale job -to modify the cluster machine sets. An example service account setup has been provided in resources/scale_role.yaml -as well as described below: - -``` -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: scale_role -rules: -- apiGroups: - - "*" - resources: - - machines - - machinesets - - nodes - - infrastructures - verbs: - - '*' ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: scaler - namespace: benchmark-operator ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: scaler - namespace: benchmark-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: scale_role -subjects: -- kind: ServiceAccount - name: scaler - namespace: benchmark-operator -``` diff --git a/e2e/001-scale_openshift.bats b/e2e/001-scale_openshift.bats index a252ba2dd..1256230f8 100755 --- a/e2e/001-scale_openshift.bats +++ b/e2e/001-scale_openshift.bats @@ -34,14 +34,9 @@ ES_INDEX=openshift-cluster-timings setup_file() { # Prevent running scale down/up simultaneously export BATS_NO_PARALLELIZE_WITHIN_FILE=true - kubectl_exec apply -f ../resources/scale_role.yaml basic_setup } -teardown_file() { - kubectl_exec delete -f ../resources/scale_role.yaml -} - teardown() { basic_teardown } diff --git a/e2e/004-backpack.bats b/e2e/004-backpack.bats index 338a076ee..e67c8f11c 100755 --- a/e2e/004-backpack.bats +++ b/e2e/004-backpack.bats @@ -27,12 +27,6 @@ indexes=(cpu_vulnerabilities-metadata cpuinfo-metadata dmidecode-metadata k8s_no setup_file() { basic_setup - kubectl apply -f ../resources/backpack_role.yaml --overwrite -} - - -teardown_file() { - kubectl delete -f ../resources/backpack_role.yaml } teardown() { diff --git a/e2e/008-kube-burner.bats b/e2e/008-kube-burner.bats index 19c90aa79..ec8cb6d5e 100755 --- a/e2e/008-kube-burner.bats +++ b/e2e/008-kube-burner.bats @@ -83,12 +83,10 @@ ES_INDEX=ripsaw-kube-burner setup_file() { basic_setup - kubectl apply -f ../resources/kube-burner-role.yml kubectl apply -f kube-burner/configmap-cfg.yaml } teardown_file() { - kubectl delete -f ../resources/kube-burner-role.yml kubectl delete -f kube-burner/configmap-cfg.yaml } diff --git a/e2e/backpack/backpack-daemonset.yaml b/e2e/backpack/backpack-daemonset.yaml index 57fe3d3df..48a529d9d 100644 --- a/e2e/backpack/backpack-daemonset.yaml +++ b/e2e/backpack/backpack-daemonset.yaml @@ -10,6 +10,5 @@ spec: collection: true targeted: false privileged: true - serviceaccount: backpack-view workload: name: backpack diff --git a/e2e/backpack/backpack-init.yaml b/e2e/backpack/backpack-init.yaml index 3b8cfbcd2..4711e6138 100644 --- a/e2e/backpack/backpack-init.yaml +++ b/e2e/backpack/backpack-init.yaml @@ -9,7 +9,6 @@ spec: metadata: collection: true privileged: true - serviceaccount: backpack-view workload: name: byowl args: diff --git a/e2e/run_tests_ci.sh b/e2e/run_tests_ci.sh index 8f63b8cfb..1c36a0f75 100755 --- a/e2e/run_tests_ci.sh +++ b/e2e/run_tests_ci.sh @@ -23,14 +23,11 @@ select_tests(){ # run all tests if [[ ${found} == "false" ]]; then echo "File ${f} not found in test_map, running all tests" + BATS_TESTS=. break fi done - - # No exporting BATS_TESTS means, run all tests - if [[ ${found} == "true" ]]; then - export BATS_TESTS - fi + export BATS_TESTS } if [[ -z ${BATS_TESTS} ]]; then diff --git a/e2e/scale_openshift/scale_down.yaml b/e2e/scale_openshift/scale_down.yaml index b950e2f05..1091cd9cc 100644 --- a/e2e/scale_openshift/scale_down.yaml +++ b/e2e/scale_openshift/scale_down.yaml @@ -19,6 +19,5 @@ spec: name: scale_openshift args: scale: 0 - serviceaccount: scaler poll_interval: 2 debug: true diff --git a/e2e/scale_openshift/scale_up.yaml b/e2e/scale_openshift/scale_up.yaml index cb45b85fe..6096f8c97 100644 --- a/e2e/scale_openshift/scale_up.yaml +++ b/e2e/scale_openshift/scale_up.yaml @@ -19,6 +19,5 @@ spec: name: scale_openshift args: scale: 1 - serviceaccount: scaler poll_interval: 1 debug: true diff --git a/playbooks/templates/metadata.yml.j2 b/playbooks/templates/metadata.yml.j2 index 663469d70..ef0222f89 100644 --- a/playbooks/templates/metadata.yml.j2 +++ b/playbooks/templates/metadata.yml.j2 @@ -1,7 +1,4 @@ {% if metadata.collection is sameas true and metadata.targeted is sameas true %} -{% if metadata.serviceaccount != "default" %} - serviceAccountName: {{ metadata.serviceaccount }} -{% endif %} initContainers: - name: backpack image: {{ metadata.image }} diff --git a/playbooks/templates/metadata_pod.yml.j2 b/playbooks/templates/metadata_pod.yml.j2 index 6464e86d5..4d48aab77 100644 --- a/playbooks/templates/metadata_pod.yml.j2 +++ b/playbooks/templates/metadata_pod.yml.j2 @@ -1,7 +1,4 @@ {% if metadata.collection is sameas true and metadata.targeted is sameas true %} -{% if metadata.serviceaccount != "default" %} - serviceAccountName: {{ metadata.serviceaccount }} -{% endif %} initContainers: - name: backpack image: {{ metadata.image }} diff --git a/resources/backpack_role.yaml b/resources/backpack_role.yaml deleted file mode 100644 index a0f8ea69f..000000000 --- a/resources/backpack_role.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: backpack_role -rules: -- apiGroups: - - "*" - resources: - - "*" - verbs: - - get - - list - - watch -- apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use - resourceNames: - - privileged -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: backpack-view - namespace: benchmark-operator ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: binding-backpack_role - namespace: benchmark-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: backpack_role -subjects: -- kind: ServiceAccount - name: backpack-view - namespace: benchmark-operator diff --git a/resources/kube-burner-role.yml b/resources/kube-burner-role.yml deleted file mode 100644 index 281f5ddf7..000000000 --- a/resources/kube-burner-role.yml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-burner - namespace: benchmark-operator ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: binding-kube-burner - namespace: benchmark-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: -- kind: ServiceAccount - name: kube-burner - namespace: benchmark-operator diff --git a/resources/scale_role.yaml b/resources/scale_role.yaml deleted file mode 100644 index e0d7fe5b0..000000000 --- a/resources/scale_role.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: scale_role -rules: -- apiGroups: - - "*" - resources: - - machines - - machinesets - - nodes - - infrastructures - verbs: - - '*' ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: scaler - namespace: benchmark-operator ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: scaler - namespace: benchmark-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: scale_role -subjects: -- kind: ServiceAccount - name: scaler - namespace: benchmark-operator diff --git a/roles/backpack/templates/backpack.yml b/roles/backpack/templates/backpack.yml index ad48ff7e9..b9a2862f7 100755 --- a/roles/backpack/templates/backpack.yml +++ b/roles/backpack/templates/backpack.yml @@ -81,5 +81,5 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - serviceAccountName: {{ metadata.serviceaccount }} + serviceAccountName: benchmark-operator terminationGracePeriodSeconds: 30 diff --git a/roles/byowl/templates/workload.yml b/roles/byowl/templates/workload.yml index 834b12509..673c8a53e 100755 --- a/roles/byowl/templates/workload.yml +++ b/roles/byowl/templates/workload.yml @@ -19,6 +19,7 @@ spec: {% endfor %} {% endif %} spec: + serviceAccountName: benchmark-operator {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} diff --git a/roles/kube-burner/templates/kube-burner.yml.j2 b/roles/kube-burner/templates/kube-burner.yml.j2 index c6b37e69f..361ce67e7 100644 --- a/roles/kube-burner/templates/kube-burner.yml.j2 +++ b/roles/kube-burner/templates/kube-burner.yml.j2 @@ -24,7 +24,7 @@ spec: {% endif %} tolerations: {{ workload_args.tolerations|default([]) }} restartPolicy: Never - serviceAccountName: kube-burner + serviceAccountName: benchmark-operator nodeSelector: {% if workload_args.pin_server is defined and workload_args.pin_server is mapping %} {% for label, value in workload_args.pin_server.items() %} diff --git a/roles/scale_openshift/templates/scale.yml b/roles/scale_openshift/templates/scale.yml index 9e264bb2d..f84dbf14b 100755 --- a/roles/scale_openshift/templates/scale.yml +++ b/roles/scale_openshift/templates/scale.yml @@ -89,6 +89,6 @@ spec: {% endif %} ; sleep {{post_sleep|default(0)}}" - serviceAccountName: {{workload_args.serviceaccount|default("default")}} + serviceAccountName: benchmark-operator restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 14bd3776b..c7ce73f7f 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -62,7 +62,7 @@ items: imagePullPolicy: Always command: ["/bin/sh","-c"] args: ["uperf -s -v -P {{ control_port|int + item|int * 100 }}"] - restartPolicy: OnFailure + restartPolicy: Never {% if workload_args.pin is sameas true %} nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_server }}' diff --git a/tests/test_api_load.sh b/tests/test_api_load.sh deleted file mode 100755 index 2e0fceb89..000000000 --- a/tests/test_api_load.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up API load Test" - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_api_load { - test_name=$1 - cr=$2 - delete_benchmark $cr - echo "Performing: ${test_name}" - kubectl apply -f ${cr} - benchmark_name=$(get_benchmark_name $cr) - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - check_benchmark_for_desired_state $benchmark_name Complete 500s - - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_api_load "API load" tests/test_crs/valid_api_load.yaml diff --git a/tests/test_backpack.sh b/tests/test_backpack.sh deleted file mode 100755 index 28c1d6b8e..000000000 --- a/tests/test_backpack.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up backpack" - kubectl delete -f resources/backpack_role.yaml - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_backpack { - backpack_requirements - cr=$1 - delete_benchmark $cr - kubectl apply -f $cr - benchmark_name=$(get_benchmark_name $cr) - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - - if [[ $1 == "tests/test_crs/valid_backpack_daemonset.yaml" ]] - then - wait_for_backpack $uuid - else - byowl_pod=$(get_pod "app=byowl-$uuid" 300) - wait_for "kubectl -n benchmark-operator wait --for=condition=Initialized pods/$byowl_pod --timeout=500s" "500s" $byowl_pod - wait_for "kubectl -n benchmark-operator wait --for=condition=complete -l app=byowl-$uuid jobs --timeout=500s" "500s" $byowl_pod - fi - - indexes="cpu_vulnerabilities-metadata cpuinfo-metadata dmidecode-metadata k8s_nodes-metadata lspci-metadata meminfo-metadata sysctl-metadata ocp_network_operator-metadata ocp_install_config-metadata ocp_kube_apiserver-metadata ocp_dns-metadata ocp_kube_controllermanager-metadata" - if check_es "${long_uuid}" "${indexes}" - then - echo "Backpack test: Success" - else - echo "Failed to find data in ES" - exit 1 - fi - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_backpack tests/test_crs/valid_backpack_daemonset.yaml -functional_test_backpack tests/test_crs/valid_backpack_init.yaml diff --git a/tests/test_crs/valid_scale_down.yaml b/tests/test_crs/valid_scale_down.yaml index 4b1d03b04..539d45906 100644 --- a/tests/test_crs/valid_scale_down.yaml +++ b/tests/test_crs/valid_scale_down.yaml @@ -19,6 +19,5 @@ spec: name: scale_openshift args: scale: 0 - serviceaccount: scaler poll_interval: 2 debug: true diff --git a/tests/test_crs/valid_scale_up.yaml b/tests/test_crs/valid_scale_up.yaml index e1f1aceff..302437fd8 100644 --- a/tests/test_crs/valid_scale_up.yaml +++ b/tests/test_crs/valid_scale_up.yaml @@ -19,6 +19,5 @@ spec: name: scale_openshift args: scale: 1 - serviceaccount: scaler poll_interval: 2 debug: true diff --git a/tests/test_kubeburner.sh b/tests/test_kubeburner.sh deleted file mode 100755 index c4f6a15a0..000000000 --- a/tests/test_kubeburner.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - [[ $check_logs == 1 ]] && kubectl logs -l app=kube-burner-benchmark-$uuid -n benchmark-operator - echo "Cleaning up kube-burner" - kubectl delete -f resources/kube-burner-role.yml --ignore-not-found - kubectl delete ns -l kube-burner-uuid=${long_uuid} - wait_clean -} - - -trap error ERR -trap finish EXIT - -function functional_test_kubeburner { - workload_name=$1 - metrics_profile=$2 - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - cr=tests/test_crs/valid_kube-burner.yaml - delete_benchmark $cr - benchmark_name=$(get_benchmark_name $cr) - check_logs=0 - kubectl apply -f resources/kube-burner-role.yml - echo "Performing kube-burner: ${workload_name}" - sed -e "s/WORKLOAD/${workload_name}/g" -e "s/PROMETHEUS_TOKEN/${token}/g" -e "s/METRICS_PROFILE/${metrics_profile}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid $benchmark_name) - uuid=${long_uuid:0:8} - check_logs=1 - check_benchmark_for_desired_state $benchmark_name Complete 1800s - - index="ripsaw-kube-burner" - if check_es "${long_uuid}" "${index}" - then - echo "kube-burner ${workload_name}: Success" - else - echo "Failed to find data for kube-burner ${workload_name} in ES" - exit 1 - fi - kubectl delete ns -l kube-burner-uuid=${long_uuid} - delete_benchmark $cr -} - -figlet $(basename $0) -functional_test_kubeburner cluster-density metrics-aggregated.yaml -functional_test_kubeburner node-density metrics.yaml -functional_test_kubeburner node-density-heavy metrics.yaml -functional_test_kubeburner max-namespaces metrics-aggregated.yaml -functional_test_kubeburner max-services metrics-aggregated.yaml -functional_test_kubeburner concurrent-builds metrics-aggregated.yaml diff --git a/tests/test_list b/tests/test_list index 0e0208c32..10491480b 100755 --- a/tests/test_list +++ b/tests/test_list @@ -1,4 +1,3 @@ -test_backpack.sh test_ycsb.sh test_fiod.sh test_fs-drift.sh @@ -7,11 +6,9 @@ test_pgbench.sh test_iperf3.sh test_sysbench.sh test_uperf.sh -test_byowl.sh test_hammerdb.sh test_vegeta.sh test_scale_openshift.sh test_stressng.sh -test_kubeburner.sh test_flent.sh test_log_generator.sh diff --git a/tests/test_scale_openshift.sh b/tests/test_scale_openshift.sh index 42b38ab79..f30bff4c3 100755 --- a/tests/test_scale_openshift.sh +++ b/tests/test_scale_openshift.sh @@ -22,8 +22,6 @@ function functional_test_scale_openshift { cr=$2 benchmark_name=$(get_benchmark_name $cr) delete_benchmark $cr - # Apply scale role and service account - kubectl apply -f resources/scale_role.yaml token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) echo "Performing: ${test_name}" sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - From 382fdc4b33ff71219c904719fa3ff7a907c79b3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Wed, 8 Jun 2022 11:34:16 +0200 Subject: [PATCH 217/249] Template serviceAccountName (#769) * Template serviceAccountName Signed-off-by: Raul Sevilla * Modify uperf tests order Signed-off-by: Raul Sevilla * Set serviceAccountName: benchmark-operator in uperf client pod Signed-off-by: Raul Sevilla --- e2e/013-uperf.bats | 18 +++++++++--------- playbooks/templates/metadata.yml.j2 | 1 + playbooks/templates/metadata_pod.yml.j2 | 1 + roles/byowl/templates/workload.yml | 1 - roles/cyclictest/templates/cyclictestjob.yaml | 1 - roles/fio_distributed/templates/servers.yaml | 1 - roles/flent/templates/server.yml.j2 | 1 - roles/flent/templates/workload.yml.j2 | 1 - roles/fs-drift/templates/workload_job.yml.j2 | 1 - roles/iperf3/templates/client.yml.j2 | 1 - roles/iperf3/templates/server.yml.j2 | 1 - roles/kube-burner/templates/kube-burner.yml.j2 | 1 - roles/oslat/templates/oslatjob.yaml | 1 - roles/scale_openshift/templates/scale.yml | 1 - roles/smallfile/templates/workload_job.yml.j2 | 1 - roles/testpmd/templates/testpmd.yml.j2 | 1 - roles/testpmd/templates/trex.yml.j2 | 1 - roles/uperf/templates/server.yml.j2 | 1 - roles/vegeta/templates/vegeta.yml.j2 | 1 - 19 files changed, 11 insertions(+), 25 deletions(-) diff --git a/e2e/013-uperf.bats b/e2e/013-uperf.bats index 525c88a61..f05c6b3f1 100755 --- a/e2e/013-uperf.bats +++ b/e2e/013-uperf.bats @@ -7,6 +7,15 @@ load helpers.bash ES_INDEX=ripsaw-uperf-results +@test "uperf-hostnetwork" { + CR=uperf/uperf_hostnetwork.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + @test "uperf-standard" { CR=uperf/uperf.yaml CR_NAME=$(get_benchmark_name ${CR}) @@ -52,15 +61,6 @@ ES_INDEX=ripsaw-uperf-results check_es } -@test "uperf-hostnetwork" { - CR=uperf/uperf_hostnetwork.yaml - CR_NAME=$(get_benchmark_name ${CR}) - envsubst < ${CR} | kubectl apply -f - - get_uuid "${CR_NAME}" - check_benchmark 1200 - check_es -} - setup_file() { basic_setup } diff --git a/playbooks/templates/metadata.yml.j2 b/playbooks/templates/metadata.yml.j2 index ef0222f89..2dcc6169e 100644 --- a/playbooks/templates/metadata.yml.j2 +++ b/playbooks/templates/metadata.yml.j2 @@ -1,3 +1,4 @@ + serviceAccountName: benchmark-operator {% if metadata.collection is sameas true and metadata.targeted is sameas true %} initContainers: - name: backpack diff --git a/playbooks/templates/metadata_pod.yml.j2 b/playbooks/templates/metadata_pod.yml.j2 index 4d48aab77..d74178580 100644 --- a/playbooks/templates/metadata_pod.yml.j2 +++ b/playbooks/templates/metadata_pod.yml.j2 @@ -1,3 +1,4 @@ + serviceAccountName: benchmark-operator {% if metadata.collection is sameas true and metadata.targeted is sameas true %} initContainers: - name: backpack diff --git a/roles/byowl/templates/workload.yml b/roles/byowl/templates/workload.yml index 673c8a53e..834b12509 100755 --- a/roles/byowl/templates/workload.yml +++ b/roles/byowl/templates/workload.yml @@ -19,7 +19,6 @@ spec: {% endfor %} {% endif %} spec: - serviceAccountName: benchmark-operator {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index 9c337dfeb..a9a421523 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -93,5 +93,4 @@ spec: name: "cyclictest-{{ trunc_uuid }}" defaultMode: 0555 restartPolicy: Never - serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} diff --git a/roles/fio_distributed/templates/servers.yaml b/roles/fio_distributed/templates/servers.yaml index 37c3a1197..e1b0a4f09 100644 --- a/roles/fio_distributed/templates/servers.yaml +++ b/roles/fio_distributed/templates/servers.yaml @@ -51,7 +51,6 @@ spec: mountPath: "{{ fio_path }}" {% endif %} restartPolicy: Never - serviceAccountName: benchmark-operator {% if workload_args.nodeselector is defined %} nodeSelector: {% for label, value in workload_args.nodeselector.items() %} diff --git a/roles/flent/templates/server.yml.j2 b/roles/flent/templates/server.yml.j2 index 07529840c..9df8dfbe2 100644 --- a/roles/flent/templates/server.yml.j2 +++ b/roles/flent/templates/server.yml.j2 @@ -26,7 +26,6 @@ spec: {% endif %} {% if workload_args.hostnetwork is sameas true %} hostNetwork: true - serviceAccountName: benchmark-operator {% endif %} containers: - name: benchmark diff --git a/roles/flent/templates/workload.yml.j2 b/roles/flent/templates/workload.yml.j2 index 767f8c4b2..d1b9af109 100644 --- a/roles/flent/templates/workload.yml.j2 +++ b/roles/flent/templates/workload.yml.j2 @@ -26,7 +26,6 @@ spec: spec: {% if workload_args.hostnetwork is sameas true %} hostNetwork : true - serviceAccountName: benchmark-operator {% endif %} affinity: podAntiAffinity: diff --git a/roles/fs-drift/templates/workload_job.yml.j2 b/roles/fs-drift/templates/workload_job.yml.j2 index 71d30b9ad..bd51c5af0 100644 --- a/roles/fs-drift/templates/workload_job.yml.j2 +++ b/roles/fs-drift/templates/workload_job.yml.j2 @@ -118,5 +118,4 @@ spec: type: DirectoryOrCreate {% endif %} restartPolicy: Never - serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} diff --git a/roles/iperf3/templates/client.yml.j2 b/roles/iperf3/templates/client.yml.j2 index 57d686bb5..a3c1b4847 100644 --- a/roles/iperf3/templates/client.yml.j2 +++ b/roles/iperf3/templates/client.yml.j2 @@ -39,7 +39,6 @@ spec: {% endif %} {% if workload_args.hostnetwork is sameas true %} hostNetwork: true - serviceAccountName: benchmark-operator {% endif %} containers: - name: benchmark diff --git a/roles/iperf3/templates/server.yml.j2 b/roles/iperf3/templates/server.yml.j2 index aded32b49..2d4a1c754 100644 --- a/roles/iperf3/templates/server.yml.j2 +++ b/roles/iperf3/templates/server.yml.j2 @@ -34,7 +34,6 @@ spec: {% endif %} {% if workload_args.hostnetwork is sameas true %} hostNetwork: true - serviceAccountName: benchmark-operator {% endif %} containers: - name: benchmark diff --git a/roles/kube-burner/templates/kube-burner.yml.j2 b/roles/kube-burner/templates/kube-burner.yml.j2 index 361ce67e7..c86435621 100644 --- a/roles/kube-burner/templates/kube-burner.yml.j2 +++ b/roles/kube-burner/templates/kube-burner.yml.j2 @@ -24,7 +24,6 @@ spec: {% endif %} tolerations: {{ workload_args.tolerations|default([]) }} restartPolicy: Never - serviceAccountName: benchmark-operator nodeSelector: {% if workload_args.pin_server is defined and workload_args.pin_server is mapping %} {% for label, value in workload_args.pin_server.items() %} diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index c679bfb18..3e506c919 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -94,5 +94,4 @@ spec: name: "oslat-{{ trunc_uuid }}" defaultMode: 0555 restartPolicy: Never - serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} diff --git a/roles/scale_openshift/templates/scale.yml b/roles/scale_openshift/templates/scale.yml index f84dbf14b..4af503f55 100755 --- a/roles/scale_openshift/templates/scale.yml +++ b/roles/scale_openshift/templates/scale.yml @@ -89,6 +89,5 @@ spec: {% endif %} ; sleep {{post_sleep|default(0)}}" - serviceAccountName: benchmark-operator restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/smallfile/templates/workload_job.yml.j2 b/roles/smallfile/templates/workload_job.yml.j2 index 9d0e198f6..1be1b1958 100644 --- a/roles/smallfile/templates/workload_job.yml.j2 +++ b/roles/smallfile/templates/workload_job.yml.j2 @@ -140,5 +140,4 @@ spec: type: DirectoryOrCreate {% endif %} restartPolicy: Never - serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} diff --git a/roles/testpmd/templates/testpmd.yml.j2 b/roles/testpmd/templates/testpmd.yml.j2 index 6823cf66f..e07b9edc3 100644 --- a/roles/testpmd/templates/testpmd.yml.j2 +++ b/roles/testpmd/templates/testpmd.yml.j2 @@ -78,6 +78,5 @@ spec: emptyDir: medium: HugePages restartPolicy: Never - serviceAccountName: benchmark-operator {% include "metadata_pod.yml.j2" %} diff --git a/roles/testpmd/templates/trex.yml.j2 b/roles/testpmd/templates/trex.yml.j2 index 1e8eaa745..54e0ec28b 100644 --- a/roles/testpmd/templates/trex.yml.j2 +++ b/roles/testpmd/templates/trex.yml.j2 @@ -139,6 +139,5 @@ spec: hostPath: path: /lib/modules restartPolicy: Never - serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index c7ce73f7f..c7ffa3ba5 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -51,7 +51,6 @@ items: {% endif %} {% if workload_args.hostnetwork is sameas true %} hostNetwork: true - serviceAccountName: benchmark-operator {% endif %} containers: - name: benchmark diff --git a/roles/vegeta/templates/vegeta.yml.j2 b/roles/vegeta/templates/vegeta.yml.j2 index 6730c5abd..279d05dfa 100644 --- a/roles/vegeta/templates/vegeta.yml.j2 +++ b/roles/vegeta/templates/vegeta.yml.j2 @@ -28,7 +28,6 @@ spec: {% endif %} {% if workload_args.hostnetwork is sameas true %} hostNetwork : true - serviceAccountName: benchmark-operator {% endif %} affinity: podAntiAffinity: From 6fa2b254c564d959d1784c7269cbb6562ce523f1 Mon Sep 17 00:00:00 2001 From: Sharada Vetsa Date: Mon, 13 Jun 2022 19:10:29 +0530 Subject: [PATCH 218/249] Point to redis multiarch image in charts/benchmark-operator (#771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Point to redis image to multi-arch image on docker.io * Point to cloud-bulldozer's image * Update manager.yaml * Update manager.yaml redis data mount Update mount point for redis to reflect mount point needed for multiarch image to test arm and amd64. * Update values.yaml Last bit of update to point to multiarch redis image to be able to run on ARM. Co-authored-by: Raúl Sevilla --- charts/benchmark-operator/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/benchmark-operator/values.yaml b/charts/benchmark-operator/values.yaml index 7a4f47ecf..1db5dddb0 100644 --- a/charts/benchmark-operator/values.yaml +++ b/charts/benchmark-operator/values.yaml @@ -16,7 +16,7 @@ operator: tag: latest redisImage: - repository: bitnami/redis + repository: quay.io/cloud-bulldozer/redis pullPolicy: Always tag: latest From d8672e9f8d3415c60ad5df8584fb1b6b0f89cebb Mon Sep 17 00:00:00 2001 From: Vishnu Challa Date: Wed, 29 Jun 2022 15:03:38 -0400 Subject: [PATCH 219/249] bug fix for ports mismatch between client and server pairs in without service and with clusterip service case (#774) --- roles/uperf/templates/configmap.yml.j2 | 6 +++--- roles/uperf/templates/server.yml.j2 | 3 ++- roles/uperf/templates/service.yml.j2 | 1 + roles/uperf/templates/workload.yml.j2 | 7 ++++--- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/roles/uperf/templates/configmap.yml.j2 b/roles/uperf/templates/configmap.yml.j2 index fc8b3b583..59c558c2d 100644 --- a/roles/uperf/templates/configmap.yml.j2 +++ b/roles/uperf/templates/configmap.yml.j2 @@ -23,7 +23,7 @@ data: {% if ( 'rr' == test ) %} - + @@ -36,7 +36,7 @@ data: {% if ( 'stream' == test or 'bidirec' == test ) %} - + @@ -49,7 +49,7 @@ data: {% if ( 'maerts' == test or 'bidirec' == test ) %} - + diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index c7ffa3ba5..9602acba9 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -32,6 +32,7 @@ items: benchmark-uuid: {{ uuid }} benchmark-operator-workload: uperf benchmark-operator-role: server + index: "{{item}}" {% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} {% else %} @@ -60,7 +61,7 @@ items: {% endif %} imagePullPolicy: Always command: ["/bin/sh","-c"] - args: ["uperf -s -v -P {{ control_port|int + item|int * 100 }}"] + args: ["uperf -s -v -P {{ control_port|int + (0 if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "clusterip" )) else (item|int * 100)) }}"] restartPolicy: Never {% if workload_args.pin is sameas true %} nodeSelector: diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index ee0c0f8bf..0ae245972 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -13,6 +13,7 @@ items: labels: benchmark-uuid: {{ uuid }} benchmark-operator-workload: uperf + index: "{{item}}" {% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} {% else %} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index b27e65c3b..c514352ae 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -10,7 +10,7 @@ items: metadata: {% if workload_args.serviceip is sameas true %} {% if workload_args.servicetype | default("clusterip") == "nodeport" %} - name: 'uperf-client-{{item.status.hostIP}}-{{ loop.index - 1 }}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.status.hostIP}}-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }}' {% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} name: 'uperf-client-{{item.status.loadBalancer.ingress[0].ip}}-{{ trunc_uuid }}' {% else %} @@ -125,7 +125,8 @@ items: - "export h={{item.status.podIP}}; {% endif %} {% endif %} - export port={{ control_port + (loop.index - 1) * 100 }} + export port={{ control_port + (0 if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "clusterip" )) else (item.metadata.labels.index|int * 100))}} + {% if (workload_args.colocate is defined) %} export colocate={{ workload_args.colocate}}; {% endif %} @@ -219,7 +220,7 @@ items: volumes: - name: config-volume configMap: - name: uperf-test-{{ loop.index - 1 }}-{{ trunc_uuid }} + name: uperf-test-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }} restartPolicy: Never {% if workload_args.pin is sameas false %} {% if workload_args.colocate is sameas true %} From ae07c14c360544f92e645c1acffc0faea8fb1042 Mon Sep 17 00:00:00 2001 From: Vishnu Challa Date: Wed, 6 Jul 2022 11:35:00 -0400 Subject: [PATCH 220/249] Standard uperf workload without scale mode features. (#775) * initial draft of standard uperf with new features * bug fix for client creation in nodeport service * bug fix for client creation conflict due to same name in metadata * added index for metallb service labels * bug fix for clusterip service ports mismatch * reverting quay repo org to cloudbulldozer * removed test configuration * fix for client start issue caused by redis variables for vm use case * renamed client_vm to client_vms for vm kind * fixing redis variables for pod use cases as well * Merge nodeport and hostnetwork tests Signed-off-by: Raul Sevilla Co-authored-by: Raul Sevilla --- e2e/013-uperf.bats | 26 +- roles/uperf/README.md | 3 +- roles/uperf/defaults/main.yml | 3 +- roles/uperf/handlers/main.yml | 2 +- roles/uperf/meta/main.yml | 29 +- roles/uperf/tasks/cleanup.yml | 82 ---- roles/uperf/tasks/init.yml | 22 - roles/uperf/tasks/main.yml | 438 ++++++++++++++++-- roles/uperf/tasks/next_set.yml | 104 ----- roles/uperf/tasks/run_a_set.yml | 26 -- roles/uperf/tasks/send_client_run_signal.yml | 18 - roles/uperf/tasks/setup.yml | 131 ------ roles/uperf/tasks/start_client.yml | 90 ---- roles/uperf/tasks/start_server.yml | 88 ---- roles/uperf/tasks/wait_client_done.yml | 40 -- roles/uperf/tasks/wait_client_ready.yml | 48 -- roles/uperf/tasks/wait_server_ready.yml | 55 --- roles/uperf/tasks/wait_set_done.yml | 28 -- roles/uperf/templates/configmap.yml.j2 | 2 +- roles/uperf/templates/configmap_script.yml.j2 | 2 +- roles/uperf/templates/server.yml.j2 | 108 ++--- roles/uperf/templates/server_vm.yml.j2 | 2 +- roles/uperf/templates/service.yml.j2 | 112 ++--- roles/uperf/templates/service_metallb.yml.j2 | 140 +++--- roles/uperf/templates/workload.yml.j2 | 304 +++++------- roles/uperf/templates/workload_vm.yml.j2 | 2 +- roles/uperf/vars/main.yml | 12 +- 27 files changed, 685 insertions(+), 1232 deletions(-) delete mode 100644 roles/uperf/tasks/cleanup.yml delete mode 100644 roles/uperf/tasks/init.yml delete mode 100644 roles/uperf/tasks/next_set.yml delete mode 100644 roles/uperf/tasks/run_a_set.yml delete mode 100644 roles/uperf/tasks/send_client_run_signal.yml delete mode 100644 roles/uperf/tasks/setup.yml delete mode 100644 roles/uperf/tasks/start_client.yml delete mode 100644 roles/uperf/tasks/start_server.yml delete mode 100644 roles/uperf/tasks/wait_client_done.yml delete mode 100644 roles/uperf/tasks/wait_client_ready.yml delete mode 100644 roles/uperf/tasks/wait_server_ready.yml delete mode 100644 roles/uperf/tasks/wait_set_done.yml diff --git a/e2e/013-uperf.bats b/e2e/013-uperf.bats index f05c6b3f1..e733b23ff 100755 --- a/e2e/013-uperf.bats +++ b/e2e/013-uperf.bats @@ -7,13 +7,16 @@ load helpers.bash ES_INDEX=ripsaw-uperf-results -@test "uperf-hostnetwork" { - CR=uperf/uperf_hostnetwork.yaml - CR_NAME=$(get_benchmark_name ${CR}) - envsubst < ${CR} | kubectl apply -f - - get_uuid "${CR_NAME}" - check_benchmark 1200 - check_es +@test "uperf-hostnetwork-nodeport" { + # Merging nodeport and hostnetwork test to avoid port binding conflicts + for CR in uperf/uperf_hostnetwork.yaml uperf/uperf_serviceip_nodeport.yaml; do + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es + kubectl_exec delete benchmark ${CR_NAME} --ignore-not-found + done } @test "uperf-standard" { @@ -52,15 +55,6 @@ ES_INDEX=ripsaw-uperf-results check_es } -@test "uperf-serviceip-nodeport" { - CR=uperf/uperf_serviceip_nodeport.yaml - CR_NAME=$(get_benchmark_name ${CR}) - envsubst < ${CR} | kubectl apply -f - - get_uuid "${CR_NAME}" - check_benchmark 1200 - check_es -} - setup_file() { basic_setup } diff --git a/roles/uperf/README.md b/roles/uperf/README.md index f8dad4e9c..71fc153b7 100644 --- a/roles/uperf/README.md +++ b/roles/uperf/README.md @@ -1,5 +1,4 @@ uperf-benchmark ========= -Network dataplane workload - +Network dataplane workload \ No newline at end of file diff --git a/roles/uperf/defaults/main.yml b/roles/uperf/defaults/main.yml index 9f9e3e09c..7e3bad17e 100644 --- a/roles/uperf/defaults/main.yml +++ b/roles/uperf/defaults/main.yml @@ -1,8 +1,9 @@ --- +# defaults file for uperf-scale resource_kind: "{{ workload.args.kind | default('pod') }}" uperf: proto: tcp test_type: stream nthr: 1 size: 1024 - runtime: 60 + runtime: 60 \ No newline at end of file diff --git a/roles/uperf/handlers/main.yml b/roles/uperf/handlers/main.yml index de0f0626f..572a9a4de 100644 --- a/roles/uperf/handlers/main.yml +++ b/roles/uperf/handlers/main.yml @@ -1,2 +1,2 @@ --- -# handlers file for bench +# handlers file for uperf \ No newline at end of file diff --git a/roles/uperf/meta/main.yml b/roles/uperf/meta/main.yml index 5d50bf41b..227ad9c34 100644 --- a/roles/uperf/meta/main.yml +++ b/roles/uperf/meta/main.yml @@ -1,34 +1,26 @@ galaxy_info: author: your name - description: your description + description: your role description company: your company (optional) # If the issue tracker for your role is not on github, uncomment the # next line and provide a value # issue_tracker_url: http://example.com/issue/tracker - # Some suggested licenses: - # - BSD (default) + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) # - MIT - # - GPLv2 - # - GPLv3 - # - Apache - # - CC-BY - license: license (GPLv2, CC-BY, etc) + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) - min_ansible_version: 2.4 + min_ansible_version: 2.9 # If this a Container Enabled role, provide the minimum Ansible Container version. # min_ansible_container_version: - # Optionally specify the branch Galaxy will use when accessing the GitHub - # repo for this role. During role install, if no tags are available, - # Galaxy will use this branch. During import Galaxy will access files on - # this branch. If Travis integration is configured, only notifications for this - # branch will be accepted. Otherwise, in all cases, the repo's default branch - # (usually master) will be used. - #github_branch: - # # Provide a list of supported platforms, and for each platform a list of versions. # If you don't wish to enumerate all versions for a particular platform, use 'all'. @@ -57,4 +49,5 @@ galaxy_info: dependencies: [] # List your role dependencies here, one per line. Be sure to remove the '[]' above, - # if you add dependencies to this list. \ No newline at end of file + # if you add dependencies to this list. + \ No newline at end of file diff --git a/roles/uperf/tasks/cleanup.yml b/roles/uperf/tasks/cleanup.yml deleted file mode 100644 index 4e8714f3b..000000000 --- a/roles/uperf/tasks/cleanup.yml +++ /dev/null @@ -1,82 +0,0 @@ ---- - -- block: - ### kind - # Cleanup servers, but leave clients around mostly for further examining of results. - - name: Get Server Jobs - k8s_facts: - kind: Job - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - register: server_jobs - - - name: Get Server Pods - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - register: server_pods - - - name: Server Job and Pod names - to clean - set_fact: - clean_jobs: | - [ - {% for item in server_jobs.resources %} - "{{ item['metadata']['name'] }}", - {% endfor %} - ] - clean_pods: | - [ - {% for item in server_pods.resources %} - "{{ item['metadata']['name'] }}", - {% endfor %} - ] - - - name: Cleanup server Job - k8s: - kind: Job - api_version: v1 - namespace: '{{ operator_namespace }}' - state: absent - name: "{{ item }}" - with_items: "{{ clean_jobs }}" - - - name: Cleanup server Pod - k8s: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - state: absent - name: "{{ item }}" - with_items: "{{ clean_pods }}" - - when: resource_kind == "pod" and cleanup == True - -- block: - - name: Cleanup redis - command: "{{ item }}" - with_items: - - redis-cli del num_completion-{{trunc_uuid}} - - redis-cli del start-{{trunc_uuid}} - when: resource_kind == "pod" - - - - -# -# no kind block - We leave VM running -# - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - diff --git a/roles/uperf/tasks/init.yml b/roles/uperf/tasks/init.yml deleted file mode 100644 index 39c65791a..000000000 --- a/roles/uperf/tasks/init.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- - -- name: Clear start flag - command: "redis-cli set start-{{trunc_uuid}} 0" - -- name: Clear num_completion - command: "redis-cli set num_completion-{{trunc_uuid}} 0" - -- name: Init node and pod indices in benchmark context - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - pod_hi_idx: "{{pod_hi_idx}}" - pod_low_idx: "{{pod_low_idx}}" - node_hi_idx: "{{node_hi_idx}}" - node_low_idx: "{{node_low_idx}}" - node_idx: "{{node_low_idx}}" - pod_idx: "{{pod_low_idx}}" - diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index 43dbdc4eb..8c6118748 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -1,70 +1,426 @@ --- -- include_tasks: setup.yml +- name: Get current state + k8s_facts: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ operator_namespace }}' + register: resource_state -- include_tasks: start_server.yml - when: benchmark_state.resources[0].status.state == "Building" +- operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Building + complete: false + when: resource_state.resources[0].status.state is not defined + +- name: Get current state - If it has changed + k8s_facts: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ operator_namespace }}' + register: resource_state + +- name: Capture ServiceIP + k8s_facts: + kind: Service + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: serviceip + when: workload_args.serviceip is defined and workload_args.serviceip + +- block: + + - name: Create service for server pods + k8s: + definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + when: + - workload_args.serviceip is defined and workload_args.serviceip + - ( workload_args.servicetype | default("clusterip") == "clusterip" ) or + ( workload_args.servicetype | default("clusterip") == "nodeport" ) + + - name: Create metal LB service for server pods + k8s: + definition: "{{ lookup('template', 'service_metallb.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + when: + - workload_args.serviceip is defined and workload_args.serviceip + - workload_args.servicetype | default("clusterip") == "metallb" + + - name: Start Server(s) + k8s: + definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" + register: servers + with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + + - name: Wait for pods to be running.... + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Servers" + + when: resource_state.resources[0].status.state == "Building" and resource_kind == "pod" + +- block: + + - name: Start Server(s) + k8s: + definition: "{{ lookup('template', 'server_vm.yml.j2') | from_yaml }}" + register: servers + with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + + - name: Wait for vms to be running.... + k8s_facts: + kind: VirtualMachineInstance + api_version: kubevirt.io/v1alpha3 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_vms + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Servers" + + when: resource_state.resources[0].status.state == "Building" and resource_kind == "vm" + +- block: + + - name: Get server pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Clients" + when: "workload_args.pair|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + + + when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" + +- block: + + - name: Wait for vms to be running.... + k8s_facts: + kind: VirtualMachineInstance + api_version: kubevirt.io/v1alpha3 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_vms + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Clients" + when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + + - name: blocking client from running uperf + command: "redis-cli set start-{{ trunc_uuid }} false" + with_items: "{{ server_vms.resources }}" + when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + + when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" and workload_args.pair|default('1')|int|int == 1 - block: - - include_tasks: wait_server_ready.yml - when: benchmark_state.resources[0].status.state == "Starting Servers" + - name: Get pod info + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: set pod_sequence + set_fact: + pod_sequence: "{{ workload_args.pair|int - 1}}" + + - name: Generate uperf xml files + k8s: + definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" + with_sequence: start=0 end={{ pod_sequence|int }} + + - block: + - name: Start Client(s) + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + with_items: "{{ server_pods.resources }}" + when: ( workload_args.serviceip|default(False) == False and server_pods.resources|length > 0 ) + + - name: Start Client(s) with nodeport serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + with_items: "{{ server_pods.resources }}" + when: + - ( workload_args.serviceip|default(False) == True and server_pods.resources|length > 0 and + workload_args.servicetype | default("clusterip") == "nodeport" ) + + - name: Start Client(s) - ServiceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + with_items: "{{ serviceip.resources }}" + when: + - workload_args.serviceip|default(False) == True and serviceip.resources|length > 0 + - workload_args.servicetype | default("clusterip") != "nodeport" + + when: resource_kind == "pod" + + - block: - - include_tasks: start_client.yml - when: benchmark_state.resources[0].status.state == "Starting Clients" + - name: Wait for vms to be running.... + k8s_facts: + kind: VirtualMachineInstance + api_version: kubevirt.io/v1alpha3 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_vms - - include_tasks: wait_client_ready.yml - when: benchmark_state.resources[0].status.state == "Waiting for Clients" - # LOOP BEGIN - # This loop iterates density_range[] and node_range[] for "scale" mode + - name: Generate uperf test files + k8s: + definition: "{{ lookup('template', 'configmap_script.yml.j2') | from_yaml }}" + with_items: "{{ server_vms.resources }}" + + - name: Start Client(s) + k8s: + definition: "{{ lookup('template', 'workload_vm.yml.j2') | from_yaml }}" + with_indexed_items: "{{ server_vms.resources }}" + when: server_vms.resources|length > 0 + + when: resource_kind == "vm" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Waiting for Clients + + when: resource_state.resources[0].status.state == "Starting Clients" + +- block: - - include_tasks: run_a_set.yml - when: benchmark_state.resources[0].status.state == "Clients Running" + - block: + - name: Get client pod status + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-bench-client-{{ trunc_uuid }} + register: client_pods - - include_tasks: wait_set_done.yml - when: benchmark_state.resources[0].status.state == "Set Running" + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + when: "workload_args.pair|default('1')|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" + when: resource_kind == "pod" - - include_tasks: next_set.yml - when: benchmark_state.resources[0].status.state == "Run Next Set" - # will loop back to "Client Running" state, or FALLTHRU to "Running" - # state below and finish + - block: + + - name: set complete to false + command: "redis-cli set complete false" + + - name: Get client vm status + k8s_facts: + kind: VirtualMachineInstance + api_version: kubevirt.io/v1alpha3 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-bench-client-{{ trunc_uuid }} + register: client_vms + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + when: "workload_args.pair|default('1')|int == client_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + + when: resource_kind == "vm" + + when: resource_state.resources[0].status.state == "Waiting for Clients" + +- block: + + - name: Signal workload + command: "redis-cli set start-{{ trunc_uuid }} true" + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Running" + + when: resource_state.resources[0].status.state == "Clients Running" + +- block: + - block: + - name: Waiting for pods to complete.... + k8s_facts: + kind: pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-bench-client-{{ trunc_uuid }} + register: client_pods - # LOOP END + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Cleanup + complete: false + when: "workload_args.pair|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: resource_kind == "pod" - - include_tasks: wait_client_done.yml - when: benchmark_state.resources[0].status.state == "Running" + - block: - - include_tasks: cleanup.yml - when: benchmark_state.resources[0].status.state == "Cleanup" + - name: get complete + command: "redis-cli get complete" + register: complete_status - when: resource_kind == "pod" + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Cleanup + complete: false + when: complete_status.stdout == "true" + when: resource_kind == "vm" -# -# kind does not support "scale" mode yet -# + when: resource_state.resources[0].status.state == "Running" - block: - - include_tasks: wait_server_ready.yml - when: benchmark_state.resources[0].status.state == "Starting Servers" + - block: + - name: Get Server Jobs + k8s_facts: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_jobs - - include_tasks: start_client.yml - when: benchmark_state.resources[0].status.state == "Starting Clients" + - name: Get Server Pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods - - include_tasks: wait_client_ready.yml - when: benchmark_state.resources[0].status.state == "Waiting for Clients" + - name: Server Job and Pod names - to clean + set_fact: + clean_jobs: | + [ + {% for item in server_jobs.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + clean_pods: | + [ + {% for item in server_pods.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] - - include_tasks: send_client_run_signal.yml - when: benchmark_state.resources[0].status.state == "Clients Running" + - name: Cleanup server Job + k8s: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_jobs }}" - - include_tasks: wait_client_done.yml - when: benchmark_state.resources[0].status.state == "Running" + - name: Cleanup server Pod + k8s: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_pods }}" - - include_tasks: cleanup.yml - when: benchmark_state.resources[0].status.state == "Cleanup" + when: resource_kind == "pod" and cleanup == True - when: resource_kind == "vm" + - block: + - name: Cleanup redis + command: "{{ item }}" + with_items: + - redis-cli del complete + - redis-cli del start-{{ trunc_uuid }} + when: resource_kind == "pod" + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + when: resource_state.resources[0].status.state == "Cleanup" \ No newline at end of file diff --git a/roles/uperf/tasks/next_set.yml b/roles/uperf/tasks/next_set.yml deleted file mode 100644 index c8c74f07b..000000000 --- a/roles/uperf/tasks/next_set.yml +++ /dev/null @@ -1,104 +0,0 @@ ---- -# -# This module logically implements an RE-ENTRANT nested "for" loops; -# -# with_items: -# range (node_low_idx, node_hi_idx) -# range (pod_low_idx, pod_hi_idx) -# -# Each iteration executes one item, and each re-entrance -# continues where it left off. -# -- block: - - name: Read previous node_idx and pod_idx - set_fact: - all_run_done: False - inc: "{{workload_args.step_size|default('add1')}}" - amount: 0 - pod_idx: "{{benchmark_state.resources[0].status.pod_idx|int}}" - node_idx: "{{benchmark_state.resources[0].status.node_idx|int}}" - - - name: Extract add amount - set_fact: - amount: "{{ inc | regex_replace('[^0-9]', '') }}" - inc: add - when: "'add' in inc" - - - name: Increment pod_idx - set_fact: - pod_idx: "{%-if inc=='add' -%}{{pod_idx|int+amount|int}} - {%-elif inc=='log2' -%}{{(pod_idx|int*2)+1}} - {%-else -%}{{pod_idx|int+1}} - {% endif %}" - - block: - # - # This block starts a new node loop - # - - name: Increment node_idx - set_fact: - node_idx: "{%- if inc=='add' -%}{{node_idx|int+amount|int}} - {%- elif inc=='log2' -%}{{(node_idx|int *2)+1}} - {%- else -%}{{node_idx|int+1}} - {% endif %}" - - - name: Check node loop for ending condition - set_fact: - all_run_done: True - when: "node_idx|int > benchmark_state.resources[0].status.node_hi_idx|int" - - # - # Reset pod_idx AFTER node_idx tasks above, else cond change - # causes it to skip node_idx tasks - # - - name: Reset pod_idx to pod_low_idx - set_fact: - pod_idx: "{{benchmark_state.resources[0].status.pod_low_idx}}" - - when: "pod_idx|int > benchmark_state.resources[0].status.pod_hi_idx|int" - - - block: - # - # All done - # - - name: Unpause pods to complete - command: "redis-cli set start-{{trunc_uuid}} done" - - - name: Change state to proceed to exit - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - - when: all_run_done == True - - - block: - # - # More round(s) to run. - # - - name: Send redis restart signal - command: "redis-cli set start-{{trunc_uuid}} restart" - - - name: Reset redis num_completion - command: "redis-cli set num_completion-{{trunc_uuid}} 0" - - - name: Change state to run next round - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Clients Running - pod_idx: "{{pod_idx}}" - node_idx: "{{node_idx}}" - - when: all_run_done == False - - when: resource_kind == "pod" - -# -# No block - Scale mode support is N/A -# diff --git a/roles/uperf/tasks/run_a_set.yml b/roles/uperf/tasks/run_a_set.yml deleted file mode 100644 index ca0e44f76..000000000 --- a/roles/uperf/tasks/run_a_set.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- - -- block: - # - # Entry Condition: - # 1. A previous task has set 'node_idx' and 'pod_idx' in benchmark ctx - # 2. All cliest are polling redis for 'start-node_idx-pod_idx' to start - # Output: Clients with node_idx <= redis node_idx && pod_idx <= redis pod_ix will run - # - - - name: Signal group to run - command: "redis-cli set start-{{trunc_uuid}} true-{{benchmark_state.resources[0].status.node_idx|int}}-{{benchmark_state.resources[0].status.pod_idx|int}}" - - - name: Update state to "Set Running" - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Set Running - when: resource_kind == "pod" - -# -# No kind - It has not been adapted to Scale mode -# diff --git a/roles/uperf/tasks/send_client_run_signal.yml b/roles/uperf/tasks/send_client_run_signal.yml deleted file mode 100644 index 756274cf7..000000000 --- a/roles/uperf/tasks/send_client_run_signal.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# This module is invoked by VM-kind only -- block: - - - name: Signal workload - command: "redis-cli set start-{{trunc_uuid}} true" - - - name: A26 Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running" - - when: benchmark_state.resources[0].status.state == "Clients Running" - diff --git a/roles/uperf/tasks/setup.yml b/roles/uperf/tasks/setup.yml deleted file mode 100644 index 9ce9e30a9..000000000 --- a/roles/uperf/tasks/setup.yml +++ /dev/null @@ -1,131 +0,0 @@ ---- -- block: - # - # This block is for scale mode where client and server pods are spreaded - # across all eligible nodes - # - - name: List Nodes Labeled as Workers - k8s_info: - api_version: v1 - kind: Node - label_selectors: - - "node-role.kubernetes.io/worker=" - register: node_list - no_log: True - - - name: Isolate Worker Role Hostnames - set_fact: - worker_node_list: "{{ node_list | json_query('resources[].metadata.labels.\"kubernetes.io/hostname\"') | list }}" - - - name: List Nodes Labeled with {{ workload_args.exclude_label }} - k8s_info: - api_version: v1 - kind: Node - label_selectors: - - '{{ item }}' - with_items: "{{ workload_args.exclude_labels }}" - register: exclude_node_list - when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 - - - name: Isolate Worker Role Hostnames for label {{ workload_args.exclude_label }} - set_fact: - worker_node_exclude_list: "{{ exclude_node_list | json_query('results[].resources[].metadata.name') }}" - - - name: Exclude labeled nodes - set_fact: - worker_node_list: "{{ worker_node_list | difference(worker_node_exclude_list) }}" - when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 - # - # Compute node and pod limits using CR params while taking into account - # of the actual number of nodes available in the system - # - - name: init pod and node low/hi idx - set_fact: - pod_low_idx: "{{ workload_args.density_range[0] | default('1')|int - 1 }}" - pod_hi_idx: "{{ workload_args.density_range[1] | default('1')|int - 1 }}" - node_low_idx: "{{ workload_args.node_range[0] | default('1')|int - 1 }}" - node_hi_idx: "{{ workload_args.node_range[1] | default('1')|int - 1 }}" - # - # Next sanity check and massage the indices if necessary. - # We shall complete gracefully and not iterate wildly. - # - - name: Adjust node_hi_idx if cluster has less nodes - set_fact: - node_hi_idx: "{{ worker_node_list|length| default('0')|int -1 }}" - when: "node_hi_idx|int >= worker_node_list|length| default('0')|int " - - - name: Adjust node_low_idx if necessary - set_fact: - node_low_idx: "{{node_hi_idx|int}}" - when: "node_low_idx|int > node_hi_idx|int" - - - name: Adjust pod_low_idx if necessary - set_fact: - pod_low_idx: "{{pod_hi_idx|int}}" - when: "pod_low_idx|int > pod_hi_idx|int" - - - name: Record num server pods using new worker_node_list - # in Scale mode, num server pods = num_node * number_pod - set_fact: - num_server_pods: "{{ (node_hi_idx|int+1) * (pod_hi_idx|int+1) }}" - - # - # End scale mode - # - when: workload_args.pin | default(False) == False - -- block: - # - # This block is for the "pin" mode where the server and the client node - # are specified by pin_server and pin_client variables. - - - name: Add "Pin" server and client node to worker list. - # The add order is significant as we will enumerate the server pods on - # the first node in the list, and client pods on the second node. - set_fact: - worker_node_list: "{{worker_node_list + [item]}}" - with_items: - - '{{workload_args.pin_server}}' - - '{{workload_args.pin_client}}' - # - # In 'Pin' mode, 'pair' specifies number of pairs (classic behavior), If 'pair' - # is undefined use 'density_range' (new bahavior with "Scale" enhancement) - # - - name: Init "Pin" mode indices using 'pair' - set_fact: - pod_low_idx: "{{ workload_args.pair | default('1')|int - 1 }}" - pod_hi_idx: "{{ workload_args.pair | default('1')|int - 1 }}" - # node indices are used as client pod 'start' parameter. - node_low_idx: "0" - node_hi_idx: "0" - when: workload_args.pair is defined - - - name: Init "Pin" mode indices using 'density_range' - set_fact: - pod_low_idx: "{{ workload_args.density_range[0] | default('1')|int - 1 }}" - pod_hi_idx: "{{ workload_args.density_range[1] | default('1')|int - 1 }}" - # node indices are used as client pod 'start' parameter. - node_low_idx: "0" - node_hi_idx: "0" - when: workload_args.pair is not defined - - - name: Record num Pin server pods using new worker_node_list - set_fact: - # in Pin mode, num server pods = number of pods - num_server_pods: "{{ pod_hi_idx|int +1 }}" - - # - # End pin mode where pin_client and pin_server are specified - # - when: workload_args.pin | default(False) == True - -- name: Capture ServiceIP - k8s_facts: - kind: Service - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - register: serviceip - when: workload_args.serviceip is defined and workload_args.serviceip - diff --git a/roles/uperf/tasks/start_client.yml b/roles/uperf/tasks/start_client.yml deleted file mode 100644 index c567c69e5..000000000 --- a/roles/uperf/tasks/start_client.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- - -- name: Get pod info - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - register: server_pods - -- name: set pod_sequence - set_fact: - pod_sequence: "{{ pod_hi_idx|int }}" - -- name: Generate uperf xml files - k8s: - definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" - with_sequence: start=0 end={{ pod_sequence|int }} - -- block: - ### kind - - name: Start Client(s) w/o serviceIP - k8s: - definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - vars: - resource_item: "{{ server_pods.resources }}" - when: - - ( workload_args.serviceip|default(False) == False and server_pods.resources|length > 0 ) - - - - name: Start Client(s) with nodeport serviceIP - k8s: - definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - vars: - resource_item: "{{ server_pods.resources }}" - when: - - ( workload_args.serviceip|default(False) == True and server_pods.resources|length > 0 and - workload_args.servicetype | default("clusterip") == "nodeport" ) - - # - # Each server annotates a "node_idx". Each peer client will - # derive its affinity according the 'colocate' variable. - # - - - name: Start Client(s) with serviceIP - k8s: - definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" - vars: - resource_item: "{{ serviceip.resources }}" - when: - - workload_args.serviceip|default(False) == True and serviceip.resources|length > 0 - - workload_args.servicetype | default("clusterip") != "nodeport" - when: resource_kind == "pod" - -- block: - ### kind - - - name: Wait for vms to be running.... - k8s_facts: - kind: VirtualMachineInstance - api_version: kubevirt.io/v1alpha3 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - register: server_vms - - - - name: Generate uperf test files - k8s: - definition: "{{ lookup('template', 'configmap_script.yml.j2') | from_yaml }}" - with_items: "{{ server_vms.resources }}" - - - name: Start Client(s) - k8s: - definition: "{{ lookup('template', 'workload_vm.yml.j2') | from_yaml }}" - with_indexed_items: "{{ server_vms.resources }}" - when: server_vms.resources|length > 0 - - when: resource_kind == "vm" - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Waiting for Clients - - diff --git a/roles/uperf/tasks/start_server.yml b/roles/uperf/tasks/start_server.yml deleted file mode 100644 index 88feb2fa9..000000000 --- a/roles/uperf/tasks/start_server.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- - -- block: - ### kind - - include_tasks: init.yml - - - name: Create service for server pods - k8s: - definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}" - vars: - pod_sequence: "{{ pod_hi_idx|int +1 }}" - node_sequence: "{{ node_hi_idx|int +1 }}" - - when: - - workload_args.serviceip is defined and workload_args.serviceip - - ( workload_args.servicetype | default("clusterip") == "clusterip" ) or - ( workload_args.servicetype | default("clusterip") == "nodeport" ) - - - name: Create metal LB service for server pods - k8s: - definition: "{{ lookup('template', 'service_metallb.yml.j2') | from_yaml }}" - vars: - pod_sequence: "{{ pod_hi_idx|int +1 }}" - node_sequence: "{{ node_hi_idx|int +1 }}" - - when: - - workload_args.serviceip is defined and workload_args.serviceip - - workload_args.servicetype | default("clusterip") == "metallb" - - - name: Start Server(s) - total = eligible nodes * density - k8s: - definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" - vars: - pod_sequence: "{{ pod_hi_idx|int +1 }}" - node_sequence: "{{ node_hi_idx|int +1 }}" - - # - # Each server annotates a "node_idx" which will allow its peer client - # to derive its affinity according the 'colocate' variable - # - - name: Wait for pods to be running.... - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - register: server_pods - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Servers" - - when: resource_kind == "pod" - -- block: - ### kind - - name: Waiting for pods to complete.... - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = uperf-bench-client-{{ trunc_uuid }} - register: client_pods - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false - when: "num_server_pods|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" - when: resource_kind == "pod" - -- block: - ### kind - - name: get complete - command: "redis-cli get complete-{{ trunc_uuid }}" - register: complete_status - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false - when: complete_status.stdout == "true" - when: resource_kind == "vm" - diff --git a/roles/uperf/tasks/wait_client_ready.yml b/roles/uperf/tasks/wait_client_ready.yml deleted file mode 100644 index a66ba3b89..000000000 --- a/roles/uperf/tasks/wait_client_ready.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- - -- block: - ### kind - - - name: Get client pod status - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = uperf-bench-client-{{ trunc_uuid }} - register: client_pods - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Clients Running - when: "num_server_pods|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (client_pods | json_query('resources[].status.podIP')|length)" - - when: resource_kind == "pod" - -- block: - ### kind - - - name: set complete to false - command: "redis-cli set complete-{{ trunc_uuid }} false" - - - name: Get count of clients ready - command: "redis-cli get clients-{{ trunc_uuid }}" - register: clients_ready_count - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Clients Running - when: "workload_args.pair|default('1')|int == clients_ready_count.stdout|int" - - when: resource_kind == "vm" - diff --git a/roles/uperf/tasks/wait_server_ready.yml b/roles/uperf/tasks/wait_server_ready.yml deleted file mode 100644 index 3bb11cecb..000000000 --- a/roles/uperf/tasks/wait_server_ready.yml +++ /dev/null @@ -1,55 +0,0 @@ ---- -- block: - ### kind - - - name: Get server pods - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - register: server_pods - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" - when: "num_server_pods|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" - - when: resource_kind == "pod" - -- block: - ### kind - - - name: Wait for vms to be running.... - k8s_facts: - kind: VirtualMachineInstance - api_version: kubevirt.io/v1alpha3 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - register: server_vms - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" - when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - - - name: blocking client from running uperf - command: "redis-cli set start-{{ trunc_uuid }} false" - with_items: "{{ server_vms.resources }}" - when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - - when: resource_kind == "vm" - - diff --git a/roles/uperf/tasks/wait_set_done.yml b/roles/uperf/tasks/wait_set_done.yml deleted file mode 100644 index f82d0ccbf..000000000 --- a/roles/uperf/tasks/wait_set_done.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- - -- block: - - block: - ### kind - - name: read pod completion count - command: "redis-cli get num_completion-{{trunc_uuid}}" - register: num_completion - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Run Next Set - - when: "num_completion.stdout|int == ((benchmark_state.resources[0].status.node_idx|int +1) * (benchmark_state.resources[0].status.pod_idx|int +1))" - - when: resource_kind == "pod" - - ### no kind block - Run a "set" is not yet supported - - when: resource_kind == "pod" - -# -# No kind block - It has not been adapted to scale mode yet. -# diff --git a/roles/uperf/templates/configmap.yml.j2 b/roles/uperf/templates/configmap.yml.j2 index 59c558c2d..a213467fe 100644 --- a/roles/uperf/templates/configmap.yml.j2 +++ b/roles/uperf/templates/configmap.yml.j2 @@ -63,4 +63,4 @@ data: {% endfor %} {% endfor %} {% endfor %} -{% endfor %} +{% endfor %} \ No newline at end of file diff --git a/roles/uperf/templates/configmap_script.yml.j2 b/roles/uperf/templates/configmap_script.yml.j2 index 884bea473..2977a4f55 100644 --- a/roles/uperf/templates/configmap_script.yml.j2 +++ b/roles/uperf/templates/configmap_script.yml.j2 @@ -57,4 +57,4 @@ data: done; redis-cli -h {{bo.resources[0].status.podIP}} set start-{{trunc_uuid}} false redis-cli -h {{bo.resources[0].status.podIP}} del clients-{{trunc_uuid}} - redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true + redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true \ No newline at end of file diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 9602acba9..95a205cb9 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -1,88 +1,60 @@ --- {% set control_port = 30000 %} -apiVersion: v1 -kind: List -metadata: {} -items: -{% macro job_template(item, node_idx_item='') %} - - kind: Job - apiVersion: batch/v1 - metadata: -{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} - name: 'uperf-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }}' -{% else %} - name: 'uperf-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }}' -{% endif %} - namespace: "{{ operator_namespace }}" +kind: Job +apiVersion: batch/v1 +metadata: + name: 'uperf-server-{{ item }}-{{ trunc_uuid }}' + namespace: "{{ operator_namespace }}" {% if workload_args.annotations is defined or workload_args.server_annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% for annotation, value in workload_args.server_annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} - spec: - ttlSecondsAfterFinished: 600 - backoffLimit: 0 - template: - metadata: - labels: - benchmark-uuid: {{ uuid }} - benchmark-operator-workload: uperf - benchmark-operator-role: server - index: "{{item}}" -{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% else %} - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% endif %} - - type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - annotations: +spec: + ttlSecondsAfterFinished: 600 + backoffLimit: 0 + template: + metadata: + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + benchmark-operator-role: server + app: uperf-bench-server-{{item}}-{{ trunc_uuid }} + index: "{{item}}" + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} {% if workload_args.multus.enabled is sameas true %} - k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} + annotations: + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} {% endif %} - node_idx: '{{ node_idx_item }}' - pod_idx: '{{ item }}' - spec: + spec: {% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" + runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} {% if workload_args.hostnetwork is sameas true %} - hostNetwork: true + hostNetwork: true {% endif %} - containers: - - name: benchmark - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} {% if workload_args.server_resources is defined %} - resources: {{ workload_args.server_resources | to_json }} + resources: {{ workload_args.server_resources | to_json }} {% endif %} - imagePullPolicy: Always - command: ["/bin/sh","-c"] - args: ["uperf -s -v -P {{ control_port|int + (0 if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "clusterip" )) else (item|int * 100)) }}"] - restartPolicy: Never + imagePullPolicy: Always + command: ["/bin/sh","-c"] + args: ["uperf -s -v -P {{ control_port|int + (0 if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "clusterip" )) else (item|int * 100)) }}"] + restartPolicy: Never {% if workload_args.pin is sameas true %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_server }}' + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_server }}' {% endif %} - nodeSelector: - kubernetes.io/hostname: '{{ worker_node_list[node_idx_item] }}' - {% if workload_args.serviceip is sameas true %} - securityContext: - sysctls: - - name: net.ipv4.ip_local_port_range - value: 30000 32011 + securityContext: + sysctls: + - name: net.ipv4.ip_local_port_range + value: 30000 32011 {% endif %} -{% filter indent(width=4, first=True) %} -{% include "metadata.yml.j2" %} -{% endfilter %} -{% endmacro %} -{% for node_idx_item in range(node_sequence|int) %} -{% for item in range(pod_sequence|int) %} -{{ job_template(item,node_idx_item) }} -{% endfor %} -{% endfor %} - +{% include "metadata.yml.j2" %} \ No newline at end of file diff --git a/roles/uperf/templates/server_vm.yml.j2 b/roles/uperf/templates/server_vm.yml.j2 index 744a58103..3a7bfb583 100644 --- a/roles/uperf/templates/server_vm.yml.j2 +++ b/roles/uperf/templates/server_vm.yml.j2 @@ -75,4 +75,4 @@ spec: - redis-cli -h {{ bo.resources[0].status.podIP }} incr {{ trunc_uuid }} - uperf -s -v -P 30000 name: cloudinitdisk -status: {} +status: {} \ No newline at end of file diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 0ae245972..84bdfa0db 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -1,76 +1,54 @@ --- {% set control_port = 30000 %} +kind: Service apiVersion: v1 -kind: List -metadata: {} -items: -{% macro job_template(item, node_idx_item='') %} - - kind: Service - apiVersion: v1 - metadata: - name: uperf-service-{{ item }}-{{ trunc_uuid }} - namespace: '{{ operator_namespace }}' - labels: - benchmark-uuid: {{ uuid }} - benchmark-operator-workload: uperf - index: "{{item}}" -{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% else %} - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% endif %} - type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - annotations: - node_idx: '{{ node_idx_item }}' - pod_idx: '{{ item }}' - spec: - selector: -{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} - app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% else %} - app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% endif %} +metadata: + name: uperf-service-{{ item }}-{{ trunc_uuid }} + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + index: "{{item}}" + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} +spec: + selector: + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} {% if workload_args.servicetype | default("clusterip") == "nodeport" %} - type: NodePort - ports: - - name: uperf - port: {{ control_port|int + item|int * 100 }} - targetPort: {{ control_port|int + item|int * 100 }} - nodePort: {{ control_port|int + item * 100 }} - protocol: TCP + type: NodePort + ports: + - name: uperf + port: {{ control_port|int + item|int * 100 }} + targetPort: {{ control_port|int + item|int * 100 }} + nodePort: {{ control_port|int + item|int * 100 }} + protocol: TCP {% for num in range(1,12,1) %} - - name: uperf-control-tcp-{{ control_port|int + item * 100 + num }} - port: {{ control_port|int + item * 100 + num }} - targetPort: {{ control_port|int + item * 100 + num }} - nodePort: {{ control_port|int + item * 100 + num }} - protocol: TCP - - name: uperf-control-udp-{{ control_port|int + item * 100 + num }} - port: {{ control_port|int + item * 100 + num }} - targetPort: {{ control_port|int + item * 100 + num }} - nodePort: {{ control_port|int + item * 100 + num }} - protocol: UDP + - name: uperf-control-tcp-{{ control_port|int + item|int * 100 + num }} + port: {{ control_port|int + item|int * 100 + num }} + targetPort: {{ control_port|int + item|int * 100 + num }} + nodePort: {{ control_port|int + item|int * 100 + num }} + protocol: TCP + - name: uperf-control-udp-{{ control_port|int + item|int * 100 + num }} + port: {{ control_port|int + item|int * 100 + num }} + targetPort: {{ control_port|int + item|int * 100 + num }} + nodePort: {{ control_port|int + item|int * 100 + num }} + protocol: UDP {% endfor %} {% else %} - type: ClusterIP - ports: - - name: uperf - port: 30000 - targetPort: 30000 - protocol: TCP + type: ClusterIP + ports: + - name: uperf + port: 30000 + targetPort: 30000 + protocol: TCP {% for num in range(30001,30012,1) %} - - name: uperf-control-tcp-{{num}} - port: {{num}} - targetPort: {{num}} - protocol: TCP - - name: uperf-control-udp-{{num}} - port: {{num}} - targetPort: {{num}} - protocol: UDP -{% endfor %} -{% endif %} -{% endmacro %} -{% for node_idx_item in range(node_sequence|int) %} -{% for item in range(pod_sequence|int) %} -{{ job_template(item,node_idx_item) }} -{% endfor %} + - name: uperf-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: TCP + - name: uperf-control-udp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: UDP {% endfor %} +{% endif %} \ No newline at end of file diff --git a/roles/uperf/templates/service_metallb.yml.j2 b/roles/uperf/templates/service_metallb.yml.j2 index e19b33c7b..7e1b550ad 100644 --- a/roles/uperf/templates/service_metallb.yml.j2 +++ b/roles/uperf/templates/service_metallb.yml.j2 @@ -1,90 +1,62 @@ --- +kind: Service apiVersion: v1 -kind: List -metadata: {} -items: -{% macro job_template(item, node_idx_item='') %} - - kind: Service - apiVersion: v1 - metadata: - name: uperf-service-{{ item }}-{{ trunc_uuid }} - namespace: '{{ operator_namespace }}' - labels: - benchmark-uuid: {{ uuid }} - benchmark-operator-workload: uperf -{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% else %} - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% endif %} - type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - annotations: - node_idx: '{{ node_idx_item }}' - pod_idx: '{{ item }}' - metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' - metallb.universe.tf/allow-shared-ip: uperf-service-{{ item }}-{{ trunc_uuid }} - spec: - selector: -{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} - app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% else %} - app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% endif %} - externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' - type: LoadBalancer - ports: - - name: uperf - port: 30000 - targetPort: 30000 - protocol: TCP +metadata: + name: uperf-service-{{ item }}-{{ trunc_uuid }} + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + index: "{{ item }}" + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + annotations: + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-service-{{ item }}-{{ trunc_uuid }} +spec: + selector: + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf + port: 30000 + targetPort: 30000 + protocol: TCP {% for num in range(30001,30012,1) %} - - name: uperf-control-tcp-{{num}} - port: {{num}} - targetPort: {{num}} - protocol: TCP + - name: uperf-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: TCP {% endfor %} - - kind: Service - apiVersion: v1 - metadata: - name: uperf-service-{{ item }}-{{ trunc_uuid }}-udp - namespace: '{{ operator_namespace }}' - labels: - benchmark-uuid: {{ uuid }} - benchmark-operator-workload: uperf -{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% else %} - app: uperf-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% endif %} - type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}-udp - annotations: - node_idx: '{{ node_idx_item }}' - pod_idx: '{{ item }}' - metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' - metallb.universe.tf/allow-shared-ip: uperf-service-{{ item }}-{{ trunc_uuid }} - spec: - selector: -{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} - app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% else %} - app: uperf-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} -{% endif %} - externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' - type: LoadBalancer - ports: - - name: uperf - port: 30000 - targetPort: 30000 - protocol: UDP +--- +kind: Service +apiVersion: v1 +metadata: + name: uperf-service-{{ item }}-{{ trunc_uuid }}-udp + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + index: "{{ item }}" + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}-udp + annotations: + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-service-{{ item }}-{{ trunc_uuid }} +spec: + selector: + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf + port: 30000 + targetPort: 30000 + protocol: UDP {% for num in range(30001,30012,1) %} - - name: uperf-control-tcp-{{num}} - port: {{num}} - targetPort: {{num}} - protocol: UDP -{% endfor %} -{% endmacro %} -{% for node_idx_item in range(node_sequence|int) %} -{% for item in range(pod_sequence|int) %} -{{ job_template(item,node_idx_item) }} -{% endfor %} + - name: uperf-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: UDP {% endfor %} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index c514352ae..7cc559cf3 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -1,172 +1,133 @@ --- {% set control_port = 30000 %} -apiVersion: v1 -kind: List -metadata: {} -items: -{% for item in resource_item %} - - kind: Job - apiVersion: batch/v1 - metadata: +kind: Job +apiVersion: batch/v1 +metadata: {% if workload_args.serviceip is sameas true %} {% if workload_args.servicetype | default("clusterip") == "nodeport" %} - name: 'uperf-client-{{item.status.hostIP}}-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.status.hostIP}}-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }}' {% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} - name: 'uperf-client-{{item.status.loadBalancer.ingress[0].ip}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.status.loadBalancer.ingress[0].ip}}-{{ trunc_uuid }}' {% else %} - name: 'uperf-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}' {% endif %} {% else %} - name: 'uperf-client-{{item.status.podIP}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.status.podIP}}-{{ trunc_uuid }}' {% endif %} - namespace: '{{ operator_namespace }}' + namespace: '{{ operator_namespace }}' {% if workload_args.annotations is defined or workload_args.server_annotations is defined %} - annotations: + annotations: {% for annotation, value in workload_args.annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% for annotation, value in workload_args.server_annotations.items() %} - "{{annotation}}": "{{value}}" + "{{annotation}}": "{{value}}" {% endfor %} {% endif %} - spec: - template: - metadata: - labels: - benchmark-uuid: {{ uuid }} - benchmark-operator-workload: uperf - benchmark-operator-role: client - app: uperf-bench-client-{{ trunc_uuid }} - clientfor: {{ item.metadata.labels.app }} - type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} +spec: + template: + metadata: + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + benchmark-operator-role: client + app: uperf-bench-client-{{ trunc_uuid }} + clientfor: {{ item.metadata.labels.app }} + type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} {% if workload_args.multus.enabled is sameas true %} - annotations: - k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} + annotations: + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} {% endif %} - spec: + spec: {% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" + runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} {% if workload_args.hostnetwork is sameas true %} - hostNetwork: true - serviceAccountName: benchmark-operator -{% endif %} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ item.metadata.labels.app }} - topologyKey: kubernetes.io/hostname - containers: - - name: benchmark - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} - env: - - name: uuid - value: "{{ uuid }}" - - name: test_user - value: "{{ test_user | default("ripsaw") }}" - - name: clustername - value: "{{ clustername }}" + hostNetwork: true + serviceAccountName: benchmark-operator +{% endif %} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ item.metadata.labels.app }} + topologyKey: kubernetes.io/hostname + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} + env: + - name: uuid + value: "{{ uuid }}" + - name: test_user + value: "{{ test_user | default("ripsaw") }}" + - name: clustername + value: "{{ clustername }}" {% if elasticsearch.url %} - - name: es - value: "{{ elasticsearch.url }}" - - name: es_index - value: "{{ elasticsearch.index_name | default("ripsaw-uperf") }}" - - name: parallel - value: "{{ elasticsearch.parallel | default(false) }}" - - name: es_verify_cert - value: "{{ elasticsearch.verify_cert | default(true) }}" + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("ripsaw-uperf") }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(true) }}" {% endif %} {% if prometheus is defined %} - - name: prom_es - value: "{{ prometheus.es_url }}" - - name: prom_parallel - value: "{{ prometheus.es_parallel | default(false) }}" - - name: prom_token - value: "{{ prometheus.prom_token | default() }}" - - name: prom_url - value: "{{ prometheus.prom_url | default() }}" -{% endif %} - - name: client_node - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: server_node - value: "{{ uperf.pin_server|default("unknown") }}" + - name: prom_es + value: "{{ prometheus.es_url }}" + - name: prom_parallel + value: "{{ prometheus.es_parallel | default(false) }}" + - name: prom_token + value: "{{ prometheus.prom_token | default() }}" + - name: prom_url + value: "{{ prometheus.prom_url | default() }}" +{% endif %} + - name: client_node + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: server_node + value: "{{ workload_args.pin_server|default("unknown") }}" {% if workload_args.client_resources is defined %} - resources: {{ workload_args.client_resources | to_json }} + resources: {{ workload_args.client_resources | to_json }} {% endif %} - imagePullPolicy: Always - command: ["/bin/sh", "-c"] - args: + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + args: {% if workload_args.serviceip is sameas true %} - - "export serviceip=true; + - "export serviceip=true; {% if workload_args.servicetype | default("clusterip") == "nodeport" %} - export h={{item.status.hostIP}}; - export servicetype={{workload_args.servicetype}}; + export h={{item.status.hostIP}}; + export servicetype={{workload_args.servicetype}}; {% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} - export h={{item.status.loadBalancer.ingress[0].ip}}; - export servicetype={{workload_args.servicetype}}; + export h={{item.status.loadBalancer.ingress[0].ip}}; + export servicetype={{workload_args.servicetype}}; {% else %} - export h={{item.spec.clusterIP}}; - export servicetype={{workload_args.servicetype | default("clusterip")}}; -{% endif %} + export h={{item.spec.clusterIP}}; + export servicetype={{workload_args.servicetype | default("clusterip")}}; +{% endif %} {% else %} {% if workload_args.multus.client is defined %} - - "export multus_client={{workload_args.multus.client}}; - export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; + - "export multus_client={{workload_args.multus.client}}; + export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; {% else %} - - "export h={{item.status.podIP}}; -{% endif %} -{% endif %} - export port={{ control_port + (0 if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "clusterip" )) else (item.metadata.labels.index|int * 100))}} - -{% if (workload_args.colocate is defined) %} - export colocate={{ workload_args.colocate}}; + - "export h={{item.status.podIP}}; {% endif %} -{% if workload_args.step_size is defined %} - export stepsize={{ workload_args.step_size }}; -{% endif %} -{% if workload_args.node_range is defined %} - export node_range='{{ workload_args.node_range[0] }}_{{ workload_args.node_range[1] }}'; -{% endif %} -{% if workload_args.density_range is defined %} - export density_range='{{ workload_args.density_range[0] }}_{{ workload_args.density_range[1] }}'; {% endif %} + export port={{ control_port + (0 if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "clusterip" )) else (item.metadata.labels.index|int * 100))}} {% if workload_args.networkpolicy is defined %} - export networkpolicy={{workload_args.networkpolicy}}; -{% endif %} - export hostnet={{workload_args.hostnetwork}}; - export my_node_idx={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; - export my_pod_idx={{ (item['metadata']['annotations']['pod_idx'] | from_json) }}; - export ips=$(hostname -I); - export num_pairs=1 - export node_count=0; - export pod_count=0; - node_limit=0; - pod_limit=0; - STR=''; - while true; do - STR=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{trunc_uuid}}); - state=$(echo $STR | cut -f1 -d-); - if [[ $state =~ 'true' ]]; then - node_limit=$(echo $STR | cut -f2 -d-); - pod_limit=$(echo $STR | cut -f3 -d-); - if [[ $my_node_idx -gt $node_limit || $my_pod_idx -gt $pod_limit ]]; then - sleep 0.5; continue; - fi; - - echo 'UPERF-run-context num_node=' $((node_limit+1)) 'density=' $((pod_limit+1)) 'my_node_idx=' $my_node_idx 'my_pod_idx=' $my_pod_idx; - node_count=$((node_limit+1)); - pod_count=$((pod_limit+1)); - num_pairs=$((pod_limit+1)); - + export networkpolicy={{workload_args.networkpolicy}}; +{% endif %} + export hostnet={{workload_args.hostnetwork}}; + export ips=$(hostname -I); + while true; do + if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{ trunc_uuid }}) =~ 'true' ]]; then {% for test in workload_args.test_types %} {% for proto in workload_args.protos %} {% for size in workload_args.sizes %} @@ -178,67 +139,34 @@ items: {% set rsize = size %} {% endif %} {% for nthr in workload_args.nthrs %} - cat /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; - run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ + cat /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; + run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ {% if workload_args.run_id is defined %} - --run-id {{workload_args.run_id}} \ + --run-id {{workload_args.run_id}} \ {% endif %} {% if workload_args.debug is defined and workload_args.debug %} - -v \ + -v \ {% endif %} ; {% endfor %} {% endfor %} {% endfor %} {% endfor %} - redis-cli -h {{bo.resources[0].status.podIP}} incr num_completion-{{trunc_uuid}}; - while true; do - state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{trunc_uuid}}); - if [[ $state =~ 'restart' ]]; then - break; - elif [[ $state =~ 'done' ]]; then - break; - else - sleep 0.5; continue; - fi; - done; - if [[ $state =~ 'restart' ]]; then - sleep 0.5; continue; - fi; - - elif [[ $state =~ 'done' ]]; then - break; - else - sleep 0.5; continue; - fi; - break; - done; - " - volumeMounts: - - name: config-volume - mountPath: "/tmp/uperf-test" - volumes: - - name: config-volume - configMap: - name: uperf-test-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }} - restartPolicy: Never -{% if workload_args.pin is sameas false %} -{% if workload_args.colocate is sameas true %} - nodeSelector: - # client node same as server node - kubernetes.io/hostname: "{{ worker_node_list[item['metadata']['annotations']['node_idx'] | from_json] }}" -{% else %} - nodeSelector: - # skew client node one position to the right in the worker_node_list - kubernetes.io/hostname: "{{ worker_node_list[ (1+(item['metadata']['annotations']['node_idx'] | from_json)) % (worker_node_list|length)] }}" -{% endif %} - -{% else %} + else + continue; + fi; + break; + done; + redis-cli -h {{bo.resources[0].status.podIP}} set start-{{ trunc_uuid }} false" + volumeMounts: + - name: config-volume + mountPath: "/tmp/uperf-test" + volumes: + - name: config-volume + configMap: + name: uperf-test-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }} + restartPolicy: Never {% if workload_args.pin is sameas true %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' -{% endif %} - -{% endif %} - -{% endfor %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client }}' +{% endif %} \ No newline at end of file diff --git a/roles/uperf/templates/workload_vm.yml.j2 b/roles/uperf/templates/workload_vm.yml.j2 index 5cc88d12a..35808f20a 100644 --- a/roles/uperf/templates/workload_vm.yml.j2 +++ b/roles/uperf/templates/workload_vm.yml.j2 @@ -95,4 +95,4 @@ spec: - configMap: name: uperf-run-script-{{item.1.status.interfaces[0].ipAddress}}-{{ trunc_uuid }} name: run-config-disk -status: {} +status: {} \ No newline at end of file diff --git a/roles/uperf/vars/main.yml b/roles/uperf/vars/main.yml index aaee090cd..fc9dc0bf4 100644 --- a/roles/uperf/vars/main.yml +++ b/roles/uperf/vars/main.yml @@ -1,11 +1,3 @@ --- -# vars file for bench -cleanup: true -worker_node_list: [] -pod_low_idx: "0" -pod_hi_idx: "0" -node_low_idx: "0" -node_hi_idx: "0" -node_idx: "0" -pod_idx: "0" -all_run_done: false +# vars file for uperf +cleanup: true \ No newline at end of file From 42582adb8048be66aadc4edbe0f6e7a3b64ece4d Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 6 Jul 2022 18:32:25 +0200 Subject: [PATCH 221/249] Fix VM scenario Signed-off-by: Raul Sevilla --- roles/uperf/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index 8c6118748..ab77da38f 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -268,7 +268,7 @@ - block: - name: set complete to false - command: "redis-cli set complete false" + command: "redis-cli set complete-{{ trunc_uuid }} false" - name: Get client vm status k8s_facts: @@ -334,7 +334,7 @@ - block: - name: get complete - command: "redis-cli get complete" + command: "redis-cli get complete-{{ trunc_uuid }}" register: complete_status - operator_sdk.util.k8s_status: @@ -423,4 +423,4 @@ state: Complete complete: true - when: resource_state.resources[0].status.state == "Cleanup" \ No newline at end of file + when: resource_state.resources[0].status.state == "Cleanup" From 94005cd2c892b14ddffcff7c73f94efa2640022d Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy <70236227+mukrishn@users.noreply.github.com> Date: Thu, 7 Jul 2022 11:27:39 -0400 Subject: [PATCH 222/249] missing env var num_pairs (#784) --- roles/uperf/templates/workload.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 7cc559cf3..c6a6fda77 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -125,6 +125,7 @@ spec: export networkpolicy={{workload_args.networkpolicy}}; {% endif %} export hostnet={{workload_args.hostnetwork}}; + export num_pairs={{workload_args.pair| default('1')}}; export ips=$(hostname -I); while true; do if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{ trunc_uuid }}) =~ 'true' ]]; then From 12701a6019ee3c619617f27cde7232efae16c905 Mon Sep 17 00:00:00 2001 From: Vishnu Challa Date: Mon, 11 Jul 2022 15:33:44 -0400 Subject: [PATCH 223/249] Added exit status for uperf failures (#782) * initial draft of standard uperf with new features * bug fix for client creation in nodeport service * bug fix for client creation conflict due to same name in metadata * added index for metallb service labels * bug fix for ports mismatch between client and server pairs in without service and with clusterip service case * bug fix for clusterip service ports mismatch * reverting quay repo org to cloudbulldozer * removed test configuration * adding exit status to capture uperf failures * fix for client start issue caused by redis variables for vm use case * renamed client_vm to client_vms for vm kind * fixing redis variables for pod use cases as well * Merge nodeport and hostnetwork tests Signed-off-by: Raul Sevilla * added a task to mark benchmark status as failure for pod errors * updating the image * fixed typo in failure message * removing an old comment Co-authored-by: root Co-authored-by: root Co-authored-by: Raul Sevilla --- roles/benchmark_state/tasks/failure.yml | 6 +++++- roles/uperf/tasks/main.yml | 8 +++++++- roles/uperf/templates/workload.yml.j2 | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/roles/benchmark_state/tasks/failure.yml b/roles/benchmark_state/tasks/failure.yml index c8ec38455..cdbc1efea 100644 --- a/roles/benchmark_state/tasks/failure.yml +++ b/roles/benchmark_state/tasks/failure.yml @@ -1,4 +1,8 @@ --- +# Removing {{ ansible_failed_task}} and {{ ansible_failed_result }} as they are being very unstable across ansible releases. +# Related issues: +# - https://stackoverflow.com/questions/41104241/ansible-failed-task-ansible-failed-result-variables-are-undefined-when-res +# - https://github.com/ansible/ansible/issues/64789 - name: Failure State operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 @@ -9,6 +13,6 @@ uuid: "{{ uuid }}" complete: True state: Failed - message: "Reconcile failed on {{ ansible_failed_task.name }} {{ ansible_failed_task.action }} \n ```{{ ansible_failed_result | to_nice_json }}```" + message: "Benchmark failed, please check the logs for more details" suuid: "{{ trunc_uuid }}" metadata: "not collected" diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index ab77da38f..410a70bc6 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -320,6 +320,12 @@ - app = uperf-bench-client-{{ trunc_uuid }} register: client_pods + - name: Check for client pod failures + include_role: + name: benchmark_state + tasks_from: failure + when: "(client_pods|json_query('resources[].status[]')|selectattr('phase','match','Failed')|list|length) > 0" + - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark @@ -423,4 +429,4 @@ state: Complete complete: true - when: resource_state.resources[0].status.state == "Cleanup" + when: resource_state.resources[0].status.state == "Cleanup" \ No newline at end of file diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index c6a6fda77..fe32a40a6 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -127,6 +127,7 @@ spec: export hostnet={{workload_args.hostnetwork}}; export num_pairs={{workload_args.pair| default('1')}}; export ips=$(hostname -I); + exit_status=0; while true; do if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{ trunc_uuid }}) =~ 'true' ]]; then {% for test in workload_args.test_types %} @@ -149,6 +150,9 @@ spec: -v \ {% endif %} ; + if [[ $? -ne 0 ]]; then + exit_status=1; + fi; {% endfor %} {% endfor %} {% endfor %} @@ -158,7 +162,8 @@ spec: fi; break; done; - redis-cli -h {{bo.resources[0].status.podIP}} set start-{{ trunc_uuid }} false" + redis-cli -h {{bo.resources[0].status.podIP}} set start-{{ trunc_uuid }} false; + exit $exit_status;" volumeMounts: - name: config-volume mountPath: "/tmp/uperf-test" From 41eb251c875cda79a8f2671a1dd6fc583b17e427 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 12 Jul 2022 09:59:57 +0200 Subject: [PATCH 224/249] The current mechanism to get the tag name doesn't work well. When a new commit lands to the master branch, the master branch is not named correctly i.e: ``` rsevilla@wonderland /tmp/foo (master) $ git describe --tags 2>/dev/null || git rev-parse --abbrev-ref HEAD | sed 's/master/latest/g' v1.0.0-1-g487fada rsevilla@wonderland /tmp/foo (master) $ git log --oneline | head 487fada Foo 12701a6 Added exit status for uperf failures (#782) 94005cd missing env var num_pairs (#784) ``` Signed-off-by: Raul Sevilla --- Makefile | 4 ++-- hack/tag_name.sh | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100755 hack/tag_name.sh diff --git a/Makefile b/Makefile index ca4b52791..33341a3c0 100644 --- a/Makefile +++ b/Makefile @@ -41,8 +41,8 @@ ARCH ?= $(shell uname -m | sed 's/x86_64/amd64/') REGISTRY ?= quay.io ORG ?= cloud-bulldozer # Get the current branch name -# In case this is the master branch, rename it to latest -VERSION ?= $(shell git describe --tags 2>/dev/null || git rev-parse --abbrev-ref HEAD | sed 's/master/latest/g') +# In case is the master branch, rename it to latest +VERSION ?= $(shell hack/tag_name.sh) IMG ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) ifdef IMAGE_ARCH IMG := $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION)-$(IMAGE_ARCH) diff --git a/hack/tag_name.sh b/hack/tag_name.sh new file mode 100755 index 000000000..0ff164f41 --- /dev/null +++ b/hack/tag_name.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +if [[ -z $(git branch --show-current) ]]; then + git describe --tags --abbrev=0 +else + git branch --show-current | sed 's/master/latest/g' +fi From 18bb2d1d4123e3ba1a6dbbc19c3ab5c33eff36b0 Mon Sep 17 00:00:00 2001 From: Vishnu Challa Date: Tue, 12 Jul 2022 18:23:00 -0400 Subject: [PATCH 225/249] Uperf scale mode code changes (#776) * initial draft of standard uperf with new features * bug fix for client creation in nodeport service * bug fix for client creation conflict due to same name in metadata * initial draft for uperf-scale mode * removed uperf to avoid confusion * added index for metallb service labels * fix for client port mismatch with server and services * fixed es index issues * fixed services issue * bug fix for clusterip service ports mismatch * reverting quay repo org to cloudbulldozer * fix for service ip usecases except for nodeport * fixed ports issues for nodeport service use case * added test for uperf_scale * restoring image repo to original * removing bash eternal history * removed test configuration * fix for client start issue caused by redis variables for vm use case * renamed client_vm to client_vms for vm kind * fixing redis variables for pod use cases as well * fixed typo * resolving PR comments * removed an old comment Co-authored-by: root Co-authored-by: root Co-authored-by: Murali Krishnasamy <70236227+mukrishn@users.noreply.github.com> --- config/samples/uperf-scale/cr.yaml | 56 ++++ roles/uperf-scale/README.md | 38 +++ roles/uperf-scale/defaults/main.yml | 9 + roles/uperf-scale/handlers/main.yml | 2 + roles/uperf-scale/meta/main.yml | 52 ++++ roles/uperf-scale/tasks/cleanup.yml | 75 ++++++ roles/uperf-scale/tasks/init.yml | 21 ++ roles/uperf-scale/tasks/main.yml | 43 ++++ roles/uperf-scale/tasks/next_set.yml | 104 ++++++++ roles/uperf-scale/tasks/run_a_set.yml | 26 ++ roles/uperf-scale/tasks/setup.yml | 79 ++++++ roles/uperf-scale/tasks/start_client.yml | 63 +++++ roles/uperf-scale/tasks/start_server.yml | 59 +++++ roles/uperf-scale/tasks/wait_client_done.yml | 28 ++ roles/uperf-scale/tasks/wait_client_ready.yml | 26 ++ roles/uperf-scale/tasks/wait_server_ready.yml | 24 ++ roles/uperf-scale/tasks/wait_set_done.yml | 28 ++ roles/uperf-scale/templates/configmap.yml.j2 | 66 +++++ .../templates/configmap_script.yml.j2 | 60 +++++ roles/uperf-scale/templates/server.yml.j2 | 98 +++++++ roles/uperf-scale/templates/service.yml.j2 | 94 +++++++ .../templates/service_metallb.yml.j2 | 96 +++++++ roles/uperf-scale/templates/workload.yml.j2 | 239 ++++++++++++++++++ roles/uperf-scale/vars/main.yml | 11 + tests/common.sh | 2 + tests/test_crs/valid_uperf_scale.yaml | 46 ++++ .../valid_uperf_scale_networkpolicy.yaml | 47 ++++ .../test_crs/valid_uperf_scale_resources.yaml | 53 ++++ .../valid_uperf_scale_serviceip_nodeport.yaml | 45 ++++ .../test_crs/valid_uperf_scale_servieip.yaml | 45 ++++ tests/test_uperf_scale.sh | 49 ++++ 31 files changed, 1684 insertions(+) create mode 100644 config/samples/uperf-scale/cr.yaml create mode 100644 roles/uperf-scale/README.md create mode 100644 roles/uperf-scale/defaults/main.yml create mode 100644 roles/uperf-scale/handlers/main.yml create mode 100644 roles/uperf-scale/meta/main.yml create mode 100644 roles/uperf-scale/tasks/cleanup.yml create mode 100644 roles/uperf-scale/tasks/init.yml create mode 100644 roles/uperf-scale/tasks/main.yml create mode 100644 roles/uperf-scale/tasks/next_set.yml create mode 100644 roles/uperf-scale/tasks/run_a_set.yml create mode 100644 roles/uperf-scale/tasks/setup.yml create mode 100644 roles/uperf-scale/tasks/start_client.yml create mode 100644 roles/uperf-scale/tasks/start_server.yml create mode 100644 roles/uperf-scale/tasks/wait_client_done.yml create mode 100644 roles/uperf-scale/tasks/wait_client_ready.yml create mode 100644 roles/uperf-scale/tasks/wait_server_ready.yml create mode 100644 roles/uperf-scale/tasks/wait_set_done.yml create mode 100644 roles/uperf-scale/templates/configmap.yml.j2 create mode 100644 roles/uperf-scale/templates/configmap_script.yml.j2 create mode 100644 roles/uperf-scale/templates/server.yml.j2 create mode 100644 roles/uperf-scale/templates/service.yml.j2 create mode 100644 roles/uperf-scale/templates/service_metallb.yml.j2 create mode 100644 roles/uperf-scale/templates/workload.yml.j2 create mode 100644 roles/uperf-scale/vars/main.yml create mode 100644 tests/test_crs/valid_uperf_scale.yaml create mode 100644 tests/test_crs/valid_uperf_scale_networkpolicy.yaml create mode 100644 tests/test_crs/valid_uperf_scale_resources.yaml create mode 100644 tests/test_crs/valid_uperf_scale_serviceip_nodeport.yaml create mode 100644 tests/test_crs/valid_uperf_scale_servieip.yaml create mode 100644 tests/test_uperf_scale.sh diff --git a/config/samples/uperf-scale/cr.yaml b/config/samples/uperf-scale/cr.yaml new file mode 100644 index 000000000..c70564243 --- /dev/null +++ b/config/samples/uperf-scale/cr.yaml @@ -0,0 +1,56 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-benchmark + namespace: benchmark-operator +spec: + clustername: myk8scluster + elasticsearch: + url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443 + #test_user: username_to_attach_to_metadata + workload: + # cleanup: true + name: uperf-scale + args: + serviceip: false + hostnetwork: false + networkpolicy: false + multus: + enabled: false + samples: 5 + kind: pod + # pin: false + # + # 'pair' sepcifies fixed number of client-server pairs for "Pin" mode, + # If 'pair' is NOT present, it will use 'density_range' which allows + # enumeration in addition to fixed number of pair. + test_types: + - stream + protos: + - tcp + sizes: + - 16384 + nthrs: + - 1 + runtime: 30 + + # The following variables are for 'Scale' mode. + # The 'Scale' mode is activated when 'pin=false' or undefined. + # The Scale mode params are: colocate, denstisy_range, node_range and step_size. + # + colocate: false + density_range: [1, 1] + node_range: [1, 1] + step_size: add1 + # Valid step_size values are: addN or log2 + # N can be any decimal number + # Enumeration examples: + # add1: 1,2,3,4 ,,, + # add2: 1,3,5,7 ... + # add10: 1,11,21,31 ... + # log2: 1,2,4,8,16,32 ,,, + # + # 'exclude_labels' specifies the list of ineligible worker nodes. + # exclude_labels: (OR conditional, every node that matches any of these labels is excluded) + # - "bad=true" + # - "fc640=true" \ No newline at end of file diff --git a/roles/uperf-scale/README.md b/roles/uperf-scale/README.md new file mode 100644 index 000000000..225dd44b9 --- /dev/null +++ b/roles/uperf-scale/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/roles/uperf-scale/defaults/main.yml b/roles/uperf-scale/defaults/main.yml new file mode 100644 index 000000000..6561363be --- /dev/null +++ b/roles/uperf-scale/defaults/main.yml @@ -0,0 +1,9 @@ +--- +# defaults file for uperf-scale +resource_kind: "{{ workload.args.kind | default('pod') }}" +uperf-scale: + proto: tcp + test_type: stream + nthr: 1 + size: 1024 + runtime: 60 \ No newline at end of file diff --git a/roles/uperf-scale/handlers/main.yml b/roles/uperf-scale/handlers/main.yml new file mode 100644 index 000000000..e0cd86577 --- /dev/null +++ b/roles/uperf-scale/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for uperf-scale diff --git a/roles/uperf-scale/meta/main.yml b/roles/uperf-scale/meta/main.yml new file mode 100644 index 000000000..c572acc9f --- /dev/null +++ b/roles/uperf-scale/meta/main.yml @@ -0,0 +1,52 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.1 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/roles/uperf-scale/tasks/cleanup.yml b/roles/uperf-scale/tasks/cleanup.yml new file mode 100644 index 000000000..34359a1a4 --- /dev/null +++ b/roles/uperf-scale/tasks/cleanup.yml @@ -0,0 +1,75 @@ +--- + +- block: + ### kind + # Cleanup servers, but leave clients around mostly for further examining of results. + - name: Get Server Jobs + k8s_facts: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_jobs + + - name: Get Server Pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: Server Job and Pod names - to clean + set_fact: + clean_jobs: | + [ + {% for item in server_jobs.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + clean_pods: | + [ + {% for item in server_pods.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + + - name: Cleanup server Job + k8s: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_jobs }}" + + - name: Cleanup server Pod + k8s: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_pods }}" + + when: resource_kind == "pod" and cleanup == True + +- block: + - name: Cleanup redis + command: "{{ item }}" + with_items: + - redis-cli del num_completion-{{trunc_uuid}} + - redis-cli del start-{{trunc_uuid}} + when: resource_kind == "pod" + +- operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + \ No newline at end of file diff --git a/roles/uperf-scale/tasks/init.yml b/roles/uperf-scale/tasks/init.yml new file mode 100644 index 000000000..253243328 --- /dev/null +++ b/roles/uperf-scale/tasks/init.yml @@ -0,0 +1,21 @@ +--- + +- name: Clear start flag + command: "redis-cli set start-{{trunc_uuid}} 0" + +- name: Clear num_completion + command: "redis-cli set num_completion-{{trunc_uuid}} 0" + +- name: Init node and pod indices in benchmark context + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + pod_hi_idx: "{{pod_hi_idx}}" + pod_low_idx: "{{pod_low_idx}}" + node_hi_idx: "{{node_hi_idx}}" + node_low_idx: "{{node_low_idx}}" + node_idx: "{{node_low_idx}}" + pod_idx: "{{pod_low_idx}}" diff --git a/roles/uperf-scale/tasks/main.yml b/roles/uperf-scale/tasks/main.yml new file mode 100644 index 000000000..6a0ac6389 --- /dev/null +++ b/roles/uperf-scale/tasks/main.yml @@ -0,0 +1,43 @@ +--- + +- include_tasks: setup.yml + +- include_tasks: start_server.yml + when: benchmark_state.resources[0].status.state == "Building" + +- block: + + - include_tasks: wait_server_ready.yml + when: benchmark_state.resources[0].status.state == "Starting Servers" + + - include_tasks: start_client.yml + when: benchmark_state.resources[0].status.state == "Starting Clients" + + - include_tasks: wait_client_ready.yml + when: benchmark_state.resources[0].status.state == "Waiting for Clients" + + # LOOP BEGIN + # This loop iterates density_range[] and node_range[] for "scale" mode + + - include_tasks: run_a_set.yml + when: benchmark_state.resources[0].status.state == "Clients Running" + + - include_tasks: wait_set_done.yml + when: benchmark_state.resources[0].status.state == "Set Running" + + + - include_tasks: next_set.yml + when: benchmark_state.resources[0].status.state == "Run Next Set" + # will loop back to "Client Running" state, or FALLTHRU to "Running" + # state below and finish + + # LOOP END + + - include_tasks: wait_client_done.yml + when: benchmark_state.resources[0].status.state == "Running" + + - include_tasks: cleanup.yml + when: benchmark_state.resources[0].status.state == "Cleanup" + + when: resource_kind == "pod" + \ No newline at end of file diff --git a/roles/uperf-scale/tasks/next_set.yml b/roles/uperf-scale/tasks/next_set.yml new file mode 100644 index 000000000..c8c74f07b --- /dev/null +++ b/roles/uperf-scale/tasks/next_set.yml @@ -0,0 +1,104 @@ +--- +# +# This module logically implements an RE-ENTRANT nested "for" loops; +# +# with_items: +# range (node_low_idx, node_hi_idx) +# range (pod_low_idx, pod_hi_idx) +# +# Each iteration executes one item, and each re-entrance +# continues where it left off. +# +- block: + - name: Read previous node_idx and pod_idx + set_fact: + all_run_done: False + inc: "{{workload_args.step_size|default('add1')}}" + amount: 0 + pod_idx: "{{benchmark_state.resources[0].status.pod_idx|int}}" + node_idx: "{{benchmark_state.resources[0].status.node_idx|int}}" + + - name: Extract add amount + set_fact: + amount: "{{ inc | regex_replace('[^0-9]', '') }}" + inc: add + when: "'add' in inc" + + - name: Increment pod_idx + set_fact: + pod_idx: "{%-if inc=='add' -%}{{pod_idx|int+amount|int}} + {%-elif inc=='log2' -%}{{(pod_idx|int*2)+1}} + {%-else -%}{{pod_idx|int+1}} + {% endif %}" + - block: + # + # This block starts a new node loop + # + - name: Increment node_idx + set_fact: + node_idx: "{%- if inc=='add' -%}{{node_idx|int+amount|int}} + {%- elif inc=='log2' -%}{{(node_idx|int *2)+1}} + {%- else -%}{{node_idx|int+1}} + {% endif %}" + + - name: Check node loop for ending condition + set_fact: + all_run_done: True + when: "node_idx|int > benchmark_state.resources[0].status.node_hi_idx|int" + + # + # Reset pod_idx AFTER node_idx tasks above, else cond change + # causes it to skip node_idx tasks + # + - name: Reset pod_idx to pod_low_idx + set_fact: + pod_idx: "{{benchmark_state.resources[0].status.pod_low_idx}}" + + when: "pod_idx|int > benchmark_state.resources[0].status.pod_hi_idx|int" + + - block: + # + # All done + # + - name: Unpause pods to complete + command: "redis-cli set start-{{trunc_uuid}} done" + + - name: Change state to proceed to exit + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Running + + when: all_run_done == True + + - block: + # + # More round(s) to run. + # + - name: Send redis restart signal + command: "redis-cli set start-{{trunc_uuid}} restart" + + - name: Reset redis num_completion + command: "redis-cli set num_completion-{{trunc_uuid}} 0" + + - name: Change state to run next round + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + pod_idx: "{{pod_idx}}" + node_idx: "{{node_idx}}" + + when: all_run_done == False + + when: resource_kind == "pod" + +# +# No block - Scale mode support is N/A +# diff --git a/roles/uperf-scale/tasks/run_a_set.yml b/roles/uperf-scale/tasks/run_a_set.yml new file mode 100644 index 000000000..ca0e44f76 --- /dev/null +++ b/roles/uperf-scale/tasks/run_a_set.yml @@ -0,0 +1,26 @@ +--- + +- block: + # + # Entry Condition: + # 1. A previous task has set 'node_idx' and 'pod_idx' in benchmark ctx + # 2. All cliest are polling redis for 'start-node_idx-pod_idx' to start + # Output: Clients with node_idx <= redis node_idx && pod_idx <= redis pod_ix will run + # + + - name: Signal group to run + command: "redis-cli set start-{{trunc_uuid}} true-{{benchmark_state.resources[0].status.node_idx|int}}-{{benchmark_state.resources[0].status.pod_idx|int}}" + + - name: Update state to "Set Running" + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Set Running + when: resource_kind == "pod" + +# +# No kind - It has not been adapted to Scale mode +# diff --git a/roles/uperf-scale/tasks/setup.yml b/roles/uperf-scale/tasks/setup.yml new file mode 100644 index 000000000..455b798d6 --- /dev/null +++ b/roles/uperf-scale/tasks/setup.yml @@ -0,0 +1,79 @@ +--- + # + # Scale mode logic where client and server pods are spreaded + # across all eligible nodes + # +- name: List Nodes Labeled as Workers + k8s_info: + api_version: v1 + kind: Node + label_selectors: + - "node-role.kubernetes.io/worker=" + register: node_list + no_log: True + +- name: Isolate Worker Role Hostnames + set_fact: + worker_node_list: "{{ node_list | json_query('resources[].metadata.labels.\"kubernetes.io/hostname\"') | list }}" + +- name: List Nodes Labeled with {{ workload_args.exclude_label }} + k8s_info: + api_version: v1 + kind: Node + label_selectors: + - '{{ item }}' + with_items: "{{ workload_args.exclude_labels }}" + register: exclude_node_list + when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 + +- name: Isolate Worker Role Hostnames for label {{ workload_args.exclude_label }} + set_fact: + worker_node_exclude_list: "{{ exclude_node_list | json_query('results[].resources[].metadata.name') }}" + +- name: Exclude labeled nodes + set_fact: + worker_node_list: "{{ worker_node_list | difference(worker_node_exclude_list) }}" + when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 + # + # Compute node and pod limits using CR params while taking into account + # of the actual number of nodes available in the system + # +- name: init pod and node low/hi idx + set_fact: + pod_low_idx: "{{ workload_args.density_range[0] | default('1')|int - 1 }}" + pod_hi_idx: "{{ workload_args.density_range[1] | default('1')|int - 1 }}" + node_low_idx: "{{ workload_args.node_range[0] | default('1')|int - 1 }}" + node_hi_idx: "{{ workload_args.node_range[1] | default('1')|int - 1 }}" + # + # Next sanity check and massage the indices if necessary. + # We shall complete gracefully and not iterate wildly. + # +- name: Adjust node_hi_idx if cluster has less nodes + set_fact: + node_hi_idx: "{{ worker_node_list|length| default('0')|int -1 }}" + when: "node_hi_idx|int >= worker_node_list|length| default('0')|int " + +- name: Adjust node_low_idx if necessary + set_fact: + node_low_idx: "{{node_hi_idx|int}}" + when: "node_low_idx|int > node_hi_idx|int" + +- name: Adjust pod_low_idx if necessary + set_fact: + pod_low_idx: "{{pod_hi_idx|int}}" + when: "pod_low_idx|int > pod_hi_idx|int" + +- name: Record num server pods using new worker_node_list + # in Scale mode, num server pods = num_node * number_pod + set_fact: + num_server_pods: "{{ (node_hi_idx|int+1) * (pod_hi_idx|int+1) }}" + +- name: Capture ServiceIP + k8s_facts: + kind: Service + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: serviceip + when: workload_args.serviceip is defined and workload_args.serviceip diff --git a/roles/uperf-scale/tasks/start_client.yml b/roles/uperf-scale/tasks/start_client.yml new file mode 100644 index 000000000..ae620066b --- /dev/null +++ b/roles/uperf-scale/tasks/start_client.yml @@ -0,0 +1,63 @@ +--- + +- name: Get pod info + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods + +- name: set pod_sequence + set_fact: + pod_sequence: "{{ (num_server_pods|int - 1) if (workload_args.serviceip|default(False) == True and workload_args.servicetype | default('clusterip') == 'nodeport') else (pod_hi_idx|int) }}" + +- name: Generate uperf-scale xml files + k8s: + definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" + with_sequence: start=0 end={{ pod_sequence|int }} + +- block: + ### kind + - name: Start Client(s) w/o serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + vars: + resource_item: "{{ server_pods.resources }}" + when: + - ( workload_args.serviceip|default(False) == False and server_pods.resources|length > 0 ) + + + - name: Start Client(s) with nodeport serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + vars: + resource_item: "{{ server_pods.resources }}" + when: + - ( workload_args.serviceip|default(False) == True and server_pods.resources|length > 0 and + workload_args.servicetype | default("clusterip") == "nodeport" ) + + # + # Each server annotates a "node_idx". Each peer client will + # derive its affinity according the 'colocate' variable. + # + + - name: Start Client(s) with serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + vars: + resource_item: "{{ serviceip.resources }}" + when: + - workload_args.serviceip|default(False) == True and serviceip.resources|length > 0 + - workload_args.servicetype | default("clusterip") != "nodeport" + when: resource_kind == "pod" + + +- operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Waiting for Clients diff --git a/roles/uperf-scale/tasks/start_server.yml b/roles/uperf-scale/tasks/start_server.yml new file mode 100644 index 000000000..fbc4325e9 --- /dev/null +++ b/roles/uperf-scale/tasks/start_server.yml @@ -0,0 +1,59 @@ +--- + +- block: + ### kind + - include_tasks: init.yml + + - name: Create service for server pods + k8s: + definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}" + vars: + pod_sequence: "{{ pod_hi_idx|int +1 }}" + node_sequence: "{{ node_hi_idx|int +1 }}" + + when: + - workload_args.serviceip is defined and workload_args.serviceip + - ( workload_args.servicetype | default("clusterip") == "clusterip" ) or + ( workload_args.servicetype | default("clusterip") == "nodeport" ) + + - name: Create metal LB service for server pods + k8s: + definition: "{{ lookup('template', 'service_metallb.yml.j2') | from_yaml }}" + vars: + pod_sequence: "{{ pod_hi_idx|int +1 }}" + node_sequence: "{{ node_hi_idx|int +1 }}" + + when: + - workload_args.serviceip is defined and workload_args.serviceip + - workload_args.servicetype | default("clusterip") == "metallb" + + - name: Start Server(s) - total = eligible nodes * density + k8s: + definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" + vars: + pod_sequence: "{{ pod_hi_idx|int +1 }}" + node_sequence: "{{ node_hi_idx|int +1 }}" + + # + # Each server annotates a "node_idx" which will allow its peer client + # to derive its affinity according the 'colocate' variable + # + - name: Wait for pods to be running.... + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Servers" + + when: resource_kind == "pod" diff --git a/roles/uperf-scale/tasks/wait_client_done.yml b/roles/uperf-scale/tasks/wait_client_done.yml new file mode 100644 index 000000000..dacbb97b6 --- /dev/null +++ b/roles/uperf-scale/tasks/wait_client_done.yml @@ -0,0 +1,28 @@ +--- +- block: + ### kind + - name: Waiting for pods to complete.... + k8s_facts: + kind: pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-scale-bench-client-{{ trunc_uuid }} + register: client_pods + + - name: Check for client pod failures + include_role: + name: benchmark_state + tasks_from: failure + when: "(client_pods|json_query('resources[].status[]')|selectattr('phase','match','Failed')|list|length) > 0" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Cleanup + complete: false + when: "num_server_pods|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: resource_kind == "pod" diff --git a/roles/uperf-scale/tasks/wait_client_ready.yml b/roles/uperf-scale/tasks/wait_client_ready.yml new file mode 100644 index 000000000..672d0d8b7 --- /dev/null +++ b/roles/uperf-scale/tasks/wait_client_ready.yml @@ -0,0 +1,26 @@ +--- + +- block: + ### kind + + - name: Get client pod status + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-scale-bench-client-{{ trunc_uuid }} + register: client_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + when: "num_server_pods|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (client_pods | json_query('resources[].status.podIP')|length)" + + when: resource_kind == "pod" + \ No newline at end of file diff --git a/roles/uperf-scale/tasks/wait_server_ready.yml b/roles/uperf-scale/tasks/wait_server_ready.yml new file mode 100644 index 000000000..ca0324744 --- /dev/null +++ b/roles/uperf-scale/tasks/wait_server_ready.yml @@ -0,0 +1,24 @@ +--- +- block: + ### kind + + - name: Get server pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Clients" + when: "num_server_pods|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + + when: resource_kind == "pod" diff --git a/roles/uperf-scale/tasks/wait_set_done.yml b/roles/uperf-scale/tasks/wait_set_done.yml new file mode 100644 index 000000000..f82d0ccbf --- /dev/null +++ b/roles/uperf-scale/tasks/wait_set_done.yml @@ -0,0 +1,28 @@ +--- + +- block: + - block: + ### kind + - name: read pod completion count + command: "redis-cli get num_completion-{{trunc_uuid}}" + register: num_completion + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Run Next Set + + when: "num_completion.stdout|int == ((benchmark_state.resources[0].status.node_idx|int +1) * (benchmark_state.resources[0].status.pod_idx|int +1))" + + when: resource_kind == "pod" + + ### no kind block - Run a "set" is not yet supported + + when: resource_kind == "pod" + +# +# No kind block - It has not been adapted to scale mode yet. +# diff --git a/roles/uperf-scale/templates/configmap.yml.j2 b/roles/uperf-scale/templates/configmap.yml.j2 new file mode 100644 index 000000000..9387b250c --- /dev/null +++ b/roles/uperf-scale/templates/configmap.yml.j2 @@ -0,0 +1,66 @@ +--- +{% set control_port = 30000 %} +apiVersion: v1 +kind: ConfigMap +metadata: + name: uperf-scale-test-{{ item }}-{{ trunc_uuid }} + namespace: '{{ operator_namespace }}' +data: +{% for test in workload_args.test_types %} +{% for proto in workload_args.protos %} +{% for size in workload_args.sizes %} +{% if size is iterable %} +{% set wsize = size[0] %} +{% set rsize = size[1] %} +{% else %} +{% set wsize = size %} +{% set rsize = size %} +{% endif %} +{% for nthr in workload_args.nthrs %} + uperf-scale-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} : | + + + {% if ( 'rr' == test ) %} + + + + + + + + + + + + {% endif %} + {% if ( 'stream' == test or 'bidirec' == test ) %} + + + + + + + + + + + + {% endif %} + {% if ( 'maerts' == test or 'bidirec' == test ) %} + + + + + + + + + + + + {% endif %} + +{% endfor %} +{% endfor %} +{% endfor %} +{% endfor %} diff --git a/roles/uperf-scale/templates/configmap_script.yml.j2 b/roles/uperf-scale/templates/configmap_script.yml.j2 new file mode 100644 index 000000000..462087f8e --- /dev/null +++ b/roles/uperf-scale/templates/configmap_script.yml.j2 @@ -0,0 +1,60 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: uperf-scale-run-script-{{item.status.interfaces[0].ipAddress}}-{{ trunc_uuid }} + namespace: '{{ operator_namespace }}' +data: + run_script.sh : | + export h={{item.status.interfaces[0].ipAddress}} +{% if elasticsearch.url %} + export es={{ elasticsearch.url }} + export es_index={{ elasticsearch.index_name | default("ripsaw-uperf") }} + export es_verify_cert={{ elasticsearch.verify_cert | default(true) }} + export parallel={{ elasticsearch.parallel | default(false) }} + export uuid={{uuid}} +{% endif %} +{% if test_user is defined %} + export test_user={{test_user}} +{% endif %} + export clustername={{clustername}} + export hostnet={{workload_args.hostnetwork}} + export ips=$(hostname -I) + echo "Setting ready, waiting for signal to start..." + redis-cli -h {{bo.resources[0].status.podIP}} setnx clients-{{trunc_uuid}} 0 + redis-cli -h {{bo.resources[0].status.podIP}} incr clients-{{trunc_uuid}} + while true; do + BO_START=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{ trunc_uuid }}) + CLIENTS_READY=$(redis-cli -h {{bo.resources[0].status.podIP}} get clients-{{ trunc_uuid }}) + SERVERS_READY=$(redis-cli -h {{bo.resources[0].status.podIP}} get {{ trunc_uuid }}) + if [[ ("${BO_START}" =~ 'true') && ("${CLIENTS_READY}" == "${SERVERS_READY}") ]]; then +{% for test in workload_args.test_types %} +{% for proto in workload_args.protos %} +{% for size in workload_args.sizes %} +{% if size is iterable %} +{% set wsize = size[0] %} +{% set rsize = size[1] %} +{% else %} +{% set wsize = size %} +{% set rsize = size %} +{% endif %} +{% for nthr in workload_args.nthrs %} + cat /tmp/uperf-scale-test/uperf-scale-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; + run_snafu --tool uperf -w /tmp/uperf-scale-test/uperf-scale-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; +{% endfor %} +{% endfor %} +{% endfor %} +{% endfor %} + else + sleep 0.1; + continue; + fi; + break; + done; + redis-cli -h {{bo.resources[0].status.podIP}} set start-{{trunc_uuid}} false + redis-cli -h {{bo.resources[0].status.podIP}} del clients-{{trunc_uuid}} + redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true diff --git a/roles/uperf-scale/templates/server.yml.j2 b/roles/uperf-scale/templates/server.yml.j2 new file mode 100644 index 000000000..e86111921 --- /dev/null +++ b/roles/uperf-scale/templates/server.yml.j2 @@ -0,0 +1,98 @@ +--- +{% set control_port = 30000 %} +{% set counter = { + 'counter': 0, + } +%} +{% macro increment(dct, key, inc=1)%} + {% if dct.update({key: dct[key] + inc}) %} {% endif %} +{% endmacro %} +apiVersion: v1 +kind: List +metadata: {} +items: +{% macro job_template(item, node_idx_item='') %} + - kind: Job + apiVersion: batch/v1 + metadata: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + name: 'uperf-scale-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }}' +{% else %} + name: 'uperf-scale-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }}' +{% endif %} + namespace: "{{ operator_namespace }}" +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} + spec: + ttlSecondsAfterFinished: 600 + backoffLimit: 0 + template: + metadata: + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf-scale + benchmark-operator-role: server +{% if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "nodeport" )) %} + index: "{{ counter["counter"] }}" +{% set port_index = counter["counter"] %} +{{ increment(counter, 'counter') }} +{% else %} +{% set port_index = item %} + index: "{{ item }}" +{% endif %} +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + annotations: +{% if workload_args.multus.enabled is sameas true %} + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} +{% endif %} + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.hostnetwork is sameas true %} + hostNetwork: true +{% endif %} + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} +{% if workload_args.server_resources is defined %} + resources: {{ workload_args.server_resources | to_json }} +{% endif %} + imagePullPolicy: Always + command: ["/bin/sh","-c"] + args: ["uperf -s -v -P {{ control_port|int + (0 if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "clusterip" )) else (port_index * 100)) }}"] + restartPolicy: Never + nodeSelector: + kubernetes.io/hostname: '{{ worker_node_list[node_idx_item] }}' + +{% if workload_args.serviceip is sameas true %} + securityContext: + sysctls: + - name: net.ipv4.ip_local_port_range + value: 30000 32011 +{% endif %} +{% filter indent(width=4, first=True) %} +{% include "metadata.yml.j2" %} +{% endfilter %} +{% endmacro %} +{% for node_idx_item in range(node_sequence|int) %} +{% for item in range(pod_sequence|int) %} +{{ job_template(item,node_idx_item) }} +{% endfor %} +{% endfor %} + diff --git a/roles/uperf-scale/templates/service.yml.j2 b/roles/uperf-scale/templates/service.yml.j2 new file mode 100644 index 000000000..6fe3de922 --- /dev/null +++ b/roles/uperf-scale/templates/service.yml.j2 @@ -0,0 +1,94 @@ +--- +{% set control_port = 30000 %} +{% set counter = { + 'counter': 0, + } +%} +{% macro increment(dct, key, inc=1)%} + {% if dct.update({key: dct[key] + inc}) %} {% endif %} +{% endmacro %} +apiVersion: v1 +kind: List +metadata: {} +items: +{% macro job_template(item, node_idx_item='') %} + - kind: Service + apiVersion: v1 + metadata: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + name: uperf-scale-service-{{worker_node_list[node_idx_item] | truncate(26,true,'') | replace('.','-')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + name: uperf-scale-service-{{worker_node_list[node_idx_item] | truncate(27,true,'') | replace('.','-')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf-scale +{% if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "nodeport" )) %} + index: "{{ counter["counter"] }}" +{% set port_index = counter["counter"] %} +{{ increment(counter, 'counter') }} +{% else %} +{% set port_index = item %} + index: "{{ item }}" +{% endif %} +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + annotations: + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' + spec: + selector: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + type: NodePort + ports: + - name: uperf-scale + port: {{ control_port|int + port_index * 100 }} + targetPort: {{ control_port|int + port_index * 100 }} + nodePort: {{ control_port|int + port_index * 100 }} + protocol: TCP +{% for num in range(1,12,1) %} + - name: uperf-scale-control-tcp-{{ control_port|int + port_index * 100 + num }} + port: {{ control_port|int + port_index * 100 + num }} + targetPort: {{ control_port|int + port_index * 100 + num }} + nodePort: {{ control_port|int + port_index * 100 + num }} + protocol: TCP + - name: uperf-scale-control-udp-{{ control_port|int + port_index * 100 + num }} + port: {{ control_port|int + port_index * 100 + num }} + targetPort: {{ control_port|int + port_index * 100 + num }} + nodePort: {{ control_port|int + port_index * 100 + num }} + protocol: UDP +{% endfor %} +{% else %} + type: ClusterIP + ports: + - name: uperf-scale + port: 30000 + targetPort: 30000 + protocol: TCP +{% for num in range(30001,30012,1) %} + - name: uperf-scale-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: TCP + - name: uperf-scale-control-udp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: UDP +{% endfor %} +{% endif %} +{% endmacro %} +{% for node_idx_item in range(node_sequence|int) %} +{% for item in range(pod_sequence|int) %} +{{ job_template(item,node_idx_item) }} +{% endfor %} +{% endfor %} diff --git a/roles/uperf-scale/templates/service_metallb.yml.j2 b/roles/uperf-scale/templates/service_metallb.yml.j2 new file mode 100644 index 000000000..1ea0a1af4 --- /dev/null +++ b/roles/uperf-scale/templates/service_metallb.yml.j2 @@ -0,0 +1,96 @@ +--- +apiVersion: v1 +kind: List +metadata: {} +items: +{% macro job_template(item, node_idx_item='') %} + - kind: Service + apiVersion: v1 + metadata: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + name: uperf-scale-service-{{worker_node_list[node_idx_item] | truncate(26,true,'') | replace('.','-')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + name: uperf-scale-service-{{worker_node_list[node_idx_item] | truncate(27,true,'') | replace('.','-')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf-scale + index: "{{ item }}" +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + annotations: + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-scale-service-{{ item }}-{{ trunc_uuid }} + spec: + selector: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf-scale + port: 30000 + targetPort: 30000 + protocol: TCP +{% for num in range(30001,30012,1) %} + - name: uperf-scale-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: TCP +{% endfor %} + - kind: Service + apiVersion: v1 + metadata: + name: uperf-scale-service-{{ item }}-{{ trunc_uuid }}-udp + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf-scale + index: "{{ item }}" +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}-udp + annotations: + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-scale-service-{{ item }}-{{ trunc_uuid }} + spec: + selector: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf-scale + port: 30000 + targetPort: 30000 + protocol: UDP +{% for num in range(30001,30012,1) %} + - name: uperf-scale-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: UDP +{% endfor %} +{% endmacro %} +{% for node_idx_item in range(node_sequence|int) %} +{% for item in range(pod_sequence|int) %} +{{ job_template(item,node_idx_item) }} +{% endfor %} +{% endfor %} diff --git a/roles/uperf-scale/templates/workload.yml.j2 b/roles/uperf-scale/templates/workload.yml.j2 new file mode 100644 index 000000000..54adf9117 --- /dev/null +++ b/roles/uperf-scale/templates/workload.yml.j2 @@ -0,0 +1,239 @@ +--- +{% set control_port = 30000 %} +apiVersion: v1 +kind: List +metadata: {} +items: +{% for item in resource_item %} + - kind: Job + apiVersion: batch/v1 + metadata: +{% if workload_args.serviceip is sameas true %} +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + name: 'uperf-scale-client-{{item.status.hostIP}}-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }}' +{% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} + name: 'uperf-scale-client-{{item.status.loadBalancer.ingress[0].ip}}-{{ trunc_uuid }}' +{% else %} + name: 'uperf-scale-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}' +{% endif %} +{% else %} + name: 'uperf-scale-client-{{item.status.podIP}}-{{ trunc_uuid }}' +{% endif %} + namespace: '{{ operator_namespace }}' +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} + spec: + template: + metadata: + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf-scale + benchmark-operator-role: client + app: uperf-scale-bench-client-{{ trunc_uuid }} + clientfor: {{ item.metadata.labels.app }} + type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} +{% if workload_args.multus.enabled is sameas true %} + annotations: + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} +{% endif %} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.hostnetwork is sameas true %} + hostNetwork: true + serviceAccountName: benchmark-operator +{% endif %} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ item.metadata.labels.app }} + topologyKey: kubernetes.io/hostname + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} + env: + - name: uuid + value: "{{ uuid }}" + - name: test_user + value: "{{ test_user | default("ripsaw") }}" + - name: clustername + value: "{{ clustername }}" +{% if elasticsearch.url %} + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("ripsaw-uperf") }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(true) }}" +{% endif %} +{% if prometheus is defined %} + - name: prom_es + value: "{{ prometheus.es_url }}" + - name: prom_parallel + value: "{{ prometheus.es_parallel | default(false) }}" + - name: prom_token + value: "{{ prometheus.prom_token | default() }}" + - name: prom_url + value: "{{ prometheus.prom_url | default() }}" +{% endif %} + - name: client_node + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: server_node + value: "{{ item.spec.nodeName|default("unknown") }}" +{% if workload_args.client_resources is defined %} + resources: {{ workload_args.client_resources | to_json }} +{% endif %} + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + args: +{% if workload_args.serviceip is sameas true %} + - "export serviceip=true; +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + export h={{item.status.hostIP}}; + export servicetype={{workload_args.servicetype}}; +{% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} + export h={{item.status.loadBalancer.ingress[0].ip}}; + export servicetype={{workload_args.servicetype}}; +{% else %} + export h={{item.spec.clusterIP}}; + export servicetype={{workload_args.servicetype | default("clusterip")}}; +{% endif %} +{% else %} +{% if workload_args.multus.client is defined %} + - "export multus_client={{workload_args.multus.client}}; + export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; +{% else %} + - "export h={{item.status.podIP}}; +{% endif %} +{% endif %} + export port=30000 +{% if (workload_args.colocate is defined) %} + export colocate={{ workload_args.colocate}}; +{% endif %} +{% if workload_args.step_size is defined %} + export stepsize={{ workload_args.step_size }}; +{% endif %} +{% if workload_args.node_range is defined %} + export node_range='{{ workload_args.node_range[0] }}_{{ workload_args.node_range[1] }}'; +{% endif %} +{% if workload_args.density_range is defined %} + export density_range='{{ workload_args.density_range[0] }}_{{ workload_args.density_range[1] }}'; +{% endif %} +{% if workload_args.networkpolicy is defined %} + export networkpolicy={{workload_args.networkpolicy}}; +{% endif %} + export hostnet={{workload_args.hostnetwork}}; + export my_node_idx={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; + export my_pod_idx={{ (item['metadata']['annotations']['pod_idx'] | from_json) }}; + export ips=$(hostname -I); + export num_pairs=1 + export node_count=0; + export pod_count=0; + node_limit=0; + pod_limit=0; + exit_status=0 + STR=''; + while true; do + STR=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{trunc_uuid}}); + state=$(echo $STR | cut -f1 -d-); + if [[ $state =~ 'true' ]]; then + node_limit=$(echo $STR | cut -f2 -d-); + pod_limit=$(echo $STR | cut -f3 -d-); + if [[ $my_node_idx -gt $node_limit || $my_pod_idx -gt $pod_limit ]]; then + sleep 0.5; continue; + fi; + + echo 'uperf-scale-run-context num_node=' $((node_limit+1)) 'density=' $((pod_limit+1)) 'my_node_idx=' $my_node_idx 'my_pod_idx=' $my_pod_idx; + node_count=$((node_limit+1)); + pod_count=$((pod_limit+1)); + num_pairs=$((pod_limit+1)); + +{% for test in workload_args.test_types %} +{% for proto in workload_args.protos %} +{% for size in workload_args.sizes %} +{% if size is iterable %} +{% set wsize = size[0] %} +{% set rsize = size[1] %} +{% else %} +{% set wsize = size %} +{% set rsize = size %} +{% endif %} +{% for nthr in workload_args.nthrs %} + cat /tmp/uperf-scale-test/uperf-scale-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; + run_snafu --tool uperf -w /tmp/uperf-scale-test/uperf-scale-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ +{% if workload_args.run_id is defined %} + --run-id {{workload_args.run_id}} \ +{% endif %} +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; + if [[ $? -ne 0 ]]; then + exit_status=1; + fi; +{% endfor %} +{% endfor %} +{% endfor %} +{% endfor %} + redis-cli -h {{bo.resources[0].status.podIP}} incr num_completion-{{trunc_uuid}}; + while true; do + state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{trunc_uuid}}); + if [[ $state =~ 'restart' ]]; then + break; + elif [[ $state =~ 'done' ]]; then + break; + else + sleep 0.5; continue; + fi; + done; + if [[ $state =~ 'restart' ]]; then + sleep 0.5; continue; + fi; + + elif [[ $state =~ 'done' ]]; then + break; + else + sleep 0.5; continue; + fi; + break; + done; + exit $exit_status; + " + volumeMounts: + - name: config-volume + mountPath: "/tmp/uperf-scale-test" + volumes: + - name: config-volume + configMap: + name: uperf-scale-test-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }} + restartPolicy: Never +{% if workload_args.colocate is sameas true %} + nodeSelector: + # client node same as server node + kubernetes.io/hostname: "{{ worker_node_list[item['metadata']['annotations']['node_idx'] | from_json] }}" +{% else %} + nodeSelector: + # skew client node one position to the right in the worker_node_list + kubernetes.io/hostname: "{{ worker_node_list[ (1+(item['metadata']['annotations']['node_idx'] | from_json)) % (worker_node_list|length)] }}" +{% endif %} + +{% endfor %} diff --git a/roles/uperf-scale/vars/main.yml b/roles/uperf-scale/vars/main.yml new file mode 100644 index 000000000..ddec85769 --- /dev/null +++ b/roles/uperf-scale/vars/main.yml @@ -0,0 +1,11 @@ +--- +# vars file for uperf-scale +cleanup: false +worker_node_list: [] +pod_low_idx: "0" +pod_hi_idx: "0" +node_low_idx: "0" +node_hi_idx: "0" +node_idx: "0" +pod_idx: "0" +all_run_done: false \ No newline at end of file diff --git a/tests/common.sh b/tests/common.sh index 3843df9d5..ac0a29fee 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -14,6 +14,7 @@ function populate_test_list { # Check for changes in roles if [[ $(echo ${item} | grep 'roles/fs-drift') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/uperf') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/uperf-scale') ]]; then echo "test_uperf_scale.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/fio_distributed') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/iperf3') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/byowl') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi @@ -42,6 +43,7 @@ function populate_test_list { if [[ $(echo ${item} | grep 'valid_smallfile*') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_sysbench*') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_uperf*') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_uperf_scale*') ]]; then echo "test_uperf_scale.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_ycsb*') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_vegeta*') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_stressng*') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi diff --git a/tests/test_crs/valid_uperf_scale.yaml b/tests/test_crs/valid_uperf_scale.yaml new file mode 100644 index 000000000..76473c9f1 --- /dev/null +++ b/tests/test_crs/valid_uperf_scale.yaml @@ -0,0 +1,46 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf-scale + args: + hostnetwork: false + serviceip: false + multus: + enabled: false + samples: 2 + kind: pod + test_types: + - stream + - rr + - bidirec + protos: + - tcp + - udp + sizes: + - 1024 + - [1024, 512] + nthrs: + - 1 + - 2 + runtime: 2 + colocate: false + density_range: [1, 1] + node_range: [1, 5] + step_size: add1 + debug: true diff --git a/tests/test_crs/valid_uperf_scale_networkpolicy.yaml b/tests/test_crs/valid_uperf_scale_networkpolicy.yaml new file mode 100644 index 000000000..5538e6eec --- /dev/null +++ b/tests/test_crs/valid_uperf_scale_networkpolicy.yaml @@ -0,0 +1,47 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-networkpolicy + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf-scale + args: + hostnetwork: false + serviceip: false + networkpolicy: true + multus: + enabled: false + samples: 2 + kind: pod + test_types: + - stream + - rr + - bidirec + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + - 2 + runtime: 2 + colocate: false + density_range: [1, 1] + node_range: [1, 5] + step_size: add1 + debug: true diff --git a/tests/test_crs/valid_uperf_scale_resources.yaml b/tests/test_crs/valid_uperf_scale_resources.yaml new file mode 100644 index 000000000..92574fd3d --- /dev/null +++ b/tests/test_crs/valid_uperf_scale_resources.yaml @@ -0,0 +1,53 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-resources + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf-scale + args: + client_resources: + requests: + cpu: 100m + memory: 100Mi + server_resources: + requests: + cpu: 100m + memory: 100Mi + hostnetwork: true + serviceip: false + multus: + enabled: false + samples: 2 + kind: pod + test_types: + - stream + - rr + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + - 2 + runtime: 2 + colocate: false + density_range: [1, 1] + node_range: [1, 1] + step_size: add1 + debug: true diff --git a/tests/test_crs/valid_uperf_scale_serviceip_nodeport.yaml b/tests/test_crs/valid_uperf_scale_serviceip_nodeport.yaml new file mode 100644 index 000000000..52b1f3f7e --- /dev/null +++ b/tests/test_crs/valid_uperf_scale_serviceip_nodeport.yaml @@ -0,0 +1,45 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-serviceip-nodeport + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf-scale + args: + hostnetwork: false + serviceip: true + servicetype: "nodeport" + multus: + enabled: false + samples: 2 + kind: pod + test_types: + - stream + - rr + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + runtime: 2 + colocate: false + density_range: [1, 1] + node_range: [1, 5] + step_size: add1 + debug: true diff --git a/tests/test_crs/valid_uperf_scale_servieip.yaml b/tests/test_crs/valid_uperf_scale_servieip.yaml new file mode 100644 index 000000000..f3c6b5e7c --- /dev/null +++ b/tests/test_crs/valid_uperf_scale_servieip.yaml @@ -0,0 +1,45 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-serviceip + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf-scale + args: + hostnetwork: false + serviceip: true + multus: + enabled: false + samples: 1 + kind: pod + test_types: + - stream + - rr + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + - 2 + runtime: 2 + colocate: false + density_range: [1, 1] + node_range: [1, 5] + step_size: add1 + debug: true diff --git a/tests/test_uperf_scale.sh b/tests/test_uperf_scale.sh new file mode 100644 index 000000000..c44eacc79 --- /dev/null +++ b/tests/test_uperf_scale.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up Uperf" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_uperf { + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + test_name=$1 + cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr + echo "Performing: ${test_name}" + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + check_benchmark_for_desired_state $benchmark_name Complete 1800s + + + + index="ripsaw-uperf-results" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_uperf "Uperf scale without resources definition" tests/test_crs/valid_uperf_scale.yaml +functional_test_uperf "Uperf scale with ServiceIP" tests/test_crs/valid_uperf_scale_serviceip.yaml +functional_test_uperf "Uperf scale with NodePort ServiceIP" tests/test_crs/valid_uperf_scale_serviceip_nodeport.yaml +functional_test_uperf "Uperf scale with resources definition and hostNetwork" tests/test_crs/valid_uperf_scale_resources.yaml +functional_test_uperf "Uperf scale with networkpolicy" tests/test_crs/valid_uperf_scale_networkpolicy.yaml From 3c5ca1b1debf6720c82090d03336803212a780c4 Mon Sep 17 00:00:00 2001 From: Vishnu Challa Date: Fri, 15 Jul 2022 11:28:24 -0400 Subject: [PATCH 226/249] added README.md for uperf-scale mode code changes (#790) Co-authored-by: root --- docs/uperf-scale.md | 303 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 docs/uperf-scale.md diff --git a/docs/uperf-scale.md b/docs/uperf-scale.md new file mode 100644 index 000000000..c596d153c --- /dev/null +++ b/docs/uperf-scale.md @@ -0,0 +1,303 @@ +# Uperf-Scale + +[Uperf](http://uperf.org/) is a network performance tool + +## Running UPerf-Scale + +Given that you followed instructions to deploy operator, +you can modify [cr.yaml](../config/samples/uperf-scale/cr.yaml) + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: "http://es-instance.com:9200" + workload: + name: uperf-scale + args: + client_resources: + requests: + cpu: 500m + memory: 500Mi + limits: + cpu: 500m + memory: 500Mi + server_resources: + requests: + cpu: 500m + memory: 500Mi + limits: + cpu: 500m + memory: 500Mi + serviceip: false + runtime_class: class_name + hostnetwork: false + networkpolicy: false + kind: pod + multus: + enabled: false + samples: 1 + test_types: + - stream + protos: + - tcp + sizes: + - 16384 + nthrs: + - 1 + runtime: 30 + colocate: false + density_range: [low, high] + node_range: [low, high] + step_size: addN, log2 +``` + +`client_resources` and `server_resources` will create uperf client's and server's containers with the given k8s compute resources respectively [k8s resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) + +`serviceip` will place the uperf server behind a K8s [Service](https://kubernetes.io/docs/concepts/services-networking/service/) + +`runtime_class` If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. + +*Note:* `runtime_class` has only been tested with Kata containers. Only include `runtime_class` if using Kata containers. + +`annotations` If this is set, the benchmark-operator will set the specified annotations on the pods' metadata. + +`server_annotations` If this is set, the benchmark-operator will set the specified annotations on the server pods' metadata. + +`client_annotations` If this is set, the benchmark-operator will set the specified annotations on the client pods' metadata. + +`hostnetwork` will test the performance of the node the pod will run on. + +`networkpolicy` will create a simple networkpolicy for ingress + +`density_range` to determine the number of pairs and `node_range` to determine the number of nodes. + +`multus[1]` Configure our pods to use multus. + +`samples` how many times to run the tests. For example + +[1] https://github.com/intel/multus-cni/tree/master/examples + +```yaml + samples: 3 + density_range: [1,1] + test_types: + - stream + protos: + - tcp + sizes: + - 1024 + - 16384 + nthrs: + - 1 + runtime: 30 +``` + +Will run `stream` w/ `tcp` and message size `1024` three times and +`stream` w/ `tcp` and message size `16384` three times. This will help us +gain confidence in our results. + +### Asymmetric Request-Response + +For the request-response (rr) `test_type`, it is possible to provide the `sizes` values as a +list of two values where the first value is the write size and the second value is the read +size. + +For example: +```yaml + samples: 3 + density_range: [1,1] + test_types: + - rr + protos: + - tcp + sizes: + - 1024 + - [8192, 4096] + nthrs: + - 1 + runtime: 30 +``` +Will run the `rr` test with `tcp`, first with a symmectic size of `1024` and then with an +asymmetric size of `8192` write and `4096` read. + +### Multus + +If the user desires to test with Multus, use the below Multus `NetworkAtachmentDefinition` +as an example: + +``` +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: macvlan-range-0 +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "macvlan", + "master": "eno1", + "mode": "bridge", + "ipam": { + "type": "host-local", + "ranges": [ + [ { + "subnet": "11.10.0.0/16", + "rangeStart": "11.10.1.20", + "rangeEnd": "11.10.3.50" + } ] ] + } + }' +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: macvlan-range-1 +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "macvlan", + "master": "eno1", + "mode": "bridge", + "ipam": { + "type": "host-local", + "ranges": [ + [ { + "subnet": "11.10.0.0/16", + "rangeStart": "11.10.1.60", + "rangeEnd": "11.10.3.90" + } ] ] + } + }' +``` + +This will use the same IP subnet across nodes, but not overlap +IP addresses. + +To enable Multus in Ripsaw, here is the relevant config. + +``` + ... + multus: + enabled: true + client: "macvlan-range-0" + server: "macvlan-range-1" + ... + +``` +### Scale mode params +Scale in this context refers to the ability to enumerate UPERF +client-server pairs during test in a control fashion using the following knobs. + +`colocate: true` will place each client and server pod pair on the same node. + +`density_range` to specify the range of client-server pairs that the test will iterate. + +`node_range` to specify the range of nodes that the test will iterate. + +`step_size` to specify the incrementing method. + +Here is one scale example: + +``` + ... + colocate: false + density_range: [1,10] + node_range: [1,128] + step_size: log2 + ... +``` +In the above sample, the `scale` mode will be activated. The `colocate` option is to keep both server and client pods on same node. In the first phase, the +pod instantion phase, the system gathers node inventory and may reduce the `node_range.high` value +to match the number of worker node available in the cluster. + +According to `node_range: [1,128]`, and `density_range:[1,10]`, the system will instantiate 10 pairs on +each of 128 nodes. Each pair has a node_idx and a pod_idx that are used later to control +which one and when they should run the UPERF workload, After all pairs are up and ready, +next comes the test execution phase. + +The scale mode iterates the test as a double nested loop as follows: +``` + for node with node_idx less-or-equal node_range(low, high. step_size): + for pod with pod_idx less-or-equal density_range(low, high, step_size): + run uperf +``` +Hence, with the above params, the first iteration runs the pair with node_idx/pod_idx of {1,1}. After the first +run has completed, the second interation runs 2 pairs of {1,1} and {1,2} and so on. + +The valid `step_size` methods are: addN and log2. `N` can be any integer and `log2` will double the value at each iteration i.e. 1,2,4,8,16 ... +By choosing the appropriate values for `density_range` and `node_range`, the user can generate most if not all +combinations of UPERF data points to exercise datapath performance from many angles. + +Once done creating/editing the resource file, you can run it by: + +```bash +# kubectl apply -f config/samples/uperf/cr.yaml # if edited the original one +# kubectl apply -f # if created a new cr file +``` +### Advanced Service types + +Benchmark operator now also supports different service types, it can create `NodePort` and `LoadBalancer` (only metallb) +type services along with the current default `ClusterIP` type. + +No pre-requisites needed for `NodePort` service, as long as the ports used by uperf(30000 to 30012) are allowed at the node level, +which is the cluster default. + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) +uperf-service-np NodePort 172.30.177.81 30000:31955/TCP,30001:31935/TCP,30002:31942/TCP,30001:31935/UDP,30002:31942/UDP +``` + +For `metallb` type, there are certain pre-requisites, +1. Installation of [MetalLB](https://metallb.universe.tf/) operator and CRD +2. Configuration of [BGP](https://github.com/metallb/metallb-operator#create-a-bgp-peer-object) +3. Configuration of [AddressPool](https://github.com/metallb/metallb-operator#create-an-address-pool-object) for lb service +4. Configuration of extenal router for BGP + +`metallb` type creates 2 services per benchmark CR (for each protocol, `tcp` and `udp`) and they will share the external IP like below + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) +uperf-service-lb LoadBalancer 172.30.177.99 192.168.216.102 30000:30976/TCP,30001:30652/TCP,30002:30099/TCP +uperf-service-lb2 LoadBalancer 172.30.126.71 192.168.216.102 30001:31312/UDP,30002:30776/UDP +``` + +#### CR file inputs + +##### For NodePort +```yaml + ... + name: uperf + serviceip: true + servicetype: "nodeport" + ... +``` + +##### For MetalLB +`metallb` +```yaml + ... + name: uperf + serviceip: true + servicetype: "metallb" + metallb: + addresspool: "addresspool-l3" + service_etp: "Cluster" # Either `Cluster` or `Local` + ... +``` +You can either access results by indexing them directly or by accessing the console. +The results are stored in /tmp/ directory + + +### Dashboard example + +Using the Elasticsearch storage describe above, we can build dashboards like the below. + +![UPerf Dashboard](https://i.imgur.com/gSVZ9MX.png) + +To reuse the dashboard above, use the json [here](https://github.com/cloud-bulldozer/arsenal/tree/master/uperf/grafana) + +Additionally, by default we will utilize the `uperf-results` index for Elasticsearch. \ No newline at end of file From 6aeaacc59c378f1577f112933658581287548b59 Mon Sep 17 00:00:00 2001 From: Vishnu Challa Date: Fri, 15 Jul 2022 11:38:26 -0400 Subject: [PATCH 227/249] fixing port conflict for uperf bidrec test (#791) Co-authored-by: root --- roles/uperf-scale/templates/configmap.yml.j2 | 2 +- roles/uperf/templates/configmap.yml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/uperf-scale/templates/configmap.yml.j2 b/roles/uperf-scale/templates/configmap.yml.j2 index 9387b250c..2f8862913 100644 --- a/roles/uperf-scale/templates/configmap.yml.j2 +++ b/roles/uperf-scale/templates/configmap.yml.j2 @@ -49,7 +49,7 @@ data: {% if ( 'maerts' == test or 'bidirec' == test ) %} - + diff --git a/roles/uperf/templates/configmap.yml.j2 b/roles/uperf/templates/configmap.yml.j2 index a213467fe..1ace4c5ef 100644 --- a/roles/uperf/templates/configmap.yml.j2 +++ b/roles/uperf/templates/configmap.yml.j2 @@ -49,7 +49,7 @@ data: {% if ( 'maerts' == test or 'bidirec' == test ) %} - + From 1a197926c4b3df71b990110387746da188634f25 Mon Sep 17 00:00:00 2001 From: Joe Talerico Date: Tue, 19 Jul 2022 15:18:21 -0400 Subject: [PATCH 228/249] Race condition in FIO With FIO we need to wait for server pods to be annotated with IP addresses. This fix adds that wait. closes #789 Signed-off-by: Joe Talerico --- roles/fio_distributed/tasks/main.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/roles/fio_distributed/tasks/main.yml b/roles/fio_distributed/tasks/main.yml index 17397e297..a4b93cab9 100644 --- a/roles/fio_distributed/tasks/main.yml +++ b/roles/fio_distributed/tasks/main.yml @@ -65,17 +65,23 @@ - app = fio-benchmark-{{ trunc_uuid }} register: server_pods + - name: Check IP addresses + set_fact: + check_ip: true + when: "( server_pods is defined ) and ( server_pods is mapping ) and (workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.podIP')|length))" + - include_role: name: benchmark_state tasks_from: set_state vars: state: StartingClient - when: "workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.podIP')|length) and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) and benchmark_state.resources[0].status.state == 'StartingServers'" + when: "check_ip|default('false')|bool and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.podIP')|length) and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) and benchmark_state.resources[0].status.state == 'StartingServers'" - name: Create IP list and nodes set_fact: pod_details: "{{ pod_details|default({}) | combine({item.status.podIP: item.spec.nodeName}) }}" with_items: "{{ server_pods.resources }}" + when: "check_ip|default('false')|bool" when: (workload.args.kind | default('pod')) == "pod" From 11b3e099f217aa436c89f62ccfceda0462d3de8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Mon, 1 Aug 2022 15:07:56 +0200 Subject: [PATCH 229/249] Use uperf pod antiAffinity rules (#781) * Affinity rules Use podAntiaffinity rules to place uperf client pods in different worker nodes where the servers are running on Use podAffinity rules to place all client and server pods in the same node, useful when running more than one pair All client pods will be scheduled in the same node and servers in another Signed-off-by: Raul Sevilla * Update docs Signed-off-by: Raul Sevilla --- docs/uperf.md | 65 +-------- roles/uperf/tasks/main.yml | 183 +++++++++----------------- roles/uperf/templates/server.yml.j2 | 12 +- roles/uperf/templates/workload.yml.j2 | 11 +- 4 files changed, 82 insertions(+), 189 deletions(-) diff --git a/docs/uperf.md b/docs/uperf.md index 55c64c7d8..61965f62b 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -54,12 +54,10 @@ spec: nthrs: - 1 runtime: 30 - colocate: false - density_range: [low, high] - node_range: [low, high] - step_size: addN, log2 ``` + By default, the uperf server and client pods will be preferably scheduled on different nodes, thanks to a `podAntiAffinity` rule. In scenarios with more than 1 pair, all clients and servers will be scheduled to the same node. + `client_resources` and `server_resources` will create uperf client's and server's containers with the given k8s compute resources respectively [k8s resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) `serviceip` will place the uperf server behind a K8s [Service](https://kubernetes.io/docs/concepts/services-networking/service/) @@ -85,9 +83,6 @@ spec: `pin_client` what node to pin the client pod to. `pair` how many instances of uperf client-server pairs. `pair` is applicable for `pin: true` only. -If `pair` is not specified, the operator will use the value in `density_range` to detemine the number of pairs. -See **Scale** section for more info. `density_range` can do more than `pair` can, but `pair` support is retained -for backward compatibility. `multus[1]` Configure our pods to use multus. @@ -203,62 +198,6 @@ To enable Multus in Ripsaw, here is the relevant config. pin_client: "openshift-master-1.dev4.kni.lab.eng.bos.redhat.com" ... -``` -### Scale -Scale in this context refers to the ability to enumerate UPERF -client-server pairs during test in a control fashion using the following knobs. - -`colocate: true` will place each client and server pod pair on the same node. - -`density_range` to specify the range of client-server pairs that the test will iterate. - -`node_range` to specify the range of nodes that the test will iterate. - -`step_size` to specify the incrementing method. - -Here is one scale example: - -``` - ... - pin: false - colocate: false - density_range: [1,10] - node_range: [1,128] - step_size: log2 - ... -``` -Note, the `scale` mode is mutually exlusive to `pin` mode with the `pin` mode having higher precedence. -In other words, if `pin:true` the test will deploy pods on `pin_server` and `pin_client` nodes -and ignore `colocate`, `node_range`, and the number of pairs to deploy is specified by the - `density_range.high` value. - -In the above sample, the `scale` mode will be activated since `pin: false`. In the first phase, the -pod instantion phase, the system gathers node inventory and may reduce the `node_range.high` value -to match the number of worker node available in the cluster. - -According to `node_range: [1,128]`, and `density_range:[1,10]`, the system will instantiate 10 pairs on -each of 128 nodes. Each pair has a node_idx and a pod_idx that are used later to control -which one and when they should run the UPERF workload, After all pairs are up and ready, -next comes the test execution phase. - -The scale mode iterates the test as a double nested loop as follows: -``` - for node with node_idx less-or-equal node_range(low, high. step_size): - for pod with pod_idx less-or-equal density_range(low, high, step_size): - run uperf -``` -Hence, with the above params, the first iteration runs the pair with node_idx/pod_idx of {1,1}. After the first -run has completed, the second interation runs 2 pairs of {1,1} and {1,2} and so on. - -The valid `step_size` methods are: addN and log2. `N` can be any integer and `log2` will double the value at each iteration i.e. 1,2,4,8,16 ... -By choosing the appropriate values for `density_range` and `node_range`, the user can generate most if not all -combinations of UPERF data points to exercise datapath performance from many angles. - -Once done creating/editing the resource file, you can run it by: - -```bash -# kubectl apply -f config/samples/uperf/cr.yaml # if edited the original one -# kubectl apply -f # if created a new cr file ``` ### Advanced Service types diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index 410a70bc6..798a9470b 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -1,31 +1,5 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ ansible_operator_meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - name: Capture ServiceIP k8s_facts: kind: Service @@ -61,25 +35,13 @@ register: servers with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} - - name: Wait for pods to be running.... - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - register: server_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting Servers - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Servers" - - when: resource_state.resources[0].status.state == "Building" and resource_kind == "pod" + when: benchmark_state.resources[0].status.state == "Building" and resource_kind == "pod" - block: @@ -89,25 +51,13 @@ register: servers with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} - - name: Wait for vms to be running.... - k8s_facts: - kind: VirtualMachineInstance - api_version: kubevirt.io/v1alpha3 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} - register: server_vms - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Servers" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting Servers - when: resource_state.resources[0].status.state == "Building" and resource_kind == "vm" + when: benchmark_state.resources[0].status.state == "Building" and resource_kind == "vm" - block: @@ -120,18 +70,15 @@ - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Clients" when: "workload_args.pair|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" - when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" + when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" - block: @@ -144,14 +91,11 @@ - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_vms - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Clients" when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - name: blocking client from running uperf @@ -159,7 +103,7 @@ with_items: "{{ server_vms.resources }}" when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" and workload_args.pair|default('1')|int|int == 1 + when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" and workload_args.pair|default('1')|int|int == 1 - block: @@ -231,15 +175,13 @@ when: resource_kind == "vm" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Waiting for Clients + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Waiting for Clients" - when: resource_state.resources[0].status.state == "Starting Clients" + when: benchmark_state.resources[0].status.state == "Starting Clients" - block: @@ -261,6 +203,11 @@ namespace: "{{ operator_namespace }}" status: state: Clients Running + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Clients Running when: "workload_args.pair|default('1')|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" when: resource_kind == "pod" @@ -279,35 +226,29 @@ - app = uperf-bench-client-{{ trunc_uuid }} register: client_vms - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Clients Running + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Clients Running when: "workload_args.pair|default('1')|int == client_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" when: resource_kind == "vm" - when: resource_state.resources[0].status.state == "Waiting for Clients" + when: benchmark_state.resources[0].status.state == "Waiting for Clients" - block: - name: Signal workload command: "redis-cli set start-{{ trunc_uuid }} true" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - when: resource_state.resources[0].status.state == "Clients Running" + when: benchmark_state.resources[0].status.state == "Clients Running" - block: - block: @@ -326,14 +267,11 @@ tasks_from: failure when: "(client_pods|json_query('resources[].status[]')|selectattr('phase','match','Failed')|list|length) > 0" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Cleanup when: "workload_args.pair|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" when: resource_kind == "pod" @@ -343,18 +281,15 @@ command: "redis-cli get complete-{{ trunc_uuid }}" register: complete_status - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ ansible_operator_meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Cleanup when: complete_status.stdout == "true" when: resource_kind == "vm" - when: resource_state.resources[0].status.state == "Running" + when: benchmark_state.resources[0].status.state == "Running" - block: @@ -429,4 +364,4 @@ state: Complete complete: true - when: resource_state.resources[0].status.state == "Cleanup" \ No newline at end of file + when: benchmark_state.resources[0].status.state == "Cleanup" diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index 95a205cb9..2002509ef 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -37,6 +37,16 @@ spec: {% if workload_args.hostnetwork is sameas true %} hostNetwork: true {% endif %} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: type + operator: In + values: + - {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + topologyKey: kubernetes.io/hostname containers: - name: benchmark image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} @@ -57,4 +67,4 @@ spec: - name: net.ipv4.ip_local_port_range value: 30000 32011 {% endif %} -{% include "metadata.yml.j2" %} \ No newline at end of file +{% include "metadata.yml.j2" %} diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index fe32a40a6..f67e6ee37 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -47,6 +47,15 @@ spec: serviceAccountName: benchmark-operator {% endif %} affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: type + operator: In + values: + - {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} + topologyKey: kubernetes.io/hostname podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 @@ -175,4 +184,4 @@ spec: {% if workload_args.pin is sameas true %} nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_client }}' -{% endif %} \ No newline at end of file +{% endif %} From 68671d1e3f705c489fa14458d75099cd7c0d52d5 Mon Sep 17 00:00:00 2001 From: Jeffrey Adkisson <106615289+Jadkisso12@users.noreply.github.com> Date: Wed, 3 Aug 2022 05:18:24 -0400 Subject: [PATCH 230/249] updated README.md (#778) * updated README.md Spelling typos! * Update README.md Another typo * Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e48ca12fc..0930ac86c 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Benchmark-operator includes a series of end 2 end tests that can be triggered in Optional locations for workload images can now be added easily without the need to rebuild the operator. To do so in the workload args section of the CR add image: [location] -NOTE: This is not a required arguement. If omitted it will default to the currently verified workload image. +NOTE: This is not a required argument. If omitted it will default to the currently verified workload image. Additionally, this is *NOT* enabled for YCSB For Example: @@ -116,8 +116,8 @@ spec: Workloads that are triggered through [benchmark-wrapper](https://github.com/cloud-bulldozer/benchmark-wrapper) can optionally pass the debug flag through the workload CR. -NOTE: This is not a required arguement. If omitted it will default to the default logging level of -the benchmark-wrapper +NOTE: This is not a required argument. If omitted it will default to the default logging level of +the benchmark-wrapper. For Example: From 91c975d9198eefc139789854793a89db5198b8ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Castillo=20Lema?= Date: Mon, 8 Aug 2022 14:55:40 +0200 Subject: [PATCH 231/249] SCTP support for pod2service scenarios in uperf workloads (#734) * SCTP support for services * Added a test now that e2e cluster supports SCTP * Added a pod2pod test --- docs/uperf.md | 2 ++ e2e/uperf/uperf.yaml | 1 + e2e/uperf/uperf_serviceip.yaml | 1 + roles/uperf/templates/service.yml.j2 | 13 ++++++++ roles/uperf/templates/service_metallb.yml.j2 | 32 ++++++++++++++++++++ 5 files changed, 49 insertions(+) diff --git a/docs/uperf.md b/docs/uperf.md index 61965f62b..3a969c501 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -84,6 +84,8 @@ spec: `pair` how many instances of uperf client-server pairs. `pair` is applicable for `pin: true` only. +`protos`: choose between `tcp`, `udp` or/and `sctp`. In case `sctp` is choosen, SCTP module has to be previously loaded in the environment. + `multus[1]` Configure our pods to use multus. `samples` how many times to run the tests. For example diff --git a/e2e/uperf/uperf.yaml b/e2e/uperf/uperf.yaml index a9d910c7a..c9706f627 100644 --- a/e2e/uperf/uperf.yaml +++ b/e2e/uperf/uperf.yaml @@ -33,6 +33,7 @@ spec: protos: - tcp - udp + - sctp sizes: - 1024 - [1024, 512] diff --git a/e2e/uperf/uperf_serviceip.yaml b/e2e/uperf/uperf_serviceip.yaml index b12a1eef4..4df1a4fdb 100644 --- a/e2e/uperf/uperf_serviceip.yaml +++ b/e2e/uperf/uperf_serviceip.yaml @@ -30,6 +30,7 @@ spec: protos: - tcp - udp + - sctp sizes: - 512 nthrs: diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 84bdfa0db..b255a936f 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -33,6 +33,13 @@ spec: targetPort: {{ control_port|int + item|int * 100 + num }} nodePort: {{ control_port|int + item|int * 100 + num }} protocol: UDP +{% if "sctp" in workload_args.protos %} + - name: uperf-control-sctp-{{ control_port|int + item|int * 100 + num }} + port: {{ control_port|int + item|int * 100 + num }} + targetPort: {{ control_port|int + item|int * 100 + num }} + nodePort: {{ control_port|int + item|int * 100 + num }} + protocol: SCTP +{% endif %} {% endfor %} {% else %} type: ClusterIP @@ -50,5 +57,11 @@ spec: port: {{num}} targetPort: {{num}} protocol: UDP +{% if "sctp" in workload_args.protos %} + - name: uperf-control-sctp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: SCTP +{% endif %} {% endfor %} {% endif %} \ No newline at end of file diff --git a/roles/uperf/templates/service_metallb.yml.j2 b/roles/uperf/templates/service_metallb.yml.j2 index 7e1b550ad..6ff7d8410 100644 --- a/roles/uperf/templates/service_metallb.yml.j2 +++ b/roles/uperf/templates/service_metallb.yml.j2 @@ -60,3 +60,35 @@ spec: targetPort: {{num}} protocol: UDP {% endfor %} +--- +{% if "sctp" in workload_args.protos %} +kind: Service +apiVersion: v1 +metadata: + name: uperf-service-{{ item }}-{{ trunc_uuid }}-sctp + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }}} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}-udp + annotations: + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-service-{{ item }}-{{ trunc_uuid }} +spec: + selector: + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf + port: 30000 + targetPort: 30000 + protocol: SCTP +{% for num in range(30001,30012,1) %} + - name: uperf-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: SCTP +{% endfor %} +{% endif %} \ No newline at end of file From 0e6043a1473c895ea381712effa35e4c27808850 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 10 Aug 2022 15:18:26 +0200 Subject: [PATCH 232/249] Benchmark-operator's affinity preference to workload node Set benchmark-operator's affinity to preferably be scheduled on workload nodes Signed-off-by: Raul Sevilla --- config/manager/manager.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 70dcdb2b1..234917434 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -22,6 +22,14 @@ spec: labels: control-plane: controller-manager spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/workload + operator: Exists containers: - name: manager args: From 351b7e20241c0428110bf197f7ff4ddce88aa8dc Mon Sep 17 00:00:00 2001 From: Shekhar Berry Date: Fri, 23 Sep 2022 16:19:12 +0530 Subject: [PATCH 233/249] WIll allow FIO to run test with pvcvolumemode as Block (#797) * Adding option to run tests with pvcvolumemode: Block * Running FIO tests with pvcvolumemode set as Block * Update kustomization.yaml --- roles/fio_distributed/tasks/main.yml | 4 ++-- roles/fio_distributed/templates/configmap.yml.j2 | 8 ++++++++ roles/fio_distributed/templates/servers.yaml | 10 +++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/roles/fio_distributed/tasks/main.yml b/roles/fio_distributed/tasks/main.yml index a4b93cab9..60ecf5378 100644 --- a/roles/fio_distributed/tasks/main.yml +++ b/roles/fio_distributed/tasks/main.yml @@ -2,8 +2,8 @@ - block: - name: Provide path for fio - PV set_fact: - fio_path: /mnt/pvc - when: workload_args.storageclass is defined + fio_path: /dev/xvda + when: workload_args.storageclass is defined - name: Provide path for fio - No PV set_fact: diff --git a/roles/fio_distributed/templates/configmap.yml.j2 b/roles/fio_distributed/templates/configmap.yml.j2 index 06dde3ab7..95d15b4f7 100644 --- a/roles/fio_distributed/templates/configmap.yml.j2 +++ b/roles/fio_distributed/templates/configmap.yml.j2 @@ -10,7 +10,11 @@ data: {% if workload_args.prefill is defined and workload_args.prefill is sameas true %} fiojob-prefill: | [global] +{% if workload_args.pvcvolumemode is defined and workload_args.pvcvolumemode == "Block" %} + filename={{fio_path}} +{% else %} directory={{fio_path}} +{% endif %} filename_format=f.\$jobnum.\$filenum clocksource=clock_gettime kb_base=1000 @@ -46,7 +50,11 @@ data: {% for job in workload_args.jobs %} fiojob-{{job}}-{{i}}-{{numjobs}}: | [global] +{% if workload_args.pvcvolumemode is defined and workload_args.pvcvolumemode == "Block" %} + filename={{fio_path}} +{% else %} directory={{fio_path}} +{% endif %} filename_format=f.\$jobnum.\$filenum write_bw_log=fio write_iops_log=fio diff --git a/roles/fio_distributed/templates/servers.yaml b/roles/fio_distributed/templates/servers.yaml index e1b0a4f09..3afc7f567 100644 --- a/roles/fio_distributed/templates/servers.yaml +++ b/roles/fio_distributed/templates/servers.yaml @@ -46,9 +46,17 @@ spec: args: - "cd /tmp; fio --server" {% if workload_args.storageclass is defined or hostpath is defined %} +{% if workload_args.pvcvolumemode is defined and workload_args.pvcvolumemode == "Block" %} + volumeDevices: +{% else %} volumeMounts: +{% endif %} - name: data-volume - mountPath: "{{ fio_path }}" +{% if workload_args.pvcvolumemode is defined and workload_args.pvcvolumemode == "Block" %} + devicePath: "{{ fio_path }}" +{% else %} + mountPath: "{{ fio_path }}" +{% endif %} {% endif %} restartPolicy: Never {% if workload_args.nodeselector is defined %} From 095443730bd9a5e5fb0bd128fe18119a7cc8dfda Mon Sep 17 00:00:00 2001 From: ebattat Date: Tue, 13 Dec 2022 10:00:45 +0200 Subject: [PATCH 234/249] disable autovacuum --- roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 index d2a0d7b32..2065841d6 100644 --- a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 +++ b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 @@ -58,7 +58,7 @@ echo host all all all trust >> /var/lib/pg echo max_connections = 100 > /var/lib/pgsql/data/postgresql.conf echo max_prepared_transactions = 0 >> /var/lib/pgsql/data/postgresql.conf echo shared_buffers = 4096MB >> /var/lib/pgsql/data/postgresql.conf -echo effective_cache_size = 8192MB >> /var/lib/pgsql/data/postgresql.conf +echo autovacuum = off >> /var/lib/pgsql/data/postgresql.conf sudo systemctl start postgresql; echo "alter role postgres password 'postgres'" > input psql -U postgres -d postgres < input From e32b4c4c3e614351180e2fe8d3d42183c4d7bc5a Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Mon, 16 Jan 2023 14:25:54 -0500 Subject: [PATCH 235/249] stress-ng: expose cpu method in cr (#804) Signed-off-by: Huamin Chen Signed-off-by: Huamin Chen --- config/samples/stressng/cr.yaml | 1 + roles/stressng/templates/jobfile.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/config/samples/stressng/cr.yaml b/config/samples/stressng/cr.yaml index f96327eb9..452da28de 100644 --- a/config/samples/stressng/cr.yaml +++ b/config/samples/stressng/cr.yaml @@ -25,6 +25,7 @@ spec: # cpu stressor options cpu_stressors: "1" cpu_percentage: "100" + cpu_method: "all" # vm stressor option vm_stressors: "1" vm_bytes: "128M" diff --git a/roles/stressng/templates/jobfile.j2 b/roles/stressng/templates/jobfile.j2 index a296f322c..42dd72019 100644 --- a/roles/stressng/templates/jobfile.j2 +++ b/roles/stressng/templates/jobfile.j2 @@ -7,6 +7,7 @@ timeout {{ workload_args.timeout}} # stop each stressor after 60 seconds {% if workload_args.cpu_stressors is defined %} cpu {{ workload_args.cpu_stressors }} # cpu stressor cpu-load {{ workload_args.cpu_percentage }} # percentage to which the cpu gets loaded +cpu-method {{ workload_args.cpu_method }} # cpu test method {% endif %} # vm stressor From a6c0040f34d2ec66811b188c3d0e620b86a72b20 Mon Sep 17 00:00:00 2001 From: Keith Valin Date: Thu, 19 Jan 2023 08:30:26 -0500 Subject: [PATCH 236/249] Added FIO prefill state to the benchmark. (#802) Have not tested using `kind: vm` --- roles/fio_distributed/tasks/main.yml | 39 ++++++++++++- roles/fio_distributed/templates/client.yaml | 10 ---- .../templates/prefill-client.yaml.j2 | 58 +++++++++++++++++++ .../templates/prefill-configmap.yml.j2 | 42 ++++++++++++++ 4 files changed, 137 insertions(+), 12 deletions(-) create mode 100644 roles/fio_distributed/templates/prefill-client.yaml.j2 create mode 100644 roles/fio_distributed/templates/prefill-configmap.yml.j2 diff --git a/roles/fio_distributed/tasks/main.yml b/roles/fio_distributed/tasks/main.yml index 60ecf5378..36dad2bf1 100644 --- a/roles/fio_distributed/tasks/main.yml +++ b/roles/fio_distributed/tasks/main.yml @@ -13,6 +13,10 @@ - name: Generate fio test k8s: definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" + + - name: Generate fio prefill job + k8s: + definition: "{{ lookup('template', 'prefill-configmap.yml.j2') | from_yaml }}" - name: Create PVC(s) k8s: @@ -132,7 +136,39 @@ port: 8765 when: (workload.args.kind | default('pod')) == "vm" with_items: "[{% for ip in pod_details.keys() %}'{{ ip }}', {% endfor %}]" + when: benchmark_state.resources[0].status.state == "StartingClient" + +- block: + - name: Run Prefill Job + k8s: + definition: "{{ lookup('template', 'prefill-client.yaml.j2') | from_yaml }}" + register: fio_prefill_pod + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Prefilling + when: workload.args.prefill and benchmark_state.resources[0].status.state == "StartingClient" + +- block: + - name: wait for db creation job to finish + k8s_facts: + kind: Job + api_version: batch/v1 + name: 'fio-prefill-{{ trunc_uuid }}' + namespace: "{{ operator_namespace }}" + register: fio_prefill_pod + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: StartBenchmark + when: fio_prefill_pod | json_query('resources[].status.succeeded') + when: workload.args.prefill and benchmark_state.resources[0].status.state == "Prefilling" + +- block: - name: Start FIO Client k8s: definition: "{{ lookup('template', 'client.yaml') | from_yaml }}" @@ -142,8 +178,7 @@ tasks_from: set_state vars: state: Running - - when: benchmark_state.resources[0].status.state == "StartingClient" + when: (benchmark_state.resources[0].status.state == "StartBenchmark" and workload.args.prefill) or (benchmark_state.resources[0].status.state == "StartingClient" and not workload.args.prefill) - include_role: name: benchmark_state diff --git a/roles/fio_distributed/templates/client.yaml b/roles/fio_distributed/templates/client.yaml index 965c6f555..51df759a2 100644 --- a/roles/fio_distributed/templates/client.yaml +++ b/roles/fio_distributed/templates/client.yaml @@ -71,16 +71,6 @@ spec: command: ["/bin/sh", "-c"] args: - "cat /tmp/host/hosts; -{% if workload_args.prefill is defined and workload_args.prefill is sameas true %} - echo ***************Prefill*****************; - cat /tmp/fio/fiojob-prefill; - mkdir -p /tmp/fiod-{{ uuid }}/fiojob-prefill; - fio --client=/tmp/host/hosts /tmp/fio/fiojob-prefill --output-format=json; -{% if workload_args.post_prefill_sleep is defined %} - sleep {{ workload_args.post_prefill_sleep }}; -{% endif %} - echo ***********End of Prefill*************; -{% endif %} {% if workload_args.bsrange is defined %} {% set loopvar = workload_args.bsrange %} {% elif workload_args.bs is defined %} diff --git a/roles/fio_distributed/templates/prefill-client.yaml.j2 b/roles/fio_distributed/templates/prefill-client.yaml.j2 new file mode 100644 index 000000000..d33af6363 --- /dev/null +++ b/roles/fio_distributed/templates/prefill-client.yaml.j2 @@ -0,0 +1,58 @@ +--- +kind: Job +apiVersion: batch/v1 +metadata: + name: 'fio-prefill-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' +spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} + template: + metadata: + labels: + benchmark-uuid: prefill-{{ uuid }} + app: fiod-prefill-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} + containers: + - name: fio-client + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/fio:latest') }} + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + args: + - "cat /tmp/host/hosts; + echo ***************Prefill*****************; + cat /tmp/fio/fiojob-prefill; + mkdir -p /tmp/fiod-{{ uuid }}/fiojob-prefill; + fio --client=/tmp/host/hosts /tmp/fio/fiojob-prefill --output-format=json; +{% if workload_args.post_prefill_sleep is defined %} + sleep {{ workload_args.post_prefill_sleep }}; +{% endif %} + echo ***********End of Prefill*************" + volumeMounts: + - name: fio-volume + mountPath: "/tmp/fio" + - name: host-volume + mountPath: "/tmp/host" + volumes: + - name: fio-volume + configMap: + name: "fio-prefill-{{ trunc_uuid }}" + defaultMode: 0777 + - name: host-volume + configMap: + name: "fio-hosts-{{ trunc_uuid }}" + defaultMode: 0777 + restartPolicy: Never +{% include "metadata.yml.j2" %} diff --git a/roles/fio_distributed/templates/prefill-configmap.yml.j2 b/roles/fio_distributed/templates/prefill-configmap.yml.j2 new file mode 100644 index 000000000..bd368cb54 --- /dev/null +++ b/roles/fio_distributed/templates/prefill-configmap.yml.j2 @@ -0,0 +1,42 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fio-prefill-{{ trunc_uuid }} + namespace: '{{ operator_namespace }}' +data: +# FIXME: I don't think prefill works correctly for a list of numjobs values, only for 1 of them +{% for numjobs in workload_args.numjobs %} +{% if workload_args.prefill is defined and workload_args.prefill is sameas true %} + fiojob-prefill: | + [global] +{% if workload_args.pvcvolumemode is defined and workload_args.pvcvolumemode == "Block" %} + filename={{fio_path}} +{% else %} + directory={{fio_path}} +{% endif %} + filename_format=f.\$jobnum.\$filenum + clocksource=clock_gettime + kb_base=1000 + unit_base=8 + ioengine=libaio + size={{workload_args.filesize}} +{% if workload_args.prefill_bs is defined %} + bs={{workload_args.prefill_bs}} +{% else %} + bs=4096KiB +{% endif %} + iodepth=1 + direct=1 + numjobs={{numjobs}} + + [write] + rw=write + create_on_open=1 + fsync_on_close=1 +{% if workload_args.cmp_ratio is defined %} + buffer_compress_percentage={{ workload_args.cmp_ratio }} + buffer_pattern=0xdeadface +{% endif %} +{% endif %} +{% endfor %} From 93725fc053de60682f4ab3806bf4e35436d35e11 Mon Sep 17 00:00:00 2001 From: Keith Valin Date: Fri, 27 Jan 2023 06:10:41 -0500 Subject: [PATCH 237/249] Added storageclass support to sysbench, if storageclass is not defined, previous behavior is default (#799) Added support for prepare, run, and cleanup phase options in sysbench fileio --- config/samples/sysbench/cr.yaml | 2 ++ docs/sysbench.md | 15 ++++++++++ roles/sysbench/tasks/main.yml | 18 ++++++++++++ roles/sysbench/templates/sysbench.sh.j2 | 37 +++++++++++++++++++++++-- roles/sysbench/templates/workload.yml | 5 ++++ 5 files changed, 74 insertions(+), 3 deletions(-) diff --git a/config/samples/sysbench/cr.yaml b/config/samples/sysbench/cr.yaml index 5674ab6f1..f52d90dfb 100644 --- a/config/samples/sysbench/cr.yaml +++ b/config/samples/sysbench/cr.yaml @@ -11,6 +11,8 @@ spec: #kind: vm # If you want to run this as a VM uncomment the above #pin_node: "worker-0.mylab.example.com" + #storageclass: ocs-storagecluster-ceph-rbd + #storagesize: 200Gi tests: - name: cpu parameters: diff --git a/docs/sysbench.md b/docs/sysbench.md index 43b1f37c3..91848a7b9 100644 --- a/docs/sysbench.md +++ b/docs/sysbench.md @@ -17,6 +17,15 @@ annotations to the pod metadata. The `pin_node` parameter allows to place the sysbench pod on a specific node, using the `hostname` label. +The `storageclass` and `storagesize` options can be set to have sysbench run on a particular StorageClass volume. +If `storageclass` is not provided, sysbench runs on an `emptyDir` volume by default. + +With the `fileio` test, you can define parameters for the `prepare`, `run` and `cleanup` steps +independently, using `prepare_parameters`, `run_parameters`, and `cleanup_parameters` respectively. +The `parameters` option is a global set of parameters each step inherits, but can be overriden. To +disable a value from `parameters` within `_parameters`, simply set the value to +`null`. + Note: please ensure you set 0 for other workloads if editing the [cr.yaml](../config/samples/sysbench/cr.yaml) file otherwise @@ -36,6 +45,8 @@ spec: #kind: vm # If you want to run this as a VM uncomment the above #pin_node: "worker-0.mylab.example.com" + #storageclass: ocs-storagecluster-ceph-rbd + #storagesize: 200Gi tests: - name: cpu parameters: @@ -43,6 +54,10 @@ spec: - name: fileio parameters: file-test-mode: rndrw + # This removes the file-test-mode parameter during the cleanup step + # Since run_parameters and prepare_parameters are not defined, they use file-test-mode: rndrw + #cleanup_parameters: + #file-test-mode: null ``` Name here refers to testname and can be cpu or fileio or memory etc and the parameters are the parametes for the particular test. diff --git a/roles/sysbench/tasks/main.yml b/roles/sysbench/tasks/main.yml index 559159eb2..5c594b790 100644 --- a/roles/sysbench/tasks/main.yml +++ b/roles/sysbench/tasks/main.yml @@ -10,6 +10,24 @@ namespace: '{{ operator_namespace }}' data: sysbenchScript: "{{ lookup('template', 'sysbench.sh.j2') }}" + - name: Create PVC + k8s: + definition: + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: "claim-sysbench-{{ trunc_uuid }}" + namespace: '{{ operator_namespace }}' + annotations: + volume.beta.kubernetes.io/storage-class: "{{ workload_args.storageclass }}" + spec: + accessModes: + - "{{ workload_args.pvcaccessmode | default('ReadWriteOnce') }}" + volumeMode: "{{ workload_args.pvcvolumemode | default('Filesystem') }}" + resources: + requests: + storage: "{{ workload_args.storagesize }}" + when: workload_args.storageclass is defined - name: Start sysbench job k8s: diff --git a/roles/sysbench/templates/sysbench.sh.j2 b/roles/sysbench/templates/sysbench.sh.j2 index 9aba92636..17f96af0a 100755 --- a/roles/sysbench/templates/sysbench.sh.j2 +++ b/roles/sysbench/templates/sysbench.sh.j2 @@ -1,4 +1,12 @@ #!/bin/bash +{% macro merge_params(global, local) -%} +{% for key, val in global.items() %} + {% if key not in local %} + {% set dummy = local.__setitem__(key, val) %} + {% endif %} +{% endfor %} +{%- endmacro -%} + {% for test in workload_args.tests %} {% if test.name == "cpu" or test.name == "memory" %} sysbench --test={{test.name}} {% for opt,value in test.parameters.items() %} --{{opt}}={{value}} {% endfor %} run @@ -6,11 +14,34 @@ sysbench --test={{test.name}} {% for opt,value in test.parameters.items() %} --{ {% if 'file-total-size' not in test.parameters %} {% set dummy = test['parameters'].__setitem__("file-total-size", "1G") %} {% endif %} -sysbench --test=fileio {% for opt,value in test.parameters.items() %} --{{opt}}={{value}} {% endfor %} prepare + +{% if 'prepare_parameters' in test %} +{% set dummy = merge_params(test.parameters, test.prepare_parameters) %} +{% set prep_params = test.prepare_parameters %} +{% else %} +{% set prep_params = test.parameters %} +{% endif %} +sysbench --test=fileio {% for opt,value in prep_params.items() %} {% if value is not none %} --{{opt}}={{value}} {% endif %} {% endfor %} prepare + {% if 'file-test-mode' not in test.parameters %} {% set dummy = test['parameters'].__setitem__("file-test-mode", "rndrw") %} {% endif %} -sysbench --test=fileio {% for opt,value in test.parameters.items() %} --{{opt}}={{value}} {% endfor %} run -sysbench --test=fileio --file-total-size={{test['parameters']['file-total-size']}} cleanup + +{% if 'run_parameters' in test %} +{% set dummy = merge_params(test.parameters, test.run_parameters) %} +{% set run_params = test.run_parameters %} +{% else %} +{% set run_params = test.parameters %} +{% endif %} +sysbench --test=fileio {% for opt,value in run_params.items() %} {% if value is not none %} --{{opt}}={{value}} {% endif %}{% endfor %} run + +{% if 'cleanup_parameters' in test %} +{% set dummy = merge_params(test.parameters, test.cleanup_parameters) %} +{% set clean_params = test.cleanup_parameters %} +{% else %} +{% set clean_params = test.parameters %} +{% endif %} +sysbench --test=fileio {% for opt, value in clean_params.items() %} {% if value is not none %} --{{opt}}={{value}} {% endif %}{% endfor %} cleanup + {% endif %} {% endfor %} diff --git a/roles/sysbench/templates/workload.yml b/roles/sysbench/templates/workload.yml index f7aa605d4..cc5443bcb 100644 --- a/roles/sysbench/templates/workload.yml +++ b/roles/sysbench/templates/workload.yml @@ -40,7 +40,12 @@ spec: mountPath: "/opt/sysbench" volumes: - name: sysbench-runtime +{% if workload_args.storageclass is defined %} + persistentVolumeClaim: + claimName: claim-sysbench-{{ trunc_uuid }} +{% else %} emptyDir: {} +{% endif %} - name: sysbench-volume configMap: name: "sysbench-config-{{ trunc_uuid }}" From 5938fa9a2b786c5f6738c004759197f6a1956592 Mon Sep 17 00:00:00 2001 From: Vishnu Challa Date: Fri, 27 Jan 2023 10:07:47 -0500 Subject: [PATCH 238/249] Initial draft for nighthawk workload (#796) * code snippets for creating nginx server side resources * corrected typo in the workload name * initial draft for nighthawk workload * updated review comments (#1) * added default values for all workload_args * restored files to avoid resolve conflicts * adding readme and keeping default service as ClusterIP * made changes to keep clusterIP default --------- Co-authored-by: root Co-authored-by: Murali Krishnasamy <70236227+mukrishn@users.noreply.github.com> Co-authored-by: Vishnu Challa --- roles/nighthawk/README.md | 136 +++++++++++ roles/nighthawk/defaults/main.yml | 3 + roles/nighthawk/handlers/main.yml | 2 + roles/nighthawk/meta/main.yml | 53 ++++ roles/nighthawk/tasks/edge-tasks.yml | 21 ++ roles/nighthawk/tasks/http-tasks.yml | 21 ++ roles/nighthawk/tasks/main.yml | 228 ++++++++++++++++++ roles/nighthawk/tasks/passthrough-tasks.yml | 21 ++ roles/nighthawk/tasks/reencrypt-tasks.yml | 21 ++ roles/nighthawk/templates/edge-route.yml.j2 | 15 ++ roles/nighthawk/templates/http-route.yml.j2 | 13 + roles/nighthawk/templates/http-service.yml.j2 | 16 ++ .../nighthawk/templates/https-service.yml.j2 | 16 ++ roles/nighthawk/templates/nginx-server.yml.j2 | 58 +++++ .../templates/passthrough-route.yml.j2 | 15 ++ .../templates/reencrypt-route.yml.j2 | 66 +++++ roles/nighthawk/templates/workload.yml.j2 | 129 ++++++++++ roles/nighthawk/vars/main.yml | 3 + 18 files changed, 837 insertions(+) create mode 100644 roles/nighthawk/README.md create mode 100644 roles/nighthawk/defaults/main.yml create mode 100644 roles/nighthawk/handlers/main.yml create mode 100644 roles/nighthawk/meta/main.yml create mode 100644 roles/nighthawk/tasks/edge-tasks.yml create mode 100644 roles/nighthawk/tasks/http-tasks.yml create mode 100644 roles/nighthawk/tasks/main.yml create mode 100644 roles/nighthawk/tasks/passthrough-tasks.yml create mode 100644 roles/nighthawk/tasks/reencrypt-tasks.yml create mode 100644 roles/nighthawk/templates/edge-route.yml.j2 create mode 100644 roles/nighthawk/templates/http-route.yml.j2 create mode 100644 roles/nighthawk/templates/http-service.yml.j2 create mode 100644 roles/nighthawk/templates/https-service.yml.j2 create mode 100644 roles/nighthawk/templates/nginx-server.yml.j2 create mode 100644 roles/nighthawk/templates/passthrough-route.yml.j2 create mode 100644 roles/nighthawk/templates/reencrypt-route.yml.j2 create mode 100644 roles/nighthawk/templates/workload.yml.j2 create mode 100644 roles/nighthawk/vars/main.yml diff --git a/roles/nighthawk/README.md b/roles/nighthawk/README.md new file mode 100644 index 000000000..695e5bb5b --- /dev/null +++ b/roles/nighthawk/README.md @@ -0,0 +1,136 @@ +nighthawk-benchmark +========= + +HTTP benchmark workload. More details [here](https://github.com/envoyproxy/nighthawk). + +## Running Nighthawk + +Here are some sample CRs and instructions to run this workload. + +### Sample CR for testing +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: nighthawk-benchmark + namespace: benchmark-operator +spec: + clustername: myk8scluster + elasticsearch: + url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443 + workload: + cleanup: true + name: nighthawk + args: + url: https://www.testurlhere.com + image: quay.io/vchalla/nighthawk:latest + terminations: ["http", "edge", "passthrough", "reencrypt"] + kind: pod + hostnetwork: false + number_of_routes: 3 + samples: 1 + concurrency: 8 + duration: 60 + connections: 80 + max_requests_per_connection: 50 + debug: true +``` +### Sample CR with default parameters +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: nighthawk-benchmark + namespace: benchmark-operator +spec: + clustername: myk8scluster + elasticsearch: + url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443 + workload: + cleanup: true + name: nighthawk + args: + terminations: ["http"] +``` + +### Parameters +`url` Target url to test. Works as a client server model with client-server pod pairs. +`image`: Image if needed to be overriden from the default one. +`terminations`: List of network terminations to be applied. +`kind`: Kind of machine to run workload on. For now only pod is supported. +`hostnetwork`: Will test the performance of the node the pod will run on. +`number_of_routes`: Number of routes to be created. +`samples`: Number of times to run the workload. +`concurrency`: The number of concurrent event loops that should be used. Specify 'auto' to "\ + "let Nighthawk leverage all vCPUs that have affinity to the Nighthawk process"\ + ". Note that increasing this results in an effective load multiplier combined"\ + " with the configured --rps and --connections values" +`duration`: The number of seconds that the test should be running. +`connections`: The maximum allowed number of concurrent connections per event loop. +`max_requests_per_connection`: Max requests per connection. +`debug`: To turn on debug logs. + +### Once the testing is done the results are published to elastic search as follows. +```yaml +"hits" : [ + { + "_index" : "ripsaw-nighthawk-results", + "_type" : "_doc", + "_id" : "4a2a5aa5c41a3ec53bdc5de9f9ea4a04f12cc8fa967fa2033b95f82e2ae356f0", + "_score" : 1.0, + "_source" : { + "concurrency" : 8, + "duration" : 60, + "connections" : 80, + "max_requests_per_connection" : 4294937295, + "rps" : 5, + "kind" : "pod", + "url" : "http://nighthawk-benchmark-route-http-2-benchmark-operator.apps.vchalla-perfscale.perfscale.devcluster.openshift.com", + "workload" : "nighthawk", + "uuid" : "e777d5e4-4b9e-5e90-a453-61e30caa9a5b", + "user" : "ripsaw", + "cluster_name" : "myk8scluster", + "targets" : [ + "http://nighthawk-benchmark-route-http-2-benchmark-operator.apps.vchalla-perfscale.perfscale.devcluster.openshift.com" + ], + "hostname" : "nighthawk-client-10.0.221.20-nginx-http-2-e777d5e4-pmxvt", + "requested_qps" : 40, + "throughput" : 39.98936405750067, + "status_codes_1xx" : 0, + "status_codes_2xx" : 2400, + "status_codes_3xx" : 0, + "status_codes_4xx" : 0, + "status_codes_5xx" : 0, + "p50_latency" : 3.188863, + "p75_latency" : 4.302335, + "p80_latency" : 5.629951, + "p90_latency" : 10.191871, + "p95_latency" : 13.958143, + "p99_latency" : 24.370175, + "p99_9_latency" : 39.522303, + "avg_latency" : 4.663531, + "timestamp" : "2022-08-18T19:57:57.357909411Z", + "bytes_in" : 1108800.0, + "bytes_out" : 331200.0, + "iteration" : 1, + "run_id" : "NA" + } + } + ] +``` +### And also you can verify the logs in your openshift namespace using the below script +``` +echo -n "`oc get all | grep -i -e 'completed' | cut -f 1 -d ' '`" > output.txt + +cat output.txt | awk '{print $0}' | while + read each +do + echo "client name: $each" + echo "client info" + echo -n "`oc get $each -o yaml | grep -i -e "clientfor" -e 'port' -A 2`" + echo -e "\n" + echo "client logs" + echo -n "`oc logs $each | tail -n 5`" + echo -e "\n\n" +done +``` \ No newline at end of file diff --git a/roles/nighthawk/defaults/main.yml b/roles/nighthawk/defaults/main.yml new file mode 100644 index 000000000..1ed4613ee --- /dev/null +++ b/roles/nighthawk/defaults/main.yml @@ -0,0 +1,3 @@ +--- +# defaults file for nighthawk +resource_kind: "{{ workload.args.kind | default('pod') }}" \ No newline at end of file diff --git a/roles/nighthawk/handlers/main.yml b/roles/nighthawk/handlers/main.yml new file mode 100644 index 000000000..5780811e8 --- /dev/null +++ b/roles/nighthawk/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for nighthawk \ No newline at end of file diff --git a/roles/nighthawk/meta/main.yml b/roles/nighthawk/meta/main.yml new file mode 100644 index 000000000..227ad9c34 --- /dev/null +++ b/roles/nighthawk/meta/main.yml @@ -0,0 +1,53 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.9 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. + \ No newline at end of file diff --git a/roles/nighthawk/tasks/edge-tasks.yml b/roles/nighthawk/tasks/edge-tasks.yml new file mode 100644 index 000000000..a9fd8027f --- /dev/null +++ b/roles/nighthawk/tasks/edge-tasks.yml @@ -0,0 +1,21 @@ +--- +# tasks for edge termination type. + +- name: Create service for edge type server pods + k8s: + definition: "{{ lookup('template', 'http-service.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + vars: + type: edge + +- name: Create route for edge type server pods + k8s: + definition: "{{ lookup('template', 'edge-route.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + +- name: Start edge server types(s) + k8s: + definition: "{{ lookup('template', 'nginx-server.yml.j2') | from_yaml }}" + vars: + type: edge + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} \ No newline at end of file diff --git a/roles/nighthawk/tasks/http-tasks.yml b/roles/nighthawk/tasks/http-tasks.yml new file mode 100644 index 000000000..e60ea8d69 --- /dev/null +++ b/roles/nighthawk/tasks/http-tasks.yml @@ -0,0 +1,21 @@ +--- +# tasks for http termination type. + +- name: Create service for http type server pods + k8s: + definition: "{{ lookup('template', 'http-service.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + vars: + type: http + +- name: Create route for http type server pods + k8s: + definition: "{{ lookup('template', 'http-route.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + +- name: Start http server types(s) + k8s: + definition: "{{ lookup('template', 'nginx-server.yml.j2') | from_yaml }}" + vars: + type: http + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} \ No newline at end of file diff --git a/roles/nighthawk/tasks/main.yml b/roles/nighthawk/tasks/main.yml new file mode 100644 index 000000000..040bb1d83 --- /dev/null +++ b/roles/nighthawk/tasks/main.yml @@ -0,0 +1,228 @@ +--- +# tasks file for nighthawk + +- block: + + - include_tasks: http-tasks.yml + when: "'http' in {{ workload_args.terminations | default([]) }} or 'mix' in {{ workload_args.terminations | default([]) }}" + + - include_tasks: edge-tasks.yml + when: "'edge' in {{ workload_args.terminations | default([]) }} or 'mix' in {{ workload_args.terminations | default([]) }}" + + - include_tasks: passthrough-tasks.yml + when: "'passthrough' in {{ workload_args.terminations | default([]) }} or 'mix' in {{ workload_args.terminations | default([]) }}" + + - include_tasks: reencrypt-tasks.yml + when: "'reencrypt' in {{ workload_args.terminations | default([]) }} or 'mix' in {{ workload_args.terminations | default([]) }}" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting Servers + + when: benchmark_state.resources[0].status.state == "Building" and resource_kind == "pod" + +- block: + + - name: Get server pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-nighthawk-server-{{ trunc_uuid }} + register: server_pods + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Clients" + when: "workload_args.terminations|default([])|length * workload_args.number_of_routes|default(1)|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + + when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" + +- block: + + - name: Get server pod info + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-nighthawk-server-{{ trunc_uuid }} + register: server_pods + + - name: Get route info + k8s_facts: + kind: Route + api_version: route.openshift.io/v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-nighthawk-route-{{ trunc_uuid }} + register: routes + + - block: + - name: Start Client(s) + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + with_items: "{{ server_pods.resources }}" + vars: + routerCanonicalName: "{{ routes.resources[0] | json_query('status.ingress[0].routerCanonicalHostname') }}" + when: ( server_pods.resources|length > 0 ) + + when: resource_kind == "pod" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Waiting for Clients" + + when: benchmark_state.resources[0].status.state == "Starting Clients" + +- block: + + - block: + - name: Get client pod status + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = nighthawk-bench-client-{{ trunc_uuid }} + register: client_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Clients Running + when: "workload_args.terminations|default([])|length * workload_args.number_of_routes|default(1)|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" + + when: resource_kind == "pod" + + when: benchmark_state.resources[0].status.state == "Waiting for Clients" + +- block: + + - name: Signal workload + command: "redis-cli set start-{{ trunc_uuid }} true" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running + + when: benchmark_state.resources[0].status.state == "Clients Running" + +- block: + - block: + - name: Waiting for pods to complete.... + k8s_facts: + kind: pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = nighthawk-bench-client-{{ trunc_uuid }} + register: client_pods + + - name: Check for client pod failures + include_role: + name: benchmark_state + tasks_from: failure + when: "(client_pods|json_query('resources[].status[]')|selectattr('phase','match','Failed')|list|length) > 0" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Cleanup + when: "workload_args.terminations|default([])|length * workload_args.number_of_routes|default(1)|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: resource_kind == "pod" + + when: benchmark_state.resources[0].status.state == "Running" + +- block: + + - block: + - name: Get Server Jobs + k8s_facts: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-nighthawk-job-{{ trunc_uuid }} + register: server_jobs + + - name: Get Server Pods + k8s_facts: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-nighthawk-server-{{ trunc_uuid }} + register: server_pods + + - name: Server Job and Pod names - to clean + set_fact: + clean_jobs: | + [ + {% for item in server_jobs.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + clean_pods: | + [ + {% for item in server_pods.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + + - name: Cleanup server Job + k8s: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_jobs }}" + + - name: Cleanup server Pod + k8s: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_pods }}" + + when: resource_kind == "pod" and cleanup == True + + - block: + - name: Cleanup redis + command: "{{ item }}" + with_items: + - redis-cli del start-{{ trunc_uuid }} + when: resource_kind == "pod" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + + when: benchmark_state.resources[0].status.state == "Cleanup" diff --git a/roles/nighthawk/tasks/passthrough-tasks.yml b/roles/nighthawk/tasks/passthrough-tasks.yml new file mode 100644 index 000000000..0cf6cfc8b --- /dev/null +++ b/roles/nighthawk/tasks/passthrough-tasks.yml @@ -0,0 +1,21 @@ +--- +# tasks for passthrough termination type. + +- name: Create service for passthrough type server pods + k8s: + definition: "{{ lookup('template', 'https-service.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + vars: + type: passthrough + +- name: Create route for passthrough type server pods + k8s: + definition: "{{ lookup('template', 'passthrough-route.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + +- name: Start passthrough server types(s) + k8s: + definition: "{{ lookup('template', 'nginx-server.yml.j2') | from_yaml }}" + vars: + type: passthrough + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} \ No newline at end of file diff --git a/roles/nighthawk/tasks/reencrypt-tasks.yml b/roles/nighthawk/tasks/reencrypt-tasks.yml new file mode 100644 index 000000000..44839f6e2 --- /dev/null +++ b/roles/nighthawk/tasks/reencrypt-tasks.yml @@ -0,0 +1,21 @@ +--- +# tasks for reencrypt termination type. + +- name: Create service for reencrypt type server pods + k8s: + definition: "{{ lookup('template', 'https-service.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + vars: + type: reencrypt + +- name: Create route for reencrypt type server pods + k8s: + definition: "{{ lookup('template', 'reencrypt-route.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + +- name: Start reencrypt server types(s) + k8s: + definition: "{{ lookup('template', 'nginx-server.yml.j2') | from_yaml }}" + vars: + type: reencrypt + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} \ No newline at end of file diff --git a/roles/nighthawk/templates/edge-route.yml.j2 b/roles/nighthawk/templates/edge-route.yml.j2 new file mode 100644 index 000000000..b5e0fc873 --- /dev/null +++ b/roles/nighthawk/templates/edge-route.yml.j2 @@ -0,0 +1,15 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + type: {{ ansible_operator_meta.name }}-bench-nighthawk-route-{{ trunc_uuid }} + name: nighthawk-benchmark-route-edge-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + port: + targetPort: http + tls: + termination: edge + to: + kind: Service + name: nighthawk-benchmark-service-edge-{{ item }} \ No newline at end of file diff --git a/roles/nighthawk/templates/http-route.yml.j2 b/roles/nighthawk/templates/http-route.yml.j2 new file mode 100644 index 000000000..4a5a390fd --- /dev/null +++ b/roles/nighthawk/templates/http-route.yml.j2 @@ -0,0 +1,13 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + type: {{ ansible_operator_meta.name }}-bench-nighthawk-route-{{ trunc_uuid }} + name: nighthawk-benchmark-route-http-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + port: + targetPort: http + to: + kind: Service + name: nighthawk-benchmark-service-http-{{ item }} \ No newline at end of file diff --git a/roles/nighthawk/templates/http-service.yml.j2 b/roles/nighthawk/templates/http-service.yml.j2 new file mode 100644 index 000000000..6a9b0320c --- /dev/null +++ b/roles/nighthawk/templates/http-service.yml.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: nighthawk-benchmark + name: nighthawk-benchmark-service-{{ type }}-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: nginx-{{ type }}-{{ item }} + type: ClusterIP \ No newline at end of file diff --git a/roles/nighthawk/templates/https-service.yml.j2 b/roles/nighthawk/templates/https-service.yml.j2 new file mode 100644 index 000000000..fed4d1c25 --- /dev/null +++ b/roles/nighthawk/templates/https-service.yml.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: nighthawk-benchmark + name: nighthawk-benchmark-service-{{ type }}-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + app: nginx-{{ type }}-{{ item }} + type: ClusterIP \ No newline at end of file diff --git a/roles/nighthawk/templates/nginx-server.yml.j2 b/roles/nighthawk/templates/nginx-server.yml.j2 new file mode 100644 index 000000000..21fc323a2 --- /dev/null +++ b/roles/nighthawk/templates/nginx-server.yml.j2 @@ -0,0 +1,58 @@ +--- +kind: Job +apiVersion: batch/v1 +metadata: + name: nighthawk-benchmark-server-{{ type }}-{{ item }} + namespace: '{{ operator_namespace }}' + labels: + run: nighthawk-benchmark-{{ type }}-{{ item }} + app: nginx + type: {{ ansible_operator_meta.name }}-bench-nighthawk-job-{{ trunc_uuid }} +spec: + ttlSecondsAfterFinished: 600 + backoffLimit: 0 + template: + metadata: + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: nighthawk + benchmark-operator-role: server + app: nginx-{{ type }}-{{ item }} + termination: {{ type }} + route: nighthawk-benchmark-route-{{ type }}-{{ item }} + type: {{ ansible_operator_meta.name }}-bench-nighthawk-server-{{ trunc_uuid }} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.hostnetwork is sameas true %} + hostNetwork: true + serviceAccountName: benchmark-operator +{% endif %} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: type + operator: In + values: + - {{ ansible_operator_meta.name }}-bench-nighthawk-server-{{ trunc_uuid }} + topologyKey: kubernetes.io/hostname + containers: + - name: nginx + image: quay.io/cloud-bulldozer/nginx:latest + imagePullPolicy: Always + ports: + - containerPort: 8080 + resources: + requests: + memory: "10Mi" + cpu: "10m" + dnsPolicy: ClusterFirst + restartPolicy: OnFailure +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_server | default("unknown") }}' +{% endif %} +{% include "metadata.yml.j2" %} \ No newline at end of file diff --git a/roles/nighthawk/templates/passthrough-route.yml.j2 b/roles/nighthawk/templates/passthrough-route.yml.j2 new file mode 100644 index 000000000..1a4804cd6 --- /dev/null +++ b/roles/nighthawk/templates/passthrough-route.yml.j2 @@ -0,0 +1,15 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + type: {{ ansible_operator_meta.name }}-bench-nighthawk-route-{{ trunc_uuid }} + name: nighthawk-benchmark-route-passthrough-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + port: + targetPort: https + tls: + termination: passthrough + to: + kind: Service + name: nighthawk-benchmark-service-passthrough-{{ item }} \ No newline at end of file diff --git a/roles/nighthawk/templates/reencrypt-route.yml.j2 b/roles/nighthawk/templates/reencrypt-route.yml.j2 new file mode 100644 index 000000000..78d1982fb --- /dev/null +++ b/roles/nighthawk/templates/reencrypt-route.yml.j2 @@ -0,0 +1,66 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + type: {{ ansible_operator_meta.name }}-bench-nighthawk-route-{{ trunc_uuid }} + name: nighthawk-benchmark-route-reencrypt-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + port: + targetPort: https + tls: + termination: reencrypt + certificate: | + -----BEGIN CERTIFICATE----- + MIIDezCCAmOgAwIBAgIUIDE86s4g/jNL6iHVCeMKzsLL/rwwDQYJKoZIhvcNAQEL + BQAwTTELMAkGA1UEBhMCRVMxDzANBgNVBAgMBk1hZHJpZDEPMA0GA1UEBwwGTWFk + cmlkMRwwGgYDVQQKDBNEZWZhdWx0IENvbXBhbnkgTHRkMB4XDTIxMDEyMDExNDAw + NFoXDTIxMDIxOTExNDAwNFowTTELMAkGA1UEBhMCRVMxDzANBgNVBAgMBk1hZHJp + ZDEPMA0GA1UEBwwGTWFkcmlkMRwwGgYDVQQKDBNEZWZhdWx0IENvbXBhbnkgTHRk + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvb+XB/1moWr0MJZoSRN/ + MIvXyPM0pR0N8Gxfpk9LF0qHLX3I35BkugWZpua85HXmurbZgW12UpplTuA/wack + YjVsTjWNb/p4cQF/o1MIm5iWWCrZY6+Du52gYb36lHNU3Uv2flsNZRHulMq9Ptov + I01mtYpr/PujU5r+QN01kiswrC4Z7xip3VTRtn4L8VYXIDapNLw4YinNJXIUd+tS + YsEYG343E9dPXFbobpHi/6JNjrcs6oe4VKvM8pgeAPTM9dbLJtX/TFLAkPLoeIoT + R/TmP/y7EbZXdAQL2schIL4Nx0iHaLoLPF9mGL/1D/9skAHr3RRC8eAmKKVoZVee + qwIDAQABo1MwUTAdBgNVHQ4EFgQU3gxPTXzT++e2GtTEnkHNXWTw2KMwHwYDVR0j + BBgwFoAU3gxPTXzT++e2GtTEnkHNXWTw2KMwDwYDVR0TAQH/BAUwAwEB/zANBgkq + hkiG9w0BAQsFAAOCAQEAf+cvaTY02vA1BwW8K8kbWS//F+GFgJhFZ7fi2szVZjBk + JErJcCr8+lwY4GVytPglhO8zWDfMY+c4LKFchPN1LnVYb7v18xbNb7Smz02SQQDl + sXgTf/f+Fc+lwVRjtVNegfwqc5wZj2ZqBcXq0UnxIBBzXS9EL6czeOqW4gPy5bPa + KejQwkkULk6KqKGT2tp71BsMyXCM8fMQkvM6FxHPKNUR/GxxQoH4mTQlOgnqrdAc + 4GUhmQZc3IQyUHTUoxMw4BodsLJBH3kCqQ5dy/O3h3GTNzivZNRpItVI6HHjrN2s + cZm6iaKNoUXr3bv3ugZwe+1R4ISvw/FXS1pMxiD0YA== + -----END CERTIFICATE----- + key: | + -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEAvb+XB/1moWr0MJZoSRN/MIvXyPM0pR0N8Gxfpk9LF0qHLX3I + 35BkugWZpua85HXmurbZgW12UpplTuA/wackYjVsTjWNb/p4cQF/o1MIm5iWWCrZ + Y6+Du52gYb36lHNU3Uv2flsNZRHulMq9PtovI01mtYpr/PujU5r+QN01kiswrC4Z + 7xip3VTRtn4L8VYXIDapNLw4YinNJXIUd+tSYsEYG343E9dPXFbobpHi/6JNjrcs + 6oe4VKvM8pgeAPTM9dbLJtX/TFLAkPLoeIoTR/TmP/y7EbZXdAQL2schIL4Nx0iH + aLoLPF9mGL/1D/9skAHr3RRC8eAmKKVoZVeeqwIDAQABAoIBAGSKrlZ3ePgzGezc + 5alDAXQRxWcfJ1gOCyLH6e7PuTRAM1xxeAyuEBFZgk8jmBdeOcHZvWqNO9MNKH0g + 6eeMzwSS1i6ixaz+BO+sIZvDFZ6Mva0+Fy5xA9ZX8XGZHrumWONhqtzNFk3lsIt6 + 2cgCCFQmYTP0gr/r/mEAkZSBIi+udRIEIsR7T4iLw+H+MWwnXEbM9yke9Qvvvzyt + ULLWzJT9TrmW7teAvtKYx7D0dyJEXnM8zEeuDd+unzeV2uqrfptU34fjvuQLaUKk + Br02YeOvs7rSSkGReZNnt8ZXWYFssYs3Kf8s/5EkXYFPukK+8LjII9L8zFgviBMf + bIqnIYECgYEA6Ns1gkkWtnBmfzSGrFkM5ztQvuLwgaMRLnzsY7JQBnTjWFXTRAqu + pBtTaxPGEg0P1A4fRnjiSjV1BGzS5yUdyZSYTbc9jiILQwdayB97chQAE+hnVokU + 1zPqtCfiMJETOnV8fRNMBaDRHoBoe1l6va7G37P9NzNgWkpAj+MNKssCgYEA0JuL + b9IEk0BhOBaV0ROsMJnErsDNbESY4kVotVssZYL1U61jDfC7KbQMmW3frPtGWJi8 + HW99ulXpGvRmwrOhRtPaXzSBwpF8KTseAaRJmPmtJAFYhYRfMFTZWE2JKNGOhkws + olO6FKkEfgR+m/HtgIqeOdQ/nI1q192SbAHff6ECgYEA4sqN5ST2gB4dVgt8l2Ps + E1JMJH63rCt8UoDNY5SKKJ+zxZdhusWErsUGjCWoJnCeV/ShNWwLSieinvq2tvYJ + ewnFBPxRcZtqyI/jNUKkYslkAf+6lifRKoCgOXMW9CJ4Tdmbs94Vju3AfyqlmG3g + A9q0S7DsENVzJL1pADst2d0CgYEAxRALOsj1FX2d2XRMdsPUx9ya1lLAO+TZX/cd + oSTN3d9GjZOfnU2qIQ07Ub1frXN50rwGCPCHnv0FRjdW09sJIXWENqfNZNY2qmR0 + RizCccZ67yZuT0LrASdGYopsZakAsJFJINdjU50O51Srnfl+2Q0Zx5tftC5Lnjxr + 06g5T8ECgYAIqSiZU3HluraETpV4OD3XtOTnuEkCA/cd1msPCWsszgvA8Ij0PYjY + Wl4wlnBAz0mrZbdbJXZ5SqXRiwA+M/bckrODU5ZFiT8Fk5mJlvROqW5rLFIpnJKZ + cKYMl5I6JC6SQHcbCQ743Yohay5TyGmLFCMnCktx/lHXjZTk8ITY7g== + -----END RSA PRIVATE KEY----- + destinationCACertificate: "-----BEGIN CERTIFICATE-----\nMIIDbTCCAlWgAwIBAgIJAJR/jN0Oa+/rMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV\nBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMQswCQYDVQQHDAJOWTEcMBoGA1UE\nCgwTRGVmYXVsdCBDb21wYW55IEx0ZDAeFw0xNzAxMjQwODExMDJaFw0yNzAxMjIw\nODExMDJaME0xCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMQswCQYD\nVQQHDAJOWTEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDCCASIwDQYJKoZI\nhvcNAQEBBQADggEPADCCAQoCggEBAMItGS9sSafyqBuOcQcQ5j7OQ0EwF9qOckhl\nfT8VzUbcOy8/L/w654MpLEa4O4Fiek3keE7SDWGVtGZWDvT9y1QUxPhkDWq1Y3rr\nyMelv1xRIyPVD7EEicga50flKe8CKd1U3D6iDQzq0uxZZ6I/VArXW/BZ4LfPauzN\n9EpCYyKq0fY7WRFIGouO9Wu800nxcHptzhLAgSpO97aaZ+V+jeM7n7fchRSNrpIR\nzPBl/lIBgCPJgkax0tcm4EIKIwlG+jXWc5mvV8sbT8rAv32HVuaP6NafyWXXP3H1\noBf2CQCcwuM0sM9ZeZ5JEDF/7x3eNtqSt1X9HjzVpQjiVBXY+E0CAwEAAaNQME4w\nHQYDVR0OBBYEFOXxMHAA1qaKWlP+gx8tKO2rQ81WMB8GA1UdIwQYMBaAFOXxMHAA\n1qaKWlP+gx8tKO2rQ81WMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEB\nAJAri7Pd0eSY/rvIIvAvjhDPvKt6gI5hJEUp+M3nWTWA/IhQFYutb9kkZGhbBeLj\nqneJa6XYKaCcUx6/N6Vvr3AFqVsbbubbejRpdpXldJC33QkwaWtTumudejxSon24\nW/ANN/3ILNJVMouspLRGkFfOYp3lq0oKAlNZ5G3YKsG0znAfqhAVtqCTG9RU24Or\nxzkEaCw8IY5N4wbjCS9FPLm7zpzdg/M3A/f/vrIoGdns62hzjzcp0QVTiWku74M8\nv7/XlUYYvXOvPQCCHgVjnAZlnjcxMTBbwtdwfxjAmdNTmFFpASnf0s3b287zQwVd\nIeSydalVtLm7rBRZ59/2DYo=\n-----END CERTIFICATE-----" + to: + kind: Service + name: nighthawk-benchmark-service-reencrypt-{{ item }} \ No newline at end of file diff --git a/roles/nighthawk/templates/workload.yml.j2 b/roles/nighthawk/templates/workload.yml.j2 new file mode 100644 index 000000000..e23289430 --- /dev/null +++ b/roles/nighthawk/templates/workload.yml.j2 @@ -0,0 +1,129 @@ +--- +kind: Job +apiVersion: batch/v1 +metadata: + name: 'nighthawk-client-{{item.status.hostIP}}-{{ item.metadata.labels.app }}-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' +spec: + template: + metadata: + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: nighthawk + benchmark-operator-role: client + app: nighthawk-bench-client-{{ trunc_uuid }} + clientfor: {{ item.metadata.labels.app }} + type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} +{% if workload_args.multus.enabled is sameas true %} + annotations: + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client | default("unknown") }} +{% endif %} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.hostnetwork is sameas true %} + hostNetwork: true + serviceAccountName: benchmark-operator +{% endif %} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: type + operator: In + values: + - {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} + topologyKey: kubernetes.io/hostname + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ item.metadata.labels.app }} + topologyKey: kubernetes.io/hostname + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/nighthawk:latest') }} + env: + - name: uuid + value: "{{ uuid }}" + - name: test_user + value: "{{ test_user | default("ripsaw") }}" + - name: clustername + value: "{{ clustername }}" +{% if elasticsearch.url %} + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("ripsaw-nighthawk") }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(true) }}" +{% endif %} +{% if prometheus is defined %} + - name: prom_es + value: "{{ prometheus.es_url }}" + - name: prom_parallel + value: "{{ prometheus.es_parallel | default(false) }}" + - name: prom_token + value: "{{ prometheus.prom_token | default() }}" + - name: prom_url + value: "{{ prometheus.prom_url | default() }}" +{% endif %} + - name: client_node + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: server_node + value: "{{ workload_args.pin_server|default("unknown") }}" +{% if workload_args.client_resources is defined %} + resources: {{ workload_args.client_resources | to_json }} +{% endif %} + imagePullPolicy: Always + workingDir: /tmp + command: ["/bin/bash", "-c"] + args: + - | +{% if item.metadata.labels.termination|default("http") == "http" %} + protocol="http"; +{% else %} + protocol="https"; +{% endif %} +{% if workload_args.url is defined %} + url={{ workload_args.url | default("unknown") }} +{% else %} + url=$protocol://{{ item.metadata.labels.route }}-{{ operator_namespace }}.{{ routerCanonicalName.split('.')[1:] | join('.') }} +{% endif %} + exit_status=0; + while true; do + if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{ trunc_uuid }}) =~ 'true' ]]; then + run_snafu -t nighthawk -s {{workload_args.samples|default(1)|int}} --resourcetype {{ resource_kind }} -u {{ uuid }} --user {{test_user | default("ripsaw")}} --url $url --concurrency {{ workload_args.concurrency|default(8)|int }} --duration {{ workload_args.duration|default(60)|int }} --connections {{ workload_args.connections|default(80)|int }} --max-requests-per-connection {{ workload_args.max_requests_per_connection|default(50)|int }} --create-archive --archive-file /tmp/nighthawk.archive \ +{% if workload_args.run_id is defined %} + --run-id {{workload_args.run_id}} \ +{% endif %} +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; + if [[ $? -ne 0 ]]; then + exit_status=1; + fi; + else + continue; + fi; + break; + done; + redis-cli -h {{bo.resources[0].status.podIP}} set start-{{ trunc_uuid }} false; + exit $exit_status; + restartPolicy: Never +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client | default("unknown") }}' +{% endif %} diff --git a/roles/nighthawk/vars/main.yml b/roles/nighthawk/vars/main.yml new file mode 100644 index 000000000..f52cf6d81 --- /dev/null +++ b/roles/nighthawk/vars/main.yml @@ -0,0 +1,3 @@ +--- +# vars file for nighthawk +cleanup: true \ No newline at end of file From 76ab6e6b6ab1e336b5fc47a87ed078fa2271cea9 Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy <70236227+mukrishn@users.noreply.github.com> Date: Fri, 31 Mar 2023 11:27:59 -0400 Subject: [PATCH 239/249] typo --- roles/log_generator/templates/log_generator.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/log_generator/templates/log_generator.yml b/roles/log_generator/templates/log_generator.yml index 76fd91cd4..3ea653678 100644 --- a/roles/log_generator/templates/log_generator.yml +++ b/roles/log_generator/templates/log_generator.yml @@ -103,7 +103,7 @@ spec: --messages-per-second "{{ workload_args.messages_per_second }}" {% endif %} {% if workload_args.messages_per_minute is defined %} - --messages-per-minue "{{ workload_args.messages_per_minute }}" + --messages-per-minute "{{ workload_args.messages_per_minute }}" {% endif %} --duration "{{ workload_args.duration | default(10) }}" {% if workload_args.es_url is defined %} From 977ad2661c27c35185740775ea2b9bb154cf2531 Mon Sep 17 00:00:00 2001 From: morenod Date: Tue, 4 Apr 2023 14:34:19 +0200 Subject: [PATCH 240/249] Add es_index parameter to log-generator workload template as it can be set on e2e-benchmarking as ES_BACKEND_INDEX --- roles/log_generator/templates/log_generator.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/log_generator/templates/log_generator.yml b/roles/log_generator/templates/log_generator.yml index 3ea653678..a3587fd7a 100644 --- a/roles/log_generator/templates/log_generator.yml +++ b/roles/log_generator/templates/log_generator.yml @@ -109,6 +109,9 @@ spec: {% if workload_args.es_url is defined %} --es-url "{{ workload_args.es_url }}" {% endif %} +{% if workload_args.es_index is defined %} + --es-index "{{ workload_args.es_index }}" +{% endif %} {% if workload_args.es_token is defined %} --es-token "{{ workload_args.es_token }}" {% endif %} From 84ee7dadaa71fcdd208d313d82e6be6300a776c1 Mon Sep 17 00:00:00 2001 From: Elena German Date: Thu, 22 Jun 2023 15:22:18 -0400 Subject: [PATCH 241/249] iPerf: work-around for ipv6 traffic type on dual-stack cluster (#815) Co-authored-by: Elena German --- roles/iperf3/templates/client.yml.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/iperf3/templates/client.yml.j2 b/roles/iperf3/templates/client.yml.j2 index a3c1b4847..9b6d29904 100644 --- a/roles/iperf3/templates/client.yml.j2 +++ b/roles/iperf3/templates/client.yml.j2 @@ -46,7 +46,9 @@ spec: imagePullPolicy: Always command: ["/bin/sh", "-c"] args: - - "iperf3 -c {{ item.status.podIP }} + - "iperf3 -c + {% if workload_args.extra_options_client is defined and '-6' in workload_args.extra_options_client %} {{ item.status.podIPs[-1].ip }} + {% else %} {{ item.status.podIP }} {% endif %} {% if workload_args.port is defined %} -p {{ workload_args.port }} {% endif %} {% if workload_args.transmit_type is defined and workload_args.transmit_value is defined %} --{{ workload_args.transmit_type }} {{ workload_args.transmit_value }} {% endif %} {% if workload_args.length_buffer is defined %} -l {{ workload_args.length_buffer }} {% endif %} From 506dd2af973cc37e23254e0cb3e5f9d6afb02723 Mon Sep 17 00:00:00 2001 From: Elvir Kuric Date: Tue, 27 Jun 2023 12:43:18 +0200 Subject: [PATCH 242/249] Update roles/fio_distributed/tasks/main.yml with (#816) 1. add storageClassName in "spec" section for PVC 2. remove volume.beta.kubernetes.io/storage-class from annotations section --- roles/fio_distributed/tasks/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/fio_distributed/tasks/main.yml b/roles/fio_distributed/tasks/main.yml index 36dad2bf1..b26e5fd3a 100644 --- a/roles/fio_distributed/tasks/main.yml +++ b/roles/fio_distributed/tasks/main.yml @@ -26,12 +26,11 @@ metadata: name: "claim-{{ item }}-{{ trunc_uuid }}" namespace: '{{ operator_namespace }}' - annotations: - volume.beta.kubernetes.io/storage-class: "{{ workload_args.storageclass }}" spec: accessModes: - "{{ workload_args.pvcaccessmode | default('ReadWriteOnce') }}" volumeMode: "{{ workload_args.pvcvolumemode | default('Filesystem') }}" + storageClassName: "{{ workload_args.storageclass }}" resources: requests: storage: "{{ workload_args.storagesize }}" From 081359d2a90ba556a603840b18015d10d05a89ec Mon Sep 17 00:00:00 2001 From: Murali Krishnasamy <70236227+mukrishn@users.noreply.github.com> Date: Tue, 27 Jun 2023 13:28:28 -0400 Subject: [PATCH 243/249] zone affinity for uperf clients (#811) * zone affinity for clients * updated affinity * corrected the mandatory fields --- roles/uperf/templates/workload.yml.j2 | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index f67e6ee37..fdcf783ab 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -56,6 +56,16 @@ spec: values: - {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} topologyKey: kubernetes.io/hostname + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: type + operator: In + values: + - {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + topologyKey: topology.kubernetes.io/zone podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 @@ -102,7 +112,7 @@ spec: fieldRef: fieldPath: spec.nodeName - name: server_node - value: "{{ workload_args.pin_server|default("unknown") }}" + value: "{{ item.spec.nodeName | default("unknown") }}" {% if workload_args.client_resources is defined %} resources: {{ workload_args.client_resources | to_json }} {% endif %} From 576ee97beaa0c15d7aceb2b2f0795d45fa4a329d Mon Sep 17 00:00:00 2001 From: Joe Talerico Date: Mon, 11 Mar 2024 17:09:02 -0400 Subject: [PATCH 244/249] Fixing BMO for CNV CNV w/ OpenShift 4.15 was broken. This should fix it. Signed-off-by: Joe Talerico --- Dockerfile | 5 ++++- playbooks/benchmark.yml | 2 +- requirements.yml | 4 ++-- roles/backpack/tasks/main.yml | 10 +++++----- roles/common/tasks/main.yml | 2 +- roles/fio_distributed/tasks/main.yml | 2 +- roles/hammerdb/tasks/main.yaml | 2 +- roles/kernel_cache_drop/tasks/main.yml | 2 +- roles/nighthawk/tasks/main.yml | 14 ++++++------- roles/pgbench/tasks/run_workload.yml | 2 +- roles/testpmd/tasks/main.yml | 6 +++--- roles/uperf-scale/tasks/cleanup.yml | 4 ++-- roles/uperf-scale/tasks/setup.yml | 2 +- roles/uperf-scale/tasks/start_client.yml | 2 +- roles/uperf-scale/tasks/start_server.yml | 2 +- roles/uperf-scale/tasks/wait_client_done.yml | 2 +- roles/uperf-scale/tasks/wait_client_ready.yml | 2 +- roles/uperf-scale/tasks/wait_server_ready.yml | 2 +- roles/uperf/tasks/main.yml | 20 +++++++++---------- roles/ycsb/tasks/main.yml | 4 ++-- 20 files changed, 47 insertions(+), 44 deletions(-) diff --git a/Dockerfile b/Dockerfile index e63517e58..081dc013a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,10 @@ -FROM quay.io/operator-framework/ansible-operator:v1.9.0 +FROM quay.io/operator-framework/ansible-operator:v1.32.0 USER root COPY requirements.yml ${HOME}/requirements.yml +RUN python3 -m pip install jmespath +RUN ansible-galaxy collection install community.general +RUN ansible-galaxy collection list RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \ && chmod -R ug+rwx ${HOME}/.ansible diff --git a/playbooks/benchmark.yml b/playbooks/benchmark.yml index e4366aa44..b60b2b8ab 100644 --- a/playbooks/benchmark.yml +++ b/playbooks/benchmark.yml @@ -22,7 +22,7 @@ when: workload.args.drop_cache_kernel is defined - name: Capture operator information - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: "{{ operator_namespace }}" diff --git a/requirements.yml b/requirements.yml index 3f626ed2f..8c8123ba9 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,6 +1,6 @@ --- collections: - name: community.kubernetes - version: "1.2.1" + version: "2.0.1" - name: operator_sdk.util - version: "0.2.0" + version: "0.5.0" diff --git a/roles/backpack/tasks/main.yml b/roles/backpack/tasks/main.yml index 40f2f1177..02e1248ce 100644 --- a/roles/backpack/tasks/main.yml +++ b/roles/backpack/tasks/main.yml @@ -10,7 +10,7 @@ when: benchmark_state.resources[0].status.state is not defined - name: Get benchmark state - k8s_facts: + k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark name: "{{ ansible_operator_meta.name }}" @@ -18,7 +18,7 @@ register: benchmark_state - name: Get DaemonSet state - k8s_facts: + k8s_info: api_version: apps/v1 kind: DaemonSet name: "backpack-{{ trunc_uuid }}" @@ -36,7 +36,7 @@ definition: "{{ lookup('template', 'backpack.yml') | from_yaml }}" - name: Get DaemonSet Status - k8s_facts: + k8s_info: api_version: apps/v1 kind: DaemonSet name: "backpack-{{ trunc_uuid }}" @@ -52,7 +52,7 @@ - block: - name: Get initial pod list - k8s_facts: + k8s_info: kind: Pod namespace: "{{ operator_namespace }}" label_selectors: @@ -82,7 +82,7 @@ when: workload.name != "backpack" - name: Get benchmark state - k8s_facts: + k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark name: "{{ ansible_operator_meta.name }}" diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 1270b776c..13f88b3c0 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -1,7 +1,7 @@ # common tasks across multiple roles go here - block: - name: Get Network Policy - k8s_facts: + k8s_info: kind: NetworkPolicy api_version: networking.k8s.io/v1 namespace: '{{ operator_namespace }}' diff --git a/roles/fio_distributed/tasks/main.yml b/roles/fio_distributed/tasks/main.yml index b26e5fd3a..db63d2195 100644 --- a/roles/fio_distributed/tasks/main.yml +++ b/roles/fio_distributed/tasks/main.yml @@ -152,7 +152,7 @@ - block: - name: wait for db creation job to finish - k8s_facts: + k8s_info: kind: Job api_version: batch/v1 name: 'fio-prefill-{{ trunc_uuid }}' diff --git a/roles/hammerdb/tasks/main.yaml b/roles/hammerdb/tasks/main.yaml index 91ebf3ceb..f5a1f07f3 100644 --- a/roles/hammerdb/tasks/main.yaml +++ b/roles/hammerdb/tasks/main.yaml @@ -196,7 +196,7 @@ - block: - block: - name: wait for db creation job to finish - k8s_facts: + k8s_info: kind: Job api_version: batch/v1 name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' diff --git a/roles/kernel_cache_drop/tasks/main.yml b/roles/kernel_cache_drop/tasks/main.yml index de1a3a94f..8b0ef4180 100644 --- a/roles/kernel_cache_drop/tasks/main.yml +++ b/roles/kernel_cache_drop/tasks/main.yml @@ -10,7 +10,7 @@ shell: "python3 /opt/ansible/roles/kernel_cache_drop/wait_for_daemonset.py 30 {{ operator_namespace }} kernel-cache-dropper" - name: get kernel cache dropper pods - k8s_facts: + k8s_info: kind: Pod label_selectors: - name = kernel-cache-dropper diff --git a/roles/nighthawk/tasks/main.yml b/roles/nighthawk/tasks/main.yml index 040bb1d83..29a02cc67 100644 --- a/roles/nighthawk/tasks/main.yml +++ b/roles/nighthawk/tasks/main.yml @@ -26,7 +26,7 @@ - block: - name: Get server pods - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -46,7 +46,7 @@ - block: - name: Get server pod info - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -55,7 +55,7 @@ register: server_pods - name: Get route info - k8s_facts: + k8s_info: kind: Route api_version: route.openshift.io/v1 namespace: '{{ operator_namespace }}' @@ -86,7 +86,7 @@ - block: - name: Get client pod status - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -129,7 +129,7 @@ - block: - block: - name: Waiting for pods to complete.... - k8s_facts: + k8s_info: kind: pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -157,7 +157,7 @@ - block: - name: Get Server Jobs - k8s_facts: + k8s_info: kind: Job api_version: v1 namespace: '{{ operator_namespace }}' @@ -166,7 +166,7 @@ register: server_jobs - name: Get Server Pods - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' diff --git a/roles/pgbench/tasks/run_workload.yml b/roles/pgbench/tasks/run_workload.yml index 7b183a69c..05c9698a3 100644 --- a/roles/pgbench/tasks/run_workload.yml +++ b/roles/pgbench/tasks/run_workload.yml @@ -3,7 +3,7 @@ command: "redis-cli set pgb_start true" - name: Check for pods to complete - k8s_facts: + k8s_info: kind: pod api_version: v1 namespace: '{{ operator_namespace }}' diff --git a/roles/testpmd/tasks/main.yml b/roles/testpmd/tasks/main.yml index ba190e76c..af96c95d4 100644 --- a/roles/testpmd/tasks/main.yml +++ b/roles/testpmd/tasks/main.yml @@ -36,7 +36,7 @@ - block: - name: Wait for pods to be running.... - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -65,7 +65,7 @@ - block: - name: Get TestPMD pod info - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -89,7 +89,7 @@ - block: - name: Wait for pods to be running.... - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' diff --git a/roles/uperf-scale/tasks/cleanup.yml b/roles/uperf-scale/tasks/cleanup.yml index 34359a1a4..4be469fc2 100644 --- a/roles/uperf-scale/tasks/cleanup.yml +++ b/roles/uperf-scale/tasks/cleanup.yml @@ -4,7 +4,7 @@ ### kind # Cleanup servers, but leave clients around mostly for further examining of results. - name: Get Server Jobs - k8s_facts: + k8s_info: kind: Job api_version: v1 namespace: '{{ operator_namespace }}' @@ -13,7 +13,7 @@ register: server_jobs - name: Get Server Pods - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' diff --git a/roles/uperf-scale/tasks/setup.yml b/roles/uperf-scale/tasks/setup.yml index 455b798d6..6b8d29513 100644 --- a/roles/uperf-scale/tasks/setup.yml +++ b/roles/uperf-scale/tasks/setup.yml @@ -69,7 +69,7 @@ num_server_pods: "{{ (node_hi_idx|int+1) * (pod_hi_idx|int+1) }}" - name: Capture ServiceIP - k8s_facts: + k8s_info: kind: Service api_version: v1 namespace: '{{ operator_namespace }}' diff --git a/roles/uperf-scale/tasks/start_client.yml b/roles/uperf-scale/tasks/start_client.yml index ae620066b..2794df285 100644 --- a/roles/uperf-scale/tasks/start_client.yml +++ b/roles/uperf-scale/tasks/start_client.yml @@ -1,7 +1,7 @@ --- - name: Get pod info - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' diff --git a/roles/uperf-scale/tasks/start_server.yml b/roles/uperf-scale/tasks/start_server.yml index fbc4325e9..440e4989c 100644 --- a/roles/uperf-scale/tasks/start_server.yml +++ b/roles/uperf-scale/tasks/start_server.yml @@ -39,7 +39,7 @@ # to derive its affinity according the 'colocate' variable # - name: Wait for pods to be running.... - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' diff --git a/roles/uperf-scale/tasks/wait_client_done.yml b/roles/uperf-scale/tasks/wait_client_done.yml index dacbb97b6..c3f8cec57 100644 --- a/roles/uperf-scale/tasks/wait_client_done.yml +++ b/roles/uperf-scale/tasks/wait_client_done.yml @@ -2,7 +2,7 @@ - block: ### kind - name: Waiting for pods to complete.... - k8s_facts: + k8s_info: kind: pod api_version: v1 namespace: '{{ operator_namespace }}' diff --git a/roles/uperf-scale/tasks/wait_client_ready.yml b/roles/uperf-scale/tasks/wait_client_ready.yml index 672d0d8b7..b9afe2099 100644 --- a/roles/uperf-scale/tasks/wait_client_ready.yml +++ b/roles/uperf-scale/tasks/wait_client_ready.yml @@ -4,7 +4,7 @@ ### kind - name: Get client pod status - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' diff --git a/roles/uperf-scale/tasks/wait_server_ready.yml b/roles/uperf-scale/tasks/wait_server_ready.yml index ca0324744..6006a9561 100644 --- a/roles/uperf-scale/tasks/wait_server_ready.yml +++ b/roles/uperf-scale/tasks/wait_server_ready.yml @@ -3,7 +3,7 @@ ### kind - name: Get server pods - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index 798a9470b..0c530c6e6 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Capture ServiceIP - k8s_facts: + k8s_info: kind: Service api_version: v1 namespace: '{{ operator_namespace }}' @@ -62,7 +62,7 @@ - block: - name: Get server pods - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -83,7 +83,7 @@ - block: - name: Wait for vms to be running.... - k8s_facts: + k8s_info: kind: VirtualMachineInstance api_version: kubevirt.io/v1alpha3 namespace: '{{ operator_namespace }}' @@ -108,7 +108,7 @@ - block: - name: Get pod info - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -153,7 +153,7 @@ - block: - name: Wait for vms to be running.... - k8s_facts: + k8s_info: kind: VirtualMachineInstance api_version: kubevirt.io/v1alpha3 namespace: '{{ operator_namespace }}' @@ -187,7 +187,7 @@ - block: - name: Get client pod status - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -218,7 +218,7 @@ command: "redis-cli set complete-{{ trunc_uuid }} false" - name: Get client vm status - k8s_facts: + k8s_info: kind: VirtualMachineInstance api_version: kubevirt.io/v1alpha3 namespace: '{{ operator_namespace }}' @@ -253,7 +253,7 @@ - block: - block: - name: Waiting for pods to complete.... - k8s_facts: + k8s_info: kind: pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -295,7 +295,7 @@ - block: - name: Get Server Jobs - k8s_facts: + k8s_info: kind: Job api_version: v1 namespace: '{{ operator_namespace }}' @@ -304,7 +304,7 @@ register: server_jobs - name: Get Server Pods - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' diff --git a/roles/ycsb/tasks/main.yml b/roles/ycsb/tasks/main.yml index cbbe1ac17..85aba2c6b 100644 --- a/roles/ycsb/tasks/main.yml +++ b/roles/ycsb/tasks/main.yml @@ -37,7 +37,7 @@ ycsb_workload_load: "workloada" - name: Wait for Load Job to Succeed... - k8s_facts: + k8s_info: kind: Job api_version: batch/v1 name: 'ycsb-data-load-job-{{ ycsb_workload_load }}-{{ trunc_uuid }}' @@ -108,7 +108,7 @@ ycsb_workload: "{{ workload_args.workloads[wrkload.stdout|int] }}" - name: Wait for YCSB Workload Job to Succeed... - k8s_facts: + k8s_info: kind: Job api_version: batch/v1 name: 'ycsb-bench-job-{{ ycsb_workload }}-{{ trunc_uuid }}' From b3fe7d16988c044af5c143babaec13914c7426af Mon Sep 17 00:00:00 2001 From: Chris Blum Date: Wed, 22 May 2024 12:50:05 +0200 Subject: [PATCH 245/249] Add Youtube video for Quickstart (#824) Co-authored-by: Chris Blum --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 0930ac86c..6c86b18f3 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ cd benchmark-operator make deploy ``` +Watch how to do this on Youtube: + +[![Benchmark Operator Quickstart](http://img.youtube.com/vi/eQxKl_uve84/0.jpg)](http://www.youtube.com/watch?v=eQxKl_uve84 "Benchmark Operator Quickstart") + If you wish to build a version of the operator from your local copy of the repo, you can run ```bash From 2a4091f6d9e8315b2396db8165d160f993190089 Mon Sep 17 00:00:00 2001 From: Pratik Raj Date: Thu, 20 Jun 2024 15:16:30 +0530 Subject: [PATCH 246/249] feat : use --no-cache-dir flag to pip in dockerfiles to save space (#826) using the "--no-cache-dir" flag in pip install, make sure downloaded packages by pip don't cache on the system. This is a best practice that makes sure to fetch from a repo instead of using a local cached one. Further, in the case of Docker Containers, by restricting caching, we can reduce image size. In terms of stats, it depends upon the number of python packages multiplied by their respective size. e.g for heavy packages with a lot of dependencies it reduces a lot by don't cache pip packages. Further, more detailed information can be found at https://medium.com/sciforce/strategies-of-docker-images-optimization-2ca9cc5719b6 Signed-off-by: Pratik Raj --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 081dc013a..046fda623 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM quay.io/operator-framework/ansible-operator:v1.32.0 USER root COPY requirements.yml ${HOME}/requirements.yml -RUN python3 -m pip install jmespath +RUN python3 -m pip install --no-cache-dir jmespath RUN ansible-galaxy collection install community.general RUN ansible-galaxy collection list RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \ From f5ffc3c837c7b84a0ce9ef4a58775c234876579c Mon Sep 17 00:00:00 2001 From: vishnuchalla Date: Tue, 8 Oct 2024 09:41:50 -0400 Subject: [PATCH 247/249] Removing Dev ES instance references (#828) Signed-off-by: Vishnu Challa Co-authored-by: Vishnu Challa --- config/samples/uperf-scale/cr.yaml | 2 +- docs/system-metrics.md | 2 +- e2e/helpers.bash | 1 - roles/nighthawk/README.md | 4 ++-- tests/test_crs/valid_fs_drift.yaml | 4 ++-- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/config/samples/uperf-scale/cr.yaml b/config/samples/uperf-scale/cr.yaml index c70564243..4fcc0b2fd 100644 --- a/config/samples/uperf-scale/cr.yaml +++ b/config/samples/uperf-scale/cr.yaml @@ -6,7 +6,7 @@ metadata: spec: clustername: myk8scluster elasticsearch: - url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443 + url: "http://es-instance.com:9200" #test_user: username_to_attach_to_metadata workload: # cleanup: true diff --git a/docs/system-metrics.md b/docs/system-metrics.md index 5586ae2da..096297604 100644 --- a/docs/system-metrics.md +++ b/docs/system-metrics.md @@ -30,7 +30,7 @@ spec: prom_token: eyJhbGciOiJSUzI1NiIsImtpZCI6IlljTUxlUHBTY2hvUVJQYUlZWmV5MTE4d3VnRFpjUUh5MWZtdE9hdnlvNFUifQ.eyJpc3MiOiJrdWJlcnopeVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJvcGVuc2hpZnQtbW9uaXRvcmluZyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cy10b2tlbi12NGo3YyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjFkYTdkMTRkLWE2MTktNDZjYS1iZGRlLTMzOTYxOWYxMmM4MiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpvcGVuc2hpZnQtbW9uaXRvcmluZzpwcm9tZXRoZXVzLWs4cyJ9.PJp5pD_CjMG05vVLFdxUDRWGA8C71TNyRsUcHmpMlnZLQWBwxZSDZ-Uh3y6g1O-Yz3nopeCoZLB6lugxxalcT1DhiEC9yNK53Lr6HLqaz8nWUbRPbex0913KcuSsnpeRj7tzlwQ2K3WbtIeyyHpG5vAeff07LDvHUcPsc3B_dyetGnInClBHFVEJRES6f5DbIUidtXZEfYKANJNcssly0qZMZicwvM4a_pRp6ctGB-zzR6Ac4lh3b1JLfl_5TLGuuoYEOAeJPVUF4TjemsNNJ5BlycEkVI377LKNdHf83wua5pn3ItJtKE5gdrG833203p-y0pj-UDJj2bAv0cjUQ metrics_profile: https://raw.githubusercontent.com/cloud-bulldozer/benchmark-operator/master/roles/kube-burner/files/metrics-aggregated.yaml elasticsearch: - url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443 + url: "http://es-instance.com:9200" index_name: ripsaw-uperf metadata: collection: false diff --git a/e2e/helpers.bash b/e2e/helpers.bash index 7e8a4414e..69cf525a4 100644 --- a/e2e/helpers.bash +++ b/e2e/helpers.bash @@ -8,7 +8,6 @@ NAMESPACE=benchmark-operator basic_setup() { export PROMETHEUS_TOKEN=$(oc sa get-token -n openshift-monitoring prometheus-k8s) - export ES_SERVER=${ES_SERVER:-https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com} } basic_teardown() { diff --git a/roles/nighthawk/README.md b/roles/nighthawk/README.md index 695e5bb5b..ec509e75a 100644 --- a/roles/nighthawk/README.md +++ b/roles/nighthawk/README.md @@ -17,7 +17,7 @@ metadata: spec: clustername: myk8scluster elasticsearch: - url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443 + url: "http://es-instance.com:9200" workload: cleanup: true name: nighthawk @@ -45,7 +45,7 @@ metadata: spec: clustername: myk8scluster elasticsearch: - url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443 + url: "http://es-instance.com:9200" workload: cleanup: true name: nighthawk diff --git a/tests/test_crs/valid_fs_drift.yaml b/tests/test_crs/valid_fs_drift.yaml index 54a73ee18..c5aa35b3d 100644 --- a/tests/test_crs/valid_fs_drift.yaml +++ b/tests/test_crs/valid_fs_drift.yaml @@ -7,12 +7,12 @@ spec: system_metrics: collection: true prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 - es_url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com + es_url: "http://es-instance.com:9200" prom_token: PROMETHEUS_TOKEN metrics_profile: node-metrics.yml # where elastic search is running elasticsearch: - url: https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com + url: "http://es-instance.com:9200" index_name: ripsaw-fs-drift metadata: collection: true From 83b5f263ca173a71eeba232d378ff1558272c274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sevilla?= Date: Fri, 25 Oct 2024 09:12:45 +0200 Subject: [PATCH 248/249] Kubevirt fixes (#830) * Update kubevirt API version Signed-off-by: Raul Sevilla * Wait for servers to be running before registering interfaces Signed-off-by: Raul Sevilla * Centos 8 appstream is EOL: Use vault repository Signed-off-by: Raul Sevilla --------- Signed-off-by: Raul Sevilla --- image_resources/centos8-appstream.repo | 6 +++--- roles/fio_distributed/tasks/main.yml | 3 ++- roles/fio_distributed/templates/server_vm.yml.j2 | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/image_resources/centos8-appstream.repo b/image_resources/centos8-appstream.repo index c2067f100..a20e22a86 100644 --- a/image_resources/centos8-appstream.repo +++ b/image_resources/centos8-appstream.repo @@ -1,17 +1,17 @@ [centos8-appstream-x86_64] name=CentOS-8-Appstream-x86_64 -baseurl=http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/ +baseurl=http://vault.centos.org/centos/8-stream/AppStream/x86_64/os/ enabled=0 gpgcheck=0 [centos8-appstream-aarch64] name=CentOS-8-Appstream-aarch64 -baseurl=http://mirror.centos.org/centos/8-stream/AppStream/aarch64/os/ +baseurl=http://vault.centos.org/centos/8-stream/AppStream/aarch64/os/ enabled=0 gpgcheck=0 [centos8-appstream-ppc64le] name=CentOS-8-Appstream-ppc64le -baseurl=http://mirror.centos.org/centos/8-stream/AppStream/ppc64le/os/ +baseurl=http://vault.centos.org/centos/8-stream/AppStream/ppc64le/os/ enabled=0 gpgcheck=0 diff --git a/roles/fio_distributed/tasks/main.yml b/roles/fio_distributed/tasks/main.yml index db63d2195..e9913a007 100644 --- a/roles/fio_distributed/tasks/main.yml +++ b/roles/fio_distributed/tasks/main.yml @@ -93,7 +93,7 @@ - name: Capture pod list k8s_info: kind: VirtualMachineInstance - api_version: kubevirt.io/v1alpha3 + api_version: kubevirt.io/v1 namespace: '{{ operator_namespace }}' label_selectors: - app = fio-benchmark-{{ trunc_uuid }} @@ -110,6 +110,7 @@ set_fact: pod_details: "{{ pod_details|default({}) | combine({item.status.interfaces[0].ipAddress: item.status.nodeName}) }}" with_items: "{{ server_pods.resources }}" + when: benchmark_state.resources[0].status.state == "StartingClient" when: (workload.args.kind | default('pod')) == "vm" diff --git a/roles/fio_distributed/templates/server_vm.yml.j2 b/roles/fio_distributed/templates/server_vm.yml.j2 index 7fe34ee96..8d42170b1 100644 --- a/roles/fio_distributed/templates/server_vm.yml.j2 +++ b/roles/fio_distributed/templates/server_vm.yml.j2 @@ -1,5 +1,5 @@ --- -apiVersion: kubevirt.io/v1alpha3 +apiVersion: kubevirt.io/v1 kind: VirtualMachineInstance metadata: name: 'fio-server-{{item | string}}-{{ trunc_uuid }}' From 31c696978c4c0e1ef3c9c739ad4223cbe78eda00 Mon Sep 17 00:00:00 2001 From: Mans Matulewicz Date: Wed, 13 Nov 2024 18:50:33 +0100 Subject: [PATCH 249/249] Update stressng.md to reflect crd requirements (#825) ``` looking for "jobfile.j2" at "/opt/ansible/roles/stressng/templates/jobfile.j2" File lookup using /opt/ansible/roles/stressng/templates/jobfile.j2 as file fatal: [localhost]: FAILED! => { "msg": "The task includes an option with an undefined variable. The error was: 'dict object' has no attribute 'cpu_method'. 'dict object' has no attribute 'cpu_method'. 'dict object' has no attribute 'cpu_method'. 'dict object' has no attribute 'cpu_method'\n\nThe error appears to be in '/opt/ansible/roles/stressng/tasks/main.yaml': line 4, column 5, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\n - name: template stressng config file\n ^ here\n" } ``` --- docs/stressng.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/stressng.md b/docs/stressng.md index 3354bea3f..7fbb9e8c4 100644 --- a/docs/stressng.md +++ b/docs/stressng.md @@ -39,6 +39,7 @@ spec: # cpu stressor options cpu_stressors: "1" cpu_percentage: "100" + cpu_method: "all" # vm stressor option vm_stressors: "1" vm_bytes: "128M"