From 495229f5112b51afff09a619e6412a271cba2467 Mon Sep 17 00:00:00 2001 From: Alex Yefimov <126113326+ayefimov-1@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:44:43 -0400 Subject: [PATCH 01/13] [zuul] Add logging test job Add a job that runs the telemetry_logging and common roles against the openstack cloud. Separate plays target compute and control nodes --- .zuul.yaml | 19 ++- ci/logging_tests_computes.yml | 41 ++++++ ci/logging_tests_local.yml | 248 ++++++++++++++++++++++++++++++++++ ci/run_playbooks_logging.yml | 57 ++++++++ ci/vars-logging-test.yml | 5 + 5 files changed, 368 insertions(+), 2 deletions(-) create mode 100644 ci/logging_tests_computes.yml create mode 100644 ci/logging_tests_local.yml create mode 100644 ci/run_playbooks_logging.yml create mode 100644 ci/vars-logging-test.yml diff --git a/.zuul.yaml b/.zuul.yaml index 62c2773d..aaa287db 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -12,7 +12,7 @@ - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml" roles: - zuul: github.com/openstack-k8s-operators/ci-framework - required-projects: + required-projects: &required_projects - name: github.com/infrawatch/service-telemetry-operator - name: openstack-k8s-operators/ci-framework override-checkout: main @@ -35,6 +35,21 @@ - README* - .*/*.md +- job: + name: functional-logging-tests-osp18 + parent: telemetry-operator-multinode-logging + description: | + Run the logging functional test on osp18+patched version + vars: + cifmw_extras: + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-logging.yml" + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-logging-test.yml" + roles: + - zuul: github.com/openstack-k8s-operators/ci-framework + required-projects: *required_projects + irrelevant-files: *irrelevant_files + - job: name: feature-verification-tests-noop parent: noop @@ -45,7 +60,6 @@ - ci/noop.yml files: *irrelevant_files - - project: name: infrawatch/feature-verification-tests github-check: @@ -54,3 +68,4 @@ - openstack-k8s-operators-content-provider: override-checkout: main - functional-tests-on-osp18 + - functional-logging-tests-osp18 diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml new file mode 100644 index 00000000..5b871eb7 --- /dev/null +++ b/ci/logging_tests_computes.yml @@ -0,0 +1,41 @@ +- name: Prepare Logging Tests + hosts: computes + gather_facts: false + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + + +- name: "Verify ctlplane logging containers and files" + hosts: computes + gather_facts: no + vars: + container_test_id: "RHOSO-12753" + container_list: + - ceilometer_agent_compute + - ceilometer_agent_ipmi + - node_exporter + + file_test_id: "RHOSO-12754" + file_list: + - /etc/rsyslog.d/10-telemetry.conf + + tasks: + - name: "Run file and container tests" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging journalctl identifiers" + hosts: computes + gather_facts: no + vars: + identifiers_test_id: "RHOSO-12681" + identifiers_list: + - ceilometer_agent_compute + - nova_compute + + tasks: + - name: "Verify journalctl logging identifiers" + ansible.builtin.import_role: + name: telemetry_logging diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml new file mode 100644 index 00000000..a2dee02e --- /dev/null +++ b/ci/logging_tests_local.yml @@ -0,0 +1,248 @@ +- name: Prepare Logging Tests + hosts: controller + gather_facts: false + vars: + id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + + tasks: + - name: "Log into OCP" + ansible.builtin.shell: + cmd: | + oc login -u kubeadmin -p "12345678" https://api.crc.testing:6443 + failed_when: false + changed_when: false + + - name: "Create id rsa file for ctlplane access" + ansible.builtin.shell: + cmd: | + oc extract -n openstack secrets/dataplane-ansible-ssh-private-key-secret --to=- | grep -v "ssh\-rsa" > "{{ id_rsa_path }}" + register: rsa_results + failed_when: rsa_results.rc != 0 + changed_when: false + + - name: "Change files permissions" + ansible.builtin.shell: + cmd: | + chmod 600 "{{ id_rsa_path }}" + changed_when: false + +- name: "Verify logging projects, endpoints, credentials, nodes, pods, services, manifests and subscriptions" + hosts: controller + gather_facts: no + vars: + proj_test_id: "RHOSO-12668" + proj_list: + - openshift-openstack-infra + - openshift + - openstack-operators + - openshift-logging + + + endpoint_test_id: "RHOSO-12682" + endpoint_list: + - [nova,compute,public] + - [nova,compute,internal] + - [placement,placement,public] + - [placement,placement,internal] + - [swift,object-store,public] + - [swift,object-store,internal] + - [cinderv3,volumev3,public] + - [cinderv3,volumev3,internal] + - [barbican,key-manager,public] + - [barbican,key-manager,internal] + - [keystone,identity,public] + - [keystone,identity,internal] + - [glance,image,public] + - [glance,image,internal] + - [neutron,network,public] + - [neutron,network,internal] + + + cred_test_id: "RHOSO-12670" + cred_list: + - alertingrules.loki.grafana.com + - lokistacks.loki.grafana.com + - recordingrules.loki.grafana.com + - rulerconfigs.loki.grafana.com + + + node_test_id: "RHOSO-12671" + node_list: + - edpm-compute-0 + - edpm-compute-1 + - crc + + + pod_test_id: "RHOS0-12672" + pod_status_str: "Running" + pod_nspace: openstack-operators + pod_list: + - telemetry-operator-controller-manager + - dataplane-operator-controller-manager + + + service_test_id: "RHOSO-12675" + service_nspace: openshift-logging + service_list: + - cluster-logging-operator-metrics + - logging-loki-compactor-grpc + - logging-loki-compactor-http + - logging-loki-distributor-grpc + - logging-loki-distributor-http + - logging-loki-gateway-http + - logging-loki-gossip-ring + - logging-loki-index-gateway-grpc + - logging-loki-index-gateway-http + - logging-loki-ingester-grpc + - logging-loki-ingester-http + - logging-loki-querier-grpc + - logging-loki-querier-http + - logging-loki-query-frontend-grpc + - logging-loki-query-frontend-http + - logging-view-plugin + - openstack-logging + + + manifest_test_id: "RHOSO-12677" + manifest_list: + - "loki-operator 2" + - "loki-helm-operator 1" + + + subscription_test_id: "RHOSO-12678" + subscription_nspace: openshift-operators-redhat + subscription_list: + - loki-operator + + tasks: + - name: "Run logging project, endpoint, credential, node, pod, service, manifest and subscription tests" + ansible.builtin.import_role: + name: common + + + +- name: "Verify logging pods are running in openstack" + hosts: controller + gather_facts: no + vars: + pod_test_id: "RHOSO-12752" + pod_status_str: "Running" + pod_nspace: openstack + pod_list: + - openstackclient + + tasks: + - name: "Verify Running Pods" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods are running in openshift-operators-redhat" + hosts: controller + gather_facts: no + vars: + pod_test_id: "RHOSO-12673" + pod_status_str: "Running" + pod_nspace: openshift-operators-redhat + pod_list: + - loki-operator-controller-manager + + tasks: + - name: "Verify Pods running" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods are running in openshift-logging" + hosts: controller + gather_facts: no + vars: + pod_test_id: "RHOSO-12676" + pod_status_str: "Running" + pod_nspace: openshift-logging + pod_list: + - cluster-logging-operator + - collector + - logging-loki-compactor + - logging-loki-distributor + - logging-loki-gateway + - logging-loki-index-gateway + - logging-loki-ingester + - logging-loki-querier + - logging-loki-query-frontend + - logging-view-plugin + + ### see JIRA LOG-5431 if pods not running + tasks: + - name: "Verify Pods running" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods are running in minio-dev" + hosts: controller + gather_facts: no + vars: + pod_test_id: "RHOSO-12674" + pod_status_str: "Running" + pod_nspace: minio-dev + pod_list: + - minio + + tasks: + - name: "Run pod running tests" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods have complete status in openstack" + hosts: controller + gather_facts: no + vars: + pod_test_id: "RHOSO-12679" + pod_nspace: openstack + pod_status_str: "Completed" + pod_list: + - bootstrap-edpm-deployment-logging + - configure-network-edpm-deployment-logging + - configure-os-edpm-deployment-logging + - download-cache-edpm-deployment-logging + - install-certs-edpm-deployment-logging + - install-os-edpm-deployment-logging + - libvirt-edpm-deployment-logging + - logging-edpm-deployment-logging-openstack-edpm + - neutron-metadata-edpm-deployment-logging + - nova-custom-edpm-deployment + - ovn-edpm-deployment-logging + - reboot-os-edpm-deployment-logging + - repo-setup-edpm-deployment-logging + - run-os-edpm-deployment-logging + - ssh-known-hosts-edpm-deployment-logging + - telemetry-edpm-deployment-logging + - validate-network-edpm-deployment-logging + + tasks: + - name: "Run pods completed tests" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging services are running in openstack" + hosts: controller + gather_facts: no + vars: + service_test_id: "RHOSO-12749" + service_nspace: openstack + service_list: + - nova-internal + - nova-metadata-internal + - nova-novncproxy-cell1-public + - nova-public + + tasks: + - name: "Run Services tests" + ansible.builtin.import_role: + name: common diff --git a/ci/run_playbooks_logging.yml b/ci/run_playbooks_logging.yml new file mode 100644 index 00000000..1e144f4f --- /dev/null +++ b/ci/run_playbooks_logging.yml @@ -0,0 +1,57 @@ +--- +- name: "Run logging functional test playbooks" + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars_files: + - vars/common.yml + tasks: + - name: "Create log dir" + ansible.builtin.file: + path: "{{ logs_dir }}" + state: directory + mode: "0755" + + - name: "Set dynamic KUBECONFIG and PATH vars" + copy: + content: | + cifmw_openshift_kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + cifmw_path: "{{ cifmw_path }}" + vars_dir: "{{ fvt_dir }}/ci/vars" + dest: "{{ env_vars_file }}" + + - name: Include vars from the extra_vars files + ansible.builtin.include_vars: + dir: "{{ cifmw_basedir }}/artifacts/parameters" + + - name: Run telemetry-logging tests on OSP18 + block: + - name: "Run local logging tests" + ansible.builtin.shell: + cmd: | + ANSIBLE_CONFIG=ci/ansible.cfg ansible-playbook -v -e @"{{ env_vars_file }}" ci/logging_tests_computes.yml + chdir: "{{ fvt_dir }}" + register: output + + - name: Save ansible output to a file + ansible.builtin.copy: + content: "{{ output.stdout }}" + dest: "{{ logs_dir }}/ansible_run.log" + + - name: "Run cltplane logging tests" + ansible.builtin.shell: + cmd: | + ANSIBLE_CONFIG=ci/ansible.cfg ansible-playbook -v -e @"{{ env_vars_file }}" ci/logging_tests_local.yml + chdir: "{{ fvt_dir }}" + register: output2 + + - name: Save ansible output to a file + ansible.builtin.lineinfile: + line: "{{ output2.stdout }}" + path: "{{ logs_dir }}/ansible_run.log" + + - name: Include report result + ansible.builtin.include_tasks: + file: report_result.yml diff --git a/ci/vars-logging-test.yml b/ci/vars-logging-test.yml new file mode 100644 index 00000000..326f0ee5 --- /dev/null +++ b/ci/vars-logging-test.yml @@ -0,0 +1,5 @@ +--- +# temp: skip os-net-setup +post_deploy_run_logging_functional_test: + source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_playbooks_logging.yml" + type: playbook From b875a0d04092c79a8ba8e959f3246bed0db42a75 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 9 Aug 2024 17:18:51 +0100 Subject: [PATCH 02/13] Try to use new hooks to run test playbooks directly --- ci/logging_tests_computes.yml | 6 +++--- ci/run_playbooks_logging.yml | 2 +- ci/vars-logging-test.yml | 16 +++++++++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index 5b871eb7..723802c0 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -22,11 +22,11 @@ tasks: - name: "Run file and container tests" - ansible.builtin.import_role: + ansible.builtin.import_role: name: common -- name: "Verify logging journalctl identifiers" +- name: "Verify logging journalctl identifiers" hosts: computes gather_facts: no vars: @@ -36,6 +36,6 @@ - nova_compute tasks: - - name: "Verify journalctl logging identifiers" + - name: "Verify journalctl logging identifiers" ansible.builtin.import_role: name: telemetry_logging diff --git a/ci/run_playbooks_logging.yml b/ci/run_playbooks_logging.yml index 1e144f4f..35e0a0cb 100644 --- a/ci/run_playbooks_logging.yml +++ b/ci/run_playbooks_logging.yml @@ -52,6 +52,6 @@ line: "{{ output2.stdout }}" path: "{{ logs_dir }}/ansible_run.log" - - name: Include report result + - name: Include report result ansible.builtin.include_tasks: file: report_result.yml diff --git a/ci/vars-logging-test.yml b/ci/vars-logging-test.yml index 326f0ee5..4722bbab 100644 --- a/ci/vars-logging-test.yml +++ b/ci/vars-logging-test.yml @@ -1,5 +1,15 @@ --- -# temp: skip os-net-setup -post_deploy_run_logging_functional_test: - source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_playbooks_logging.yml" +#post_deploy_run_logging_functional_test: +# source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_playbooks_logging.yml" +# config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" +# type: playbook +post_deploy_fvt_logging_01_computes: + source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_computes.yml" + config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" type: playbook + +post_deploy_fvt_logging_00_local: + source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_local.yml" + config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" + type: playbook + From a7796db3f3dc18813dc3c7dad42d858c34d85fce Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 13 Aug 2024 19:48:45 +0100 Subject: [PATCH 03/13] Fix syntax error in variable definition --- .zuul.yaml | 4 ++-- ci/vars-logging-test.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index aaa287db..9109f1a1 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -64,8 +64,8 @@ name: infrawatch/feature-verification-tests github-check: jobs: - - feature-verification-tests-noop + #- feature-verification-tests-noop - openstack-k8s-operators-content-provider: override-checkout: main - - functional-tests-on-osp18 + #- functional-tests-on-osp18 - functional-logging-tests-osp18 diff --git a/ci/vars-logging-test.yml b/ci/vars-logging-test.yml index 4722bbab..62f43338 100644 --- a/ci/vars-logging-test.yml +++ b/ci/vars-logging-test.yml @@ -4,12 +4,12 @@ # config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" # type: playbook post_deploy_fvt_logging_01_computes: - source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_computes.yml" + source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_computes.yml" config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" type: playbook post_deploy_fvt_logging_00_local: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_local.yml" - config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" - type: playbook + config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" + type: playbook From 3e94fafd35e0093e7aead6a67beda0bca2764f27 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 14 Aug 2024 13:51:27 +0100 Subject: [PATCH 04/13] Use ignore_errors in plays for test_logging The logging plugins we use need all the tests to run in order to report properly. Adding ignore_errors to the plays that run the tests allows all the tests to run even when one fails. Also added verbosity to the ansible config to get more info about the failing tests --- ci/ansible.cfg | 3 +++ ci/logging_tests_computes.yml | 5 +++-- ci/logging_tests_local.yml | 24 +++++++++--------------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/ci/ansible.cfg b/ci/ansible.cfg index 3c9d90c0..fb76f2b9 100644 --- a/ci/ansible.cfg +++ b/ci/ansible.cfg @@ -3,3 +3,6 @@ callbacks_enabled = custom_logger callback_plugins = ../callback_plugins # additional paths to search for roles roles_path = ../roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.ansible/roles:../../ci-framework/roles + +# temp update to get more information from the playbooks running functional tests +verbosity = 5 diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index 723802c0..94f1d8b4 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -1,3 +1,4 @@ +--- - name: Prepare Logging Tests hosts: computes gather_facts: false @@ -9,6 +10,7 @@ - name: "Verify ctlplane logging containers and files" hosts: computes gather_facts: no + ignore_errors: true vars: container_test_id: "RHOSO-12753" container_list: @@ -19,7 +21,6 @@ file_test_id: "RHOSO-12754" file_list: - /etc/rsyslog.d/10-telemetry.conf - tasks: - name: "Run file and container tests" ansible.builtin.import_role: @@ -29,12 +30,12 @@ - name: "Verify logging journalctl identifiers" hosts: computes gather_facts: no + ignore_errors: true vars: identifiers_test_id: "RHOSO-12681" identifiers_list: - ceilometer_agent_compute - nova_compute - tasks: - name: "Verify journalctl logging identifiers" ansible.builtin.import_role: diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index a2dee02e..d20f45b5 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -1,6 +1,8 @@ +--- - name: Prepare Logging Tests hosts: controller gather_facts: false + ignore_errors: true vars: id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" environment: @@ -32,6 +34,7 @@ - name: "Verify logging projects, endpoints, credentials, nodes, pods, services, manifests and subscriptions" hosts: controller gather_facts: no + ignore_errors: true vars: proj_test_id: "RHOSO-12668" proj_list: @@ -39,8 +42,6 @@ - openshift - openstack-operators - openshift-logging - - endpoint_test_id: "RHOSO-12682" endpoint_list: - [nova,compute,public] @@ -60,7 +61,6 @@ - [neutron,network,public] - [neutron,network,internal] - cred_test_id: "RHOSO-12670" cred_list: - alertingrules.loki.grafana.com @@ -68,14 +68,12 @@ - recordingrules.loki.grafana.com - rulerconfigs.loki.grafana.com - node_test_id: "RHOSO-12671" node_list: - edpm-compute-0 - edpm-compute-1 - crc - pod_test_id: "RHOS0-12672" pod_status_str: "Running" pod_nspace: openstack-operators @@ -83,7 +81,6 @@ - telemetry-operator-controller-manager - dataplane-operator-controller-manager - service_test_id: "RHOSO-12675" service_nspace: openshift-logging service_list: @@ -104,36 +101,30 @@ - logging-loki-query-frontend-http - logging-view-plugin - openstack-logging - - manifest_test_id: "RHOSO-12677" manifest_list: - "loki-operator 2" - "loki-helm-operator 1" - - subscription_test_id: "RHOSO-12678" subscription_nspace: openshift-operators-redhat subscription_list: - loki-operator - tasks: - name: "Run logging project, endpoint, credential, node, pod, service, manifest and subscription tests" ansible.builtin.import_role: name: common - - name: "Verify logging pods are running in openstack" hosts: controller gather_facts: no + ignore_errors: true vars: pod_test_id: "RHOSO-12752" pod_status_str: "Running" pod_nspace: openstack pod_list: - openstackclient - tasks: - name: "Verify Running Pods" ansible.builtin.import_role: @@ -143,6 +134,7 @@ - name: "Verify logging pods are running in openshift-operators-redhat" hosts: controller gather_facts: no + ignore_errors: true vars: pod_test_id: "RHOSO-12673" pod_status_str: "Running" @@ -159,6 +151,7 @@ - name: "Verify logging pods are running in openshift-logging" hosts: controller gather_facts: no + ignore_errors: true vars: pod_test_id: "RHOSO-12676" pod_status_str: "Running" @@ -185,6 +178,7 @@ - name: "Verify logging pods are running in minio-dev" hosts: controller gather_facts: no + ignore_errors: true vars: pod_test_id: "RHOSO-12674" pod_status_str: "Running" @@ -201,6 +195,7 @@ - name: "Verify logging pods have complete status in openstack" hosts: controller gather_facts: no + ignore_errors: true vars: pod_test_id: "RHOSO-12679" pod_nspace: openstack @@ -223,7 +218,6 @@ - ssh-known-hosts-edpm-deployment-logging - telemetry-edpm-deployment-logging - validate-network-edpm-deployment-logging - tasks: - name: "Run pods completed tests" ansible.builtin.import_role: @@ -233,6 +227,7 @@ - name: "Verify logging services are running in openstack" hosts: controller gather_facts: no + ignore_errors: true vars: service_test_id: "RHOSO-12749" service_nspace: openstack @@ -241,7 +236,6 @@ - nova-metadata-internal - nova-novncproxy-cell1-public - nova-public - tasks: - name: "Run Services tests" ansible.builtin.import_role: From 90c1a3effc601d0cd908559360edadc72b2b2b43 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 16 Aug 2024 11:58:34 +0100 Subject: [PATCH 05/13] Update playbooks to remove login play --- ci/logging_tests_local.yml | 71 +++++++++++++++++++++----------------- ci/vars-logging-test.yml | 1 + 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index d20f45b5..82c9cce4 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -1,35 +1,40 @@ --- -- name: Prepare Logging Tests - hosts: controller - gather_facts: false - ignore_errors: true - vars: - id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" - environment: - KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" - PATH: "{{ cifmw_path }}" - - tasks: - - name: "Log into OCP" - ansible.builtin.shell: - cmd: | - oc login -u kubeadmin -p "12345678" https://api.crc.testing:6443 - failed_when: false - changed_when: false - - - name: "Create id rsa file for ctlplane access" - ansible.builtin.shell: - cmd: | - oc extract -n openstack secrets/dataplane-ansible-ssh-private-key-secret --to=- | grep -v "ssh\-rsa" > "{{ id_rsa_path }}" - register: rsa_results - failed_when: rsa_results.rc != 0 - changed_when: false - - - name: "Change files permissions" - ansible.builtin.shell: - cmd: | - chmod 600 "{{ id_rsa_path }}" - changed_when: false +#- name: Prepare Logging Tests +# hosts: controller +# gather_facts: true +# ignore_errors: false +# vars: +# id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" +# environment: +# KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" +# PATH: "{{ cifmw_path }}" +# +# tasks: +# # This needs to be updated; what do we use elsewhere? +# # Elsewhere, we assume that we're already logged in +# - name: "Log into OCP" +# ansible.builtin.shell: +# cmd: | +# oc login -u kubeadmin -p "12345678" https://api.crc.testing:6443 +# failed_when: false +# changed_when: false +# +# # I don't think we need this, Zuul __should__ have taken care of this already +# - name: "Create id rsa file for ctlplane access" +# ansible.builtin.shell: +# cmd: | +# oc extract -n openstack secrets/dataplane-ansible-ssh-private-key-secret --to=- | grep -v "ssh\-rsa" > "{{ id_rsa_path }}" +# register: rsa_results +# failed_when: rsa_results.rc != 0 +# changed_when: false +# +# # (efoley): Once again, I don't think this is necessary; if it's not needed for zuul, because it's already done, but is needed before running this on other systems, then the README should be updated to reflect this; +# # A troubleshooting section can be added too so users can identify when they are missing these configuration steps +# - name: "Change files permissions" +# ansible.builtin.shell: +# cmd: | +# chmod 600 "{{ id_rsa_path }}" +# changed_when: false - name: "Verify logging projects, endpoints, credentials, nodes, pods, services, manifests and subscriptions" hosts: controller @@ -146,7 +151,9 @@ - name: "Verify Pods running" ansible.builtin.import_role: name: common - + # vars can be passed here, so we can use the same play for this and the next play. + # vars: + # - - name: "Verify logging pods are running in openshift-logging" hosts: controller diff --git a/ci/vars-logging-test.yml b/ci/vars-logging-test.yml index 62f43338..f9c4278b 100644 --- a/ci/vars-logging-test.yml +++ b/ci/vars-logging-test.yml @@ -3,6 +3,7 @@ # source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_playbooks_logging.yml" # config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" # type: playbook +# pre_tests or post_tests are also options for when the FVT roles are run post_deploy_fvt_logging_01_computes: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_computes.yml" config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" From 0d409d2983c769fd02c97bbc929c3cb54c7eea2e Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 16 Aug 2024 18:08:14 +0100 Subject: [PATCH 06/13] custom_logger: except AnsibleError and try to get more info on failing callback plugin --- callback_plugins/custom_logger.py | 40 +++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/callback_plugins/custom_logger.py b/callback_plugins/custom_logger.py index 8e1c109e..38d5e261 100644 --- a/callback_plugins/custom_logger.py +++ b/callback_plugins/custom_logger.py @@ -4,6 +4,9 @@ import os import re + +from ansible.errors import AnsibleCallbackError +from ansible.errors import AnsibleError from ansible.plugins.callback import CallbackBase DOCUMENTATION = ''' @@ -63,18 +66,35 @@ def log_task_result(self, host, result, task_name): self.results[host][result] += 1 def log_summary_results(self, host): + # The issue is presenting itself in this method file_path = os.path.join(self.output_dir, f"summary_results.log") + # temperorily add more detail to the output + print(self.results) + with open(file_path, 'w') as f: f.write(f"Host: {host}\n") - f.write(f"Tasks Succeeded: {self.results[host]['passed']}\n") - f.write(f"Tasks Failed: {self.results[host]['failed']}\n") - f.write(f"Tasks Skipped: {self.results[host]['skipped']}\n") - f.write("Failed Tasks:\n") - for task_name in self.results[host]['failed_task_names']: - f.write(f" - {task_name}\n") - f.write("Succeeded Tasks:\n") - for task_name in self.results[host]['ok_task_names']: - f.write(f" - {task_name}\n") + # This is the failed issue. + # Why is it happening? + # Unknown. But does it happen for the other values. + # It is happening for compute-0 + # I need to see the partial log file, so this should NOT cause a failure. + # try/catch ANY exception + # It could be a nice upgrade to use jinja2 template to render this file. + # IS the issue that there were 'f's preceeding the strings? + try: + f.write("Tasks Succeeded: {self.results[host]['passed']}\n") + f.write("Tasks Failed: {self.results[host]['failed']}\n") + f.write("Tasks Skipped: {self.results[host]['skipped']}\n") + f.write("Failed Tasks:\n") + for task_name in self.results[host]['failed_task_names']: + f.write(f" - {task_name}\n") + f.write("Succeeded Tasks:\n") + for task_name in self.results[host]['ok_task_names']: + f.write(f" - {task_name}\n") + except AnsibleError as e: + # this is probably an AnsibleCallbackError + print("Ooops, there was an error") + print(e) def v2_runner_on_ok(self, result): host = result._host.get_name() @@ -89,4 +109,4 @@ def v2_runner_on_failed(self, result, ignore_errors=False): def v2_runner_on_skipped(self, result): host = result._host.get_name() task_name = result._task.get_name() - self.log_task_result(host, 'skipped', task_name) \ No newline at end of file + self.log_task_result(host, 'skipped', task_name) From c3122c7c2813393a16fbb9a8d83b3842225a1888 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 16 Aug 2024 20:39:11 +0100 Subject: [PATCH 07/13] [ci/logging_tests] Add KUBECONFIG and PATH to play environment --- ci/logging_tests_computes.yml | 6 ++++++ ci/logging_tests_local.yml | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index 94f1d8b4..b8d37da5 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -11,6 +11,9 @@ hosts: computes gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: container_test_id: "RHOSO-12753" container_list: @@ -31,6 +34,9 @@ hosts: computes gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: identifiers_test_id: "RHOSO-12681" identifiers_list: diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index 82c9cce4..cbf93e29 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -40,6 +40,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: proj_test_id: "RHOSO-12668" proj_list: @@ -124,6 +127,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: pod_test_id: "RHOSO-12752" pod_status_str: "Running" @@ -140,6 +146,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: pod_test_id: "RHOSO-12673" pod_status_str: "Running" @@ -159,6 +168,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: pod_test_id: "RHOSO-12676" pod_status_str: "Running" @@ -186,6 +198,11 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + + vars: pod_test_id: "RHOSO-12674" pod_status_str: "Running" @@ -203,6 +220,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: pod_test_id: "RHOSO-12679" pod_nspace: openstack @@ -235,6 +255,9 @@ hosts: controller gather_facts: no ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" vars: service_test_id: "RHOSO-12749" service_nspace: openstack From a60bc35ce22270179494f3b06865dbc235974d2c Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 19 Aug 2024 18:12:12 +0100 Subject: [PATCH 08/13] Add report_results to logging tests --- callback_plugins/custom_logger.py | 1 + ci/logging_tests_computes.yml | 17 ++++++++++++++++- ci/logging_tests_local.yml | 27 +++++++++++++++++++++------ 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/callback_plugins/custom_logger.py b/callback_plugins/custom_logger.py index 38d5e261..63ecfcf0 100644 --- a/callback_plugins/custom_logger.py +++ b/callback_plugins/custom_logger.py @@ -44,6 +44,7 @@ def playbook_on_stats(self, stats): self.log_summary_results(host) def log_task_result(self, host, result, task_name): + print("logging task result: %s - %s - %s" % (host, result, task_name)) # test_run_result.out only interested in the test tasks, not setup or debug. if "RHELOSP" in task_name or "RHOSO" in task_name: if "RHELOSP" in task_name: diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index b8d37da5..6f33ef61 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -5,7 +5,14 @@ environment: KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" - + vars_files: + - vars/common.yml + tasks: + - name: Create log dir + ansible.builtin.file: + path: "{{ logs_dir }}/logging_compute" + state: directory + mode: "0755" - name: "Verify ctlplane logging containers and files" hosts: computes @@ -46,3 +53,11 @@ - name: "Verify journalctl logging identifiers" ansible.builtin.import_role: name: telemetry_logging + +- name: Include report result + hosts: computes + tasks: + - ansible.builtin.include_tasks: + file: report_result.yml + vars: + test_dir: "{{ logs_dir }}/logging_local" diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index cbf93e29..2258ee2e 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -1,15 +1,22 @@ --- -#- name: Prepare Logging Tests -# hosts: controller -# gather_facts: true -# ignore_errors: false +- name: Prepare Logging Tests + hosts: controller + gather_facts: true + ignore_errors: false # vars: # id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" # environment: # KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" # PATH: "{{ cifmw_path }}" -# -# tasks: + vars_files: + - vars/common.yml + tasks: + - name: Create log dir + ansible.builtin.file: + path: "{{ logs_dir }}/logging_local" + state: directory + mode: "0755" + # # This needs to be updated; what do we use elsewhere? # # Elsewhere, we assume that we're already logged in # - name: "Log into OCP" @@ -270,3 +277,11 @@ - name: "Run Services tests" ansible.builtin.import_role: name: common + +- name: Include report result + hosts: controller + tasks: + - ansible.builtin.include_tasks: + file: report_result.yml + vars: + test_dir: "{{ logs_dir }}/logging_local" From 99ad79e708be0cff579e26f69e1b9730c423083a Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 19 Aug 2024 21:09:53 +0100 Subject: [PATCH 09/13] remove trailing spaces --- ci/logging_tests_local.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index 2258ee2e..e10cc1a6 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -41,7 +41,7 @@ # ansible.builtin.shell: # cmd: | # chmod 600 "{{ id_rsa_path }}" -# changed_when: false +# changed_when: false - name: "Verify logging projects, endpoints, credentials, nodes, pods, services, manifests and subscriptions" hosts: controller @@ -54,7 +54,7 @@ proj_test_id: "RHOSO-12668" proj_list: - openshift-openstack-infra - - openshift + - openshift - openstack-operators - openshift-logging endpoint_test_id: "RHOSO-12682" @@ -90,7 +90,7 @@ - crc pod_test_id: "RHOS0-12672" - pod_status_str: "Running" + pod_status_str: "Running" pod_nspace: openstack-operators pod_list: - telemetry-operator-controller-manager From e4bf12af7cb799478732761d3bb05a82799f80b3 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 20 Aug 2024 16:59:01 +0100 Subject: [PATCH 10/13] [zuul][logging] Update var names for the telemetry_logging role --- ci/logging_tests_computes.yml | 4 ++-- roles/telemetry_logging/tasks/main.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml index 6f33ef61..538655b2 100644 --- a/ci/logging_tests_computes.yml +++ b/ci/logging_tests_computes.yml @@ -45,8 +45,8 @@ KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" vars: - identifiers_test_id: "RHOSO-12681" - identifiers_list: + journal_test_id: "RHOSO-12681" + journal_list: - ceilometer_agent_compute - nova_compute tasks: diff --git a/roles/telemetry_logging/tasks/main.yml b/roles/telemetry_logging/tasks/main.yml index 2d43bcae..1e67024e 100644 --- a/roles/telemetry_logging/tasks/main.yml +++ b/roles/telemetry_logging/tasks/main.yml @@ -2,4 +2,4 @@ - name: "Verify journal entries - {{ journal_test_id }}" when: journal_list is defined ansible.builtin.include_tasks: "journal_tests.yml" - loop: "{{ identifiers_list }}" + loop: "{{ journal_list }}" From 0e4cd992e75cb3d112d06f033fc89540fb601fff Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 21 Aug 2024 15:31:33 +0100 Subject: [PATCH 11/13] debug: add print statements to custom_logger --- callback_plugins/custom_logger.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/callback_plugins/custom_logger.py b/callback_plugins/custom_logger.py index 63ecfcf0..a7b425a7 100644 --- a/callback_plugins/custom_logger.py +++ b/callback_plugins/custom_logger.py @@ -37,7 +37,7 @@ def __init__(self): self.output_dir = os.path.expanduser("~/") self.results = {} - def playbook_on_stats(self, stats): + def v2_playbook_on_stats(self, stats): #Log results for each host hosts= stats.processed for host in hosts: @@ -59,6 +59,7 @@ def log_task_result(self, host, result, task_name): # Gather the result data to be used in the summary log. if host not in self.results: + print("host was not in results, adding a new value to self.results") self.results[host] = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_task_names':[], 'ok_task_names':[] } if result == 'failed': self.results[host]['failed_task_names'].append(task_name) @@ -69,6 +70,7 @@ def log_task_result(self, host, result, task_name): def log_summary_results(self, host): # The issue is presenting itself in this method file_path = os.path.join(self.output_dir, f"summary_results.log") + print("Writing results to %s" % file_path) # temperorily add more detail to the output print(self.results) From 6eeb1e57ccdbb27adb4d202647cec265bca28acc Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 21 Aug 2024 15:47:26 +0100 Subject: [PATCH 12/13] add gather_facts: true so report result knos about logs_dir --- ci/logging_tests_local.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml index e10cc1a6..b7e37512 100644 --- a/ci/logging_tests_local.yml +++ b/ci/logging_tests_local.yml @@ -280,6 +280,7 @@ - name: Include report result hosts: controller + gather_facts: true tasks: - ansible.builtin.include_tasks: file: report_result.yml From ed77a76556d348befdfb0f85e4a082f5f52c4d95 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 26 Aug 2024 20:06:58 +0100 Subject: [PATCH 13/13] Find the report file --- ci/report_result.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ci/report_result.yml b/ci/report_result.yml index 3375b9a7..3929d2bf 100644 --- a/ci/report_result.yml +++ b/ci/report_result.yml @@ -1,9 +1,18 @@ --- +- name: find the test run results + ansible.builtin.shell: + cmd: | + find ~ -name *.out + register: file + +- ansible.builtin.debug: + var: file.stdout + - name: Move callback_plugin result to log dir ansible.builtin.shell: cmd: | cp ~/test_run_result.out {{ test_dir }}/test_run_result.log - # cp ~/summary_results.log {{ autoscaling_logs_dir }}/summary_results.log + cp ~/summary_results.log {{ autoscaling_logs_dir }}/summary_results.log - name: Grep the number of failed tasks ansible.builtin.shell: