diff --git a/.zuul.yaml b/.zuul.yaml index 62c2773d..9109f1a1 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -12,7 +12,7 @@ - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml" roles: - zuul: github.com/openstack-k8s-operators/ci-framework - required-projects: + required-projects: &required_projects - name: github.com/infrawatch/service-telemetry-operator - name: openstack-k8s-operators/ci-framework override-checkout: main @@ -35,6 +35,21 @@ - README* - .*/*.md +- job: + name: functional-logging-tests-osp18 + parent: telemetry-operator-multinode-logging + description: | + Run the logging functional test on osp18+patched version + vars: + cifmw_extras: + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-logging.yml" + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-logging-test.yml" + roles: + - zuul: github.com/openstack-k8s-operators/ci-framework + required-projects: *required_projects + irrelevant-files: *irrelevant_files + - job: name: feature-verification-tests-noop parent: noop @@ -45,12 +60,12 @@ - ci/noop.yml files: *irrelevant_files - - project: name: infrawatch/feature-verification-tests github-check: jobs: - - feature-verification-tests-noop + #- feature-verification-tests-noop - openstack-k8s-operators-content-provider: override-checkout: main - - functional-tests-on-osp18 + #- functional-tests-on-osp18 + - functional-logging-tests-osp18 diff --git a/callback_plugins/custom_logger.py b/callback_plugins/custom_logger.py index 8e1c109e..a7b425a7 100644 --- a/callback_plugins/custom_logger.py +++ b/callback_plugins/custom_logger.py @@ -4,6 +4,9 @@ import os import re + +from ansible.errors import AnsibleCallbackError +from ansible.errors import AnsibleError from ansible.plugins.callback import CallbackBase DOCUMENTATION = ''' @@ -34,13 +37,14 @@ def __init__(self): self.output_dir = os.path.expanduser("~/") self.results = {} - def playbook_on_stats(self, stats): + def v2_playbook_on_stats(self, stats): #Log results for each host hosts= stats.processed for host in hosts: self.log_summary_results(host) def log_task_result(self, host, result, task_name): + print("logging task result: %s - %s - %s" % (host, result, task_name)) # test_run_result.out only interested in the test tasks, not setup or debug. if "RHELOSP" in task_name or "RHOSO" in task_name: if "RHELOSP" in task_name: @@ -55,6 +59,7 @@ def log_task_result(self, host, result, task_name): # Gather the result data to be used in the summary log. if host not in self.results: + print("host was not in results, adding a new value to self.results") self.results[host] = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_task_names':[], 'ok_task_names':[] } if result == 'failed': self.results[host]['failed_task_names'].append(task_name) @@ -63,18 +68,36 @@ def log_task_result(self, host, result, task_name): self.results[host][result] += 1 def log_summary_results(self, host): + # The issue is presenting itself in this method file_path = os.path.join(self.output_dir, f"summary_results.log") + print("Writing results to %s" % file_path) + # temperorily add more detail to the output + print(self.results) + with open(file_path, 'w') as f: f.write(f"Host: {host}\n") - f.write(f"Tasks Succeeded: {self.results[host]['passed']}\n") - f.write(f"Tasks Failed: {self.results[host]['failed']}\n") - f.write(f"Tasks Skipped: {self.results[host]['skipped']}\n") - f.write("Failed Tasks:\n") - for task_name in self.results[host]['failed_task_names']: - f.write(f" - {task_name}\n") - f.write("Succeeded Tasks:\n") - for task_name in self.results[host]['ok_task_names']: - f.write(f" - {task_name}\n") + # This is the failed issue. + # Why is it happening? + # Unknown. But does it happen for the other values. + # It is happening for compute-0 + # I need to see the partial log file, so this should NOT cause a failure. + # try/catch ANY exception + # It could be a nice upgrade to use jinja2 template to render this file. + # IS the issue that there were 'f's preceeding the strings? + try: + f.write("Tasks Succeeded: {self.results[host]['passed']}\n") + f.write("Tasks Failed: {self.results[host]['failed']}\n") + f.write("Tasks Skipped: {self.results[host]['skipped']}\n") + f.write("Failed Tasks:\n") + for task_name in self.results[host]['failed_task_names']: + f.write(f" - {task_name}\n") + f.write("Succeeded Tasks:\n") + for task_name in self.results[host]['ok_task_names']: + f.write(f" - {task_name}\n") + except AnsibleError as e: + # this is probably an AnsibleCallbackError + print("Ooops, there was an error") + print(e) def v2_runner_on_ok(self, result): host = result._host.get_name() @@ -89,4 +112,4 @@ def v2_runner_on_failed(self, result, ignore_errors=False): def v2_runner_on_skipped(self, result): host = result._host.get_name() task_name = result._task.get_name() - self.log_task_result(host, 'skipped', task_name) \ No newline at end of file + self.log_task_result(host, 'skipped', task_name) diff --git a/ci/ansible.cfg b/ci/ansible.cfg index 3c9d90c0..fb76f2b9 100644 --- a/ci/ansible.cfg +++ b/ci/ansible.cfg @@ -3,3 +3,6 @@ callbacks_enabled = custom_logger callback_plugins = ../callback_plugins # additional paths to search for roles roles_path = ../roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.ansible/roles:../../ci-framework/roles + +# temp update to get more information from the playbooks running functional tests +verbosity = 5 diff --git a/ci/logging_tests_computes.yml b/ci/logging_tests_computes.yml new file mode 100644 index 00000000..538655b2 --- /dev/null +++ b/ci/logging_tests_computes.yml @@ -0,0 +1,63 @@ +--- +- name: Prepare Logging Tests + hosts: computes + gather_facts: false + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars_files: + - vars/common.yml + tasks: + - name: Create log dir + ansible.builtin.file: + path: "{{ logs_dir }}/logging_compute" + state: directory + mode: "0755" + +- name: "Verify ctlplane logging containers and files" + hosts: computes + gather_facts: no + ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars: + container_test_id: "RHOSO-12753" + container_list: + - ceilometer_agent_compute + - ceilometer_agent_ipmi + - node_exporter + + file_test_id: "RHOSO-12754" + file_list: + - /etc/rsyslog.d/10-telemetry.conf + tasks: + - name: "Run file and container tests" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging journalctl identifiers" + hosts: computes + gather_facts: no + ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars: + journal_test_id: "RHOSO-12681" + journal_list: + - ceilometer_agent_compute + - nova_compute + tasks: + - name: "Verify journalctl logging identifiers" + ansible.builtin.import_role: + name: telemetry_logging + +- name: Include report result + hosts: computes + tasks: + - ansible.builtin.include_tasks: + file: report_result.yml + vars: + test_dir: "{{ logs_dir }}/logging_local" diff --git a/ci/logging_tests_local.yml b/ci/logging_tests_local.yml new file mode 100644 index 00000000..b7e37512 --- /dev/null +++ b/ci/logging_tests_local.yml @@ -0,0 +1,288 @@ +--- +- name: Prepare Logging Tests + hosts: controller + gather_facts: true + ignore_errors: false +# vars: +# id_rsa_path: "{{ playbook_dir }}/id_rsa.ctlplane" +# environment: +# KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" +# PATH: "{{ cifmw_path }}" + vars_files: + - vars/common.yml + tasks: + - name: Create log dir + ansible.builtin.file: + path: "{{ logs_dir }}/logging_local" + state: directory + mode: "0755" + +# # This needs to be updated; what do we use elsewhere? +# # Elsewhere, we assume that we're already logged in +# - name: "Log into OCP" +# ansible.builtin.shell: +# cmd: | +# oc login -u kubeadmin -p "12345678" https://api.crc.testing:6443 +# failed_when: false +# changed_when: false +# +# # I don't think we need this, Zuul __should__ have taken care of this already +# - name: "Create id rsa file for ctlplane access" +# ansible.builtin.shell: +# cmd: | +# oc extract -n openstack secrets/dataplane-ansible-ssh-private-key-secret --to=- | grep -v "ssh\-rsa" > "{{ id_rsa_path }}" +# register: rsa_results +# failed_when: rsa_results.rc != 0 +# changed_when: false +# +# # (efoley): Once again, I don't think this is necessary; if it's not needed for zuul, because it's already done, but is needed before running this on other systems, then the README should be updated to reflect this; +# # A troubleshooting section can be added too so users can identify when they are missing these configuration steps +# - name: "Change files permissions" +# ansible.builtin.shell: +# cmd: | +# chmod 600 "{{ id_rsa_path }}" +# changed_when: false + +- name: "Verify logging projects, endpoints, credentials, nodes, pods, services, manifests and subscriptions" + hosts: controller + gather_facts: no + ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars: + proj_test_id: "RHOSO-12668" + proj_list: + - openshift-openstack-infra + - openshift + - openstack-operators + - openshift-logging + endpoint_test_id: "RHOSO-12682" + endpoint_list: + - [nova,compute,public] + - [nova,compute,internal] + - [placement,placement,public] + - [placement,placement,internal] + - [swift,object-store,public] + - [swift,object-store,internal] + - [cinderv3,volumev3,public] + - [cinderv3,volumev3,internal] + - [barbican,key-manager,public] + - [barbican,key-manager,internal] + - [keystone,identity,public] + - [keystone,identity,internal] + - [glance,image,public] + - [glance,image,internal] + - [neutron,network,public] + - [neutron,network,internal] + + cred_test_id: "RHOSO-12670" + cred_list: + - alertingrules.loki.grafana.com + - lokistacks.loki.grafana.com + - recordingrules.loki.grafana.com + - rulerconfigs.loki.grafana.com + + node_test_id: "RHOSO-12671" + node_list: + - edpm-compute-0 + - edpm-compute-1 + - crc + + pod_test_id: "RHOS0-12672" + pod_status_str: "Running" + pod_nspace: openstack-operators + pod_list: + - telemetry-operator-controller-manager + - dataplane-operator-controller-manager + + service_test_id: "RHOSO-12675" + service_nspace: openshift-logging + service_list: + - cluster-logging-operator-metrics + - logging-loki-compactor-grpc + - logging-loki-compactor-http + - logging-loki-distributor-grpc + - logging-loki-distributor-http + - logging-loki-gateway-http + - logging-loki-gossip-ring + - logging-loki-index-gateway-grpc + - logging-loki-index-gateway-http + - logging-loki-ingester-grpc + - logging-loki-ingester-http + - logging-loki-querier-grpc + - logging-loki-querier-http + - logging-loki-query-frontend-grpc + - logging-loki-query-frontend-http + - logging-view-plugin + - openstack-logging + manifest_test_id: "RHOSO-12677" + manifest_list: + - "loki-operator 2" + - "loki-helm-operator 1" + subscription_test_id: "RHOSO-12678" + subscription_nspace: openshift-operators-redhat + subscription_list: + - loki-operator + tasks: + - name: "Run logging project, endpoint, credential, node, pod, service, manifest and subscription tests" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods are running in openstack" + hosts: controller + gather_facts: no + ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars: + pod_test_id: "RHOSO-12752" + pod_status_str: "Running" + pod_nspace: openstack + pod_list: + - openstackclient + tasks: + - name: "Verify Running Pods" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods are running in openshift-operators-redhat" + hosts: controller + gather_facts: no + ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars: + pod_test_id: "RHOSO-12673" + pod_status_str: "Running" + pod_nspace: openshift-operators-redhat + pod_list: + - loki-operator-controller-manager + + tasks: + - name: "Verify Pods running" + ansible.builtin.import_role: + name: common + # vars can be passed here, so we can use the same play for this and the next play. + # vars: + # - + +- name: "Verify logging pods are running in openshift-logging" + hosts: controller + gather_facts: no + ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars: + pod_test_id: "RHOSO-12676" + pod_status_str: "Running" + pod_nspace: openshift-logging + pod_list: + - cluster-logging-operator + - collector + - logging-loki-compactor + - logging-loki-distributor + - logging-loki-gateway + - logging-loki-index-gateway + - logging-loki-ingester + - logging-loki-querier + - logging-loki-query-frontend + - logging-view-plugin + + ### see JIRA LOG-5431 if pods not running + tasks: + - name: "Verify Pods running" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods are running in minio-dev" + hosts: controller + gather_facts: no + ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + + + vars: + pod_test_id: "RHOSO-12674" + pod_status_str: "Running" + pod_nspace: minio-dev + pod_list: + - minio + + tasks: + - name: "Run pod running tests" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging pods have complete status in openstack" + hosts: controller + gather_facts: no + ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars: + pod_test_id: "RHOSO-12679" + pod_nspace: openstack + pod_status_str: "Completed" + pod_list: + - bootstrap-edpm-deployment-logging + - configure-network-edpm-deployment-logging + - configure-os-edpm-deployment-logging + - download-cache-edpm-deployment-logging + - install-certs-edpm-deployment-logging + - install-os-edpm-deployment-logging + - libvirt-edpm-deployment-logging + - logging-edpm-deployment-logging-openstack-edpm + - neutron-metadata-edpm-deployment-logging + - nova-custom-edpm-deployment + - ovn-edpm-deployment-logging + - reboot-os-edpm-deployment-logging + - repo-setup-edpm-deployment-logging + - run-os-edpm-deployment-logging + - ssh-known-hosts-edpm-deployment-logging + - telemetry-edpm-deployment-logging + - validate-network-edpm-deployment-logging + tasks: + - name: "Run pods completed tests" + ansible.builtin.import_role: + name: common + + +- name: "Verify logging services are running in openstack" + hosts: controller + gather_facts: no + ignore_errors: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars: + service_test_id: "RHOSO-12749" + service_nspace: openstack + service_list: + - nova-internal + - nova-metadata-internal + - nova-novncproxy-cell1-public + - nova-public + tasks: + - name: "Run Services tests" + ansible.builtin.import_role: + name: common + +- name: Include report result + hosts: controller + gather_facts: true + tasks: + - ansible.builtin.include_tasks: + file: report_result.yml + vars: + test_dir: "{{ logs_dir }}/logging_local" diff --git a/ci/report_result.yml b/ci/report_result.yml index 3375b9a7..3929d2bf 100644 --- a/ci/report_result.yml +++ b/ci/report_result.yml @@ -1,9 +1,18 @@ --- +- name: find the test run results + ansible.builtin.shell: + cmd: | + find ~ -name *.out + register: file + +- ansible.builtin.debug: + var: file.stdout + - name: Move callback_plugin result to log dir ansible.builtin.shell: cmd: | cp ~/test_run_result.out {{ test_dir }}/test_run_result.log - # cp ~/summary_results.log {{ autoscaling_logs_dir }}/summary_results.log + cp ~/summary_results.log {{ autoscaling_logs_dir }}/summary_results.log - name: Grep the number of failed tasks ansible.builtin.shell: diff --git a/ci/run_playbooks_logging.yml b/ci/run_playbooks_logging.yml new file mode 100644 index 00000000..35e0a0cb --- /dev/null +++ b/ci/run_playbooks_logging.yml @@ -0,0 +1,57 @@ +--- +- name: "Run logging functional test playbooks" + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: true + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars_files: + - vars/common.yml + tasks: + - name: "Create log dir" + ansible.builtin.file: + path: "{{ logs_dir }}" + state: directory + mode: "0755" + + - name: "Set dynamic KUBECONFIG and PATH vars" + copy: + content: | + cifmw_openshift_kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + cifmw_path: "{{ cifmw_path }}" + vars_dir: "{{ fvt_dir }}/ci/vars" + dest: "{{ env_vars_file }}" + + - name: Include vars from the extra_vars files + ansible.builtin.include_vars: + dir: "{{ cifmw_basedir }}/artifacts/parameters" + + - name: Run telemetry-logging tests on OSP18 + block: + - name: "Run local logging tests" + ansible.builtin.shell: + cmd: | + ANSIBLE_CONFIG=ci/ansible.cfg ansible-playbook -v -e @"{{ env_vars_file }}" ci/logging_tests_computes.yml + chdir: "{{ fvt_dir }}" + register: output + + - name: Save ansible output to a file + ansible.builtin.copy: + content: "{{ output.stdout }}" + dest: "{{ logs_dir }}/ansible_run.log" + + - name: "Run cltplane logging tests" + ansible.builtin.shell: + cmd: | + ANSIBLE_CONFIG=ci/ansible.cfg ansible-playbook -v -e @"{{ env_vars_file }}" ci/logging_tests_local.yml + chdir: "{{ fvt_dir }}" + register: output2 + + - name: Save ansible output to a file + ansible.builtin.lineinfile: + line: "{{ output2.stdout }}" + path: "{{ logs_dir }}/ansible_run.log" + + - name: Include report result + ansible.builtin.include_tasks: + file: report_result.yml diff --git a/ci/vars-logging-test.yml b/ci/vars-logging-test.yml new file mode 100644 index 00000000..f9c4278b --- /dev/null +++ b/ci/vars-logging-test.yml @@ -0,0 +1,16 @@ +--- +#post_deploy_run_logging_functional_test: +# source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_playbooks_logging.yml" +# config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" +# type: playbook +# pre_tests or post_tests are also options for when the FVT roles are run +post_deploy_fvt_logging_01_computes: + source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_computes.yml" + config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" + type: playbook + +post_deploy_fvt_logging_00_local: + source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_local.yml" + config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg" + type: playbook + diff --git a/roles/telemetry_logging/tasks/main.yml b/roles/telemetry_logging/tasks/main.yml index 2d43bcae..1e67024e 100644 --- a/roles/telemetry_logging/tasks/main.yml +++ b/roles/telemetry_logging/tasks/main.yml @@ -2,4 +2,4 @@ - name: "Verify journal entries - {{ journal_test_id }}" when: journal_list is defined ansible.builtin.include_tasks: "journal_tests.yml" - loop: "{{ identifiers_list }}" + loop: "{{ journal_list }}"