[zuul] Add logging test job

Add a job that runs the telemetry_logging and common roles against the
openstack cloud.
Separate plays target compute and control nodes

.zuul.yaml

@@ -12,7 +12,7 @@
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml"
     roles:
       - zuul: github.com/openstack-k8s-operators/ci-framework
-    required-projects:
+    required-projects: &required_projects
       - name: github.com/infrawatch/service-telemetry-operator
       - name: openstack-k8s-operators/ci-framework
         override-checkout: main
@@ -35,6 +35,21 @@
       - README*
       - .*/*.md
 
+- job:
+    name: functional-logging-tests-osp18
+    parent: telemetry-operator-multinode-logging
+    description: |
+      Run the logging functional test on osp18+patched version
+    vars:
+      cifmw_extras:
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-logging.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-logging-test.yml"
+    roles:
+      - zuul: github.com/openstack-k8s-operators/ci-framework
+    required-projects: *required_projects
+    irrelevant-files: *irrelevant_files
+
 - job:
     name: feature-verification-tests-noop
     parent: noop
@@ -45,7 +60,6 @@
       - ci/noop.yml
     files: *irrelevant_files
 
-
 - project:
     name: infrawatch/feature-verification-tests
     github-check:
@@ -54,3 +68,4 @@
         - openstack-k8s-operators-content-provider:
             override-checkout: main
         - functional-tests-on-osp18
+        - functional-logging-tests-osp18

ci/logging_tests_computes.yml

@@ -0,0 +1,41 @@
+- name: Prepare Logging Tests
+  hosts: computes
+  gather_facts: false
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+
+
+- name: "Verify ctlplane logging containers and files"
+  hosts: computes
+  gather_facts: no
+  vars:
+    container_test_id: "RHOSO-12753"
+    container_list:
+      - ceilometer_agent_compute
+      - ceilometer_agent_ipmi
+      - node_exporter
+
+    file_test_id: "RHOSO-12754"
+    file_list:
+      - /etc/rsyslog.d/10-telemetry.conf
+
+  tasks:
+    - name: "Run file and container tests"
+      ansible.builtin.import_role: 
+        name: common
+
+
+- name: "Verify logging journalctl identifiers" 
+  hosts: computes
+  gather_facts: no
+  vars:
+    identifiers_test_id: "RHOSO-12681"
+    identifiers_list:
+      - ceilometer_agent_compute
+      - nova_compute
+
+  tasks:
+    - name: "Verify journalctl logging identifiers" 
+      ansible.builtin.import_role:
+        name: telemetry_logging

ci/logging_tests_local.yml

@@ -0,0 +1,248 @@
+- name: Prepare Logging Tests
+  hosts: controller
+  gather_facts: false
+  vars:
+    id_rsa_path:  "{{ playbook_dir }}/id_rsa.ctlplane"
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+
+  tasks:
+    - name: "Log into OCP"
+      ansible.builtin.shell:
+        cmd: |
+          oc login -u kubeadmin -p "12345678" https://api.crc.testing:6443
+      failed_when: false
+      changed_when: false
+
+    - name: "Create id rsa file for ctlplane access"
+      ansible.builtin.shell:
+        cmd: |
+          oc extract -n openstack secrets/dataplane-ansible-ssh-private-key-secret --to=- | grep -v "ssh\-rsa" > "{{ id_rsa_path }}"
+      register: rsa_results
+      failed_when: rsa_results.rc != 0
+      changed_when: false
+
+    - name: "Change files permissions"
+      ansible.builtin.shell:
+        cmd: |
+          chmod 600 "{{ id_rsa_path }}"
+      changed_when: false  
+
+- name: "Verify logging projects, endpoints, credentials, nodes, pods, services, manifests and subscriptions"
+  hosts: controller
+  gather_facts: no
+  vars:
+    proj_test_id: "RHOSO-12668"
+    proj_list:
+      - openshift-openstack-infra
+      - openshift       
+      - openstack-operators
+      - openshift-logging
+
+
+    endpoint_test_id: "RHOSO-12682"
+    endpoint_list:
+      - [nova,compute,public]
+      - [nova,compute,internal]
+      - [placement,placement,public]
+      - [placement,placement,internal]
+      - [swift,object-store,public]
+      - [swift,object-store,internal]
+      - [cinderv3,volumev3,public]
+      - [cinderv3,volumev3,internal]
+      - [barbican,key-manager,public]
+      - [barbican,key-manager,internal]
+      - [keystone,identity,public]
+      - [keystone,identity,internal]
+      - [glance,image,public]
+      - [glance,image,internal]
+      - [neutron,network,public]
+      - [neutron,network,internal]
+
+
+    cred_test_id: "RHOSO-12670"
+    cred_list:
+      - alertingrules.loki.grafana.com
+      - lokistacks.loki.grafana.com
+      - recordingrules.loki.grafana.com
+      - rulerconfigs.loki.grafana.com
+
+
+    node_test_id: "RHOSO-12671"
+    node_list:
+      - edpm-compute-0
+      - edpm-compute-1
+      - crc
+
+
+    pod_test_id: "RHOS0-12672"
+    pod_status_str: "Running"      
+    pod_nspace: openstack-operators
+    pod_list:
+      - telemetry-operator-controller-manager
+      - dataplane-operator-controller-manager
+
+
+    service_test_id: "RHOSO-12675"
+    service_nspace: openshift-logging
+    service_list:
+      - cluster-logging-operator-metrics
+      - logging-loki-compactor-grpc
+      - logging-loki-compactor-http
+      - logging-loki-distributor-grpc
+      - logging-loki-distributor-http
+      - logging-loki-gateway-http
+      - logging-loki-gossip-ring
+      - logging-loki-index-gateway-grpc
+      - logging-loki-index-gateway-http
+      - logging-loki-ingester-grpc
+      - logging-loki-ingester-http
+      - logging-loki-querier-grpc
+      - logging-loki-querier-http
+      - logging-loki-query-frontend-grpc
+      - logging-loki-query-frontend-http
+      - logging-view-plugin
+      - openstack-logging
+
+
+    manifest_test_id: "RHOSO-12677"
+    manifest_list:
+      - "loki-operator 2"
+      - "loki-helm-operator 1"
+
+
+    subscription_test_id: "RHOSO-12678"
+    subscription_nspace: openshift-operators-redhat
+    subscription_list:
+      - loki-operator
+
+  tasks:
+    - name: "Run logging project, endpoint, credential, node, pod, service, manifest and subscription tests"
+      ansible.builtin.import_role:
+        name: common
+
+
+
+- name: "Verify logging pods are running in openstack"
+  hosts: controller
+  gather_facts: no
+  vars:
+    pod_test_id: "RHOSO-12752"
+    pod_status_str: "Running"
+    pod_nspace: openstack
+    pod_list:
+      - openstackclient
+
+  tasks:
+    - name: "Verify Running Pods"
+      ansible.builtin.import_role:
+        name: common
+
+
+- name: "Verify logging pods are running in openshift-operators-redhat"
+  hosts: controller
+  gather_facts: no
+  vars:
+    pod_test_id: "RHOSO-12673"
+    pod_status_str: "Running"
+    pod_nspace: openshift-operators-redhat
+    pod_list:
+      - loki-operator-controller-manager
+
+  tasks:
+    - name: "Verify Pods running"
+      ansible.builtin.import_role:
+        name: common
+
+
+- name: "Verify logging pods are running in openshift-logging"
+  hosts: controller
+  gather_facts: no
+  vars:
+    pod_test_id: "RHOSO-12676"
+    pod_status_str: "Running"
+    pod_nspace: openshift-logging
+    pod_list:
+      - cluster-logging-operator
+      - collector
+      - logging-loki-compactor
+      - logging-loki-distributor
+      - logging-loki-gateway
+      - logging-loki-index-gateway
+      - logging-loki-ingester
+      - logging-loki-querier
+      - logging-loki-query-frontend
+      - logging-view-plugin
+
+  ### see JIRA LOG-5431 if pods not running
+  tasks:
+    - name: "Verify Pods running" 
+      ansible.builtin.import_role:
+        name: common
+
+
+- name: "Verify logging pods are running in minio-dev"
+  hosts: controller
+  gather_facts: no
+  vars:
+    pod_test_id: "RHOSO-12674"
+    pod_status_str: "Running"
+    pod_nspace: minio-dev
+    pod_list:
+      - minio
+
+  tasks:
+    - name: "Run pod running tests"
+      ansible.builtin.import_role:
+        name: common
+
+
+- name: "Verify logging pods have complete status in openstack"
+  hosts: controller
+  gather_facts: no
+  vars:
+    pod_test_id: "RHOSO-12679"
+    pod_nspace: openstack
+    pod_status_str: "Completed"
+    pod_list:
+      - bootstrap-edpm-deployment-logging
+      - configure-network-edpm-deployment-logging
+      - configure-os-edpm-deployment-logging
+      - download-cache-edpm-deployment-logging
+      - install-certs-edpm-deployment-logging
+      - install-os-edpm-deployment-logging
+      - libvirt-edpm-deployment-logging
+      - logging-edpm-deployment-logging-openstack-edpm
+      - neutron-metadata-edpm-deployment-logging
+      - nova-custom-edpm-deployment
+      - ovn-edpm-deployment-logging
+      - reboot-os-edpm-deployment-logging
+      - repo-setup-edpm-deployment-logging
+      - run-os-edpm-deployment-logging
+      - ssh-known-hosts-edpm-deployment-logging
+      - telemetry-edpm-deployment-logging
+      - validate-network-edpm-deployment-logging
+
+  tasks:
+    - name: "Run pods completed tests"
+      ansible.builtin.import_role:
+        name: common
+
+
+- name: "Verify logging services are running in openstack"
+  hosts: controller
+  gather_facts: no
+  vars:
+    service_test_id: "RHOSO-12749"
+    service_nspace: openstack
+    service_list:
+      - nova-internal
+      - nova-metadata-internal
+      - nova-novncproxy-cell1-public
+      - nova-public
+
+  tasks:
+    - name: "Run Services tests" 
+      ansible.builtin.import_role:
+        name: common

ci/run_playbooks_logging.yml

@@ -0,0 +1,57 @@
+---
+- name: "Run logging functional test playbooks"
+  hosts:  "{{ cifmw_target_hook_host | default('localhost')  }}"
+  gather_facts: true
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  vars_files:
+    - vars/common.yml
+  tasks:
+    - name: "Create log dir"
+      ansible.builtin.file:
+        path: "{{ logs_dir }}"
+        state: directory
+        mode: "0755"
+
+    - name: "Set dynamic KUBECONFIG and PATH vars"
+      copy:
+        content: |
+          cifmw_openshift_kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+          cifmw_path: "{{ cifmw_path }}"
+          vars_dir: "{{ fvt_dir }}/ci/vars"
+        dest: "{{ env_vars_file }}"
+
+    - name: Include vars from the extra_vars files
+      ansible.builtin.include_vars:
+        dir: "{{ cifmw_basedir }}/artifacts/parameters"
+
+    - name: Run telemetry-logging tests on OSP18
+      block:
+        - name: "Run local logging tests"
+          ansible.builtin.shell:
+            cmd: |
+              ANSIBLE_CONFIG=ci/ansible.cfg ansible-playbook -v  -e @"{{ env_vars_file }}" ci/logging_tests_computes.yml
+            chdir: "{{ fvt_dir }}"
+          register: output
+
+        - name: Save ansible output to a file
+          ansible.builtin.copy:
+            content: "{{ output.stdout }}"
+            dest: "{{ logs_dir }}/ansible_run.log"
+
+        - name: "Run cltplane logging tests"
+          ansible.builtin.shell:
+            cmd: |
+              ANSIBLE_CONFIG=ci/ansible.cfg ansible-playbook -v  -e @"{{ env_vars_file }}" ci/logging_tests_local.yml
+            chdir: "{{ fvt_dir }}"
+          register: output2
+
+        - name: Save ansible output to a file
+          ansible.builtin.lineinfile:
+            line: "{{ output2.stdout }}"
+            path: "{{ logs_dir }}/ansible_run.log"
+
+    - name: Include report result 
+      ansible.builtin.include_tasks:
+        file: report_result.yml

ci/vars-logging-test.yml

@@ -0,0 +1,5 @@
+---
+# temp: skip os-net-setup
+post_deploy_run_logging_functional_test:
+  source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_playbooks_logging.yml"
+  type: playbook