diff --git a/.azure-pipelines/pr_test_scripts.yaml b/.azure-pipelines/pr_test_scripts.yaml index 75df297865..8d8271e515 100644 --- a/.azure-pipelines/pr_test_scripts.yaml +++ b/.azure-pipelines/pr_test_scripts.yaml @@ -1,10 +1,16 @@ t0: + - acl/custom_acl_table/test_custom_acl_table.py + - acl/null_route/test_null_route_helper.py + - acl/test_acl.py + - acl/test_acl_outer_vlan.py + - acl/test_stress_acl.py - arp/test_arp_extended.py - - arp/test_neighbor_mac.py - arp/test_neighbor_mac_noptf.py + - arp/test_stress_arp.py - arp/test_tagged_arp.py - # Due to the auto start fails on container restapi, comment this case temporarily -# - autorestart/test_container_autorestart.py + - arp/test_wr_arp.py + - arp/test_unknown_mac.py + - autorestart/test_container_autorestart.py - bgp/test_bgp_dual_asn.py - bgp/test_bgp_fact.py - bgp/test_bgp_gr_helper.py @@ -13,6 +19,7 @@ t0: - bgp/test_bgp_speaker.py - bgp/test_bgp_update_timer.py - bgp/test_bgpmon.py + - bgp/test_bgp_session.py - cacl/test_cacl_application.py - cacl/test_cacl_function.py - console/test_console_availability.py @@ -24,21 +31,36 @@ t0: - container_hardening/test_container_hardening.py - database/test_db_config.py - database/test_db_scripts.py + - decap/test_decap.py - dhcp_relay/test_dhcp_pkt_recv.py - dhcp_relay/test_dhcp_relay.py + - dhcp_relay/test_dhcp_relay_stress.py - dhcp_relay/test_dhcpv6_relay.py + - disk/test_disk_exhaustion.py - dns/static_dns/test_static_dns.py - dns/test_dns_resolv_conf.py - dualtor/test_orch_stress.py + - dualtor/test_orchagent_active_tor_downstream.py + - dualtor/test_orchagent_mac_move.py + - dualtor/test_orchagent_standby_tor_downstream.py + - dualtor/test_standby_tor_upstream_mux_toggle.py - dualtor_mgmt/test_server_failure.py - dualtor_mgmt/test_toggle_mux.py - dut_console/test_console_baud_rate.py - dut_console/test_escape_character.py - dut_console/test_idle_timeout.py + - ecmp/inner_hashing/test_inner_hashing_lag.py + - ecmp/inner_hashing/test_inner_hashing.py + - ecmp/inner_hashing/test_wr_inner_hashing_lag.py + - ecmp/inner_hashing/test_wr_inner_hashing.py + - everflow/test_everflow_ipv6.py + - everflow/test_everflow_per_interface.py + - everflow/test_everflow_testbed.py - fdb/test_fdb.py - fdb/test_fdb_flush.py - fdb/test_fdb_mac_expire.py - fdb/test_fdb_mac_move.py + - fib/test_fib.py - generic_config_updater/test_aaa.py - generic_config_updater/test_bgp_speaker.py - generic_config_updater/test_bgpl.py @@ -56,14 +78,16 @@ t0: - generic_config_updater/test_portchannel_interface.py - generic_config_updater/test_syslog.py - generic_config_updater/test_vlan_interface.py - # Temporarily remove these cases because of PR https://github.com/sonic-net/sonic-buildimage/pull/16957 - # - gnmi/test_gnmi.py - # - gnmi/test_gnmi_appldb.py - # - gnmi/test_gnmi_configdb.py + - gnmi/test_gnmi.py + - gnmi/test_gnmi_appldb.py + - gnmi/test_gnmi_configdb.py + - golden_config_infra/test_config_reload_with_rendered_golden_config.py - http/test_http_copy.py - iface_loopback_action/test_iface_loopback_action.py - iface_namingmode/test_iface_namingmode.py + - ip/test_ip_packet.py - ipfwd/test_dip_sip.py + - ipfwd/test_dir_bcast.py - lldp/test_lldp.py - log_fidelity/test_bgp_shutdown.py - macsec/test_controlplane.py @@ -76,18 +100,14 @@ t0: - pc/test_lag_2.py - pc/test_po_cleanup.py - pc/test_po_update.py + - pfcwd/test_pfc_config.py - platform_tests/broadcom/test_ser.py - platform_tests/counterpoll/test_counterpoll_watermark.py - platform_tests/fwutil/test_fwutil.py - platform_tests/link_flap/test_cont_link_flap.py - - platform_tests/mellanox/test_check_sfp_eeprom.py - - platform_tests/mellanox/test_check_sfp_presence.py - - platform_tests/mellanox/test_check_sfp_using_ethtool.py - - platform_tests/mellanox/test_check_sysfs.py - - platform_tests/mellanox/test_hw_management_service.py - - platform_tests/mellanox/test_psu_power_threshold.py - - platform_tests/mellanox/test_reboot_cause.py + - platform_tests/sfp/test_show_intf_xcvr.py - platform_tests/sfp/test_sfpshow.py + - platform_tests/sfp/test_sfputil.py - platform_tests/test_kdump.py - platform_tests/test_advanced_reboot.py::test_warm_reboot - platform_tests/test_auto_negotiation.py @@ -95,10 +115,15 @@ t0: # - platform_tests/test_cont_warm_reboot.py - platform_tests/test_cpu_memory_usage.py - platform_tests/test_first_time_boot_password_change/test_first_time_boot_password_change.py + - platform_tests/test_platform_info.py - platform_tests/test_port_toggle.py + - platform_tests/test_power_off_reboot.py + - platform_tests/test_reboot.py - platform_tests/test_secure_upgrade.py - platform_tests/test_sensors.py + - platform_tests/test_sequential_restart.py - platform_tests/test_service_warm_restart.py + - platform_tests/test_xcvr_info_in_db.py - portstat/test_portstat.py - process_monitoring/test_critical_process_monitoring.py - qos/test_buffer.py @@ -106,14 +131,12 @@ t0: - radv/test_radv_restart.py - radv/test_radv_run.py - reset_factory/test_reset_factory.py - - restapi/test_restapi.py - restapi/test_restapi_vxlan_ecmp.py - route/test_default_route.py - route/test_duplicate_route.py - route/test_forced_mgmt_route.py - route/test_route_consistency.py - route/test_route_flap.py - - route/test_route_flow_counter.py - route/test_route_perf.py - route/test_static_route.py - scp/test_scp_copy.py @@ -149,7 +172,7 @@ t0: - tacacs/test_ro_user.py - tacacs/test_rw_user.py - telemetry/test_events.py - - telemetry/test_telemetry.py + - telemetry/test_telemetry_cert_rotation.py - test_features.py - test_interfaces.py - test_procdockerstatsd.py @@ -160,11 +183,49 @@ t0: - vlan/test_vlan.py - vlan/test_vlan_ping.py - vxlan/test_vnet_route_leak.py - - ip/test_mgmt_ipv6_only.py + - vxlan/test_vnet_vxlan.py + - vxlan/test_vxlan_decap.py + - test_pktgen.py + - bgp/test_bgp_peer_shutdown.py + - clock/test_clock.py + - generic_config_updater/test_pfcwd_status.py + - pfcwd/test_pfc_config.py + - platform_tests/link_flap/test_link_flap.py + - platform_tests/test_memory_exhaustion.py + - generic_config_updater/test_pg_headroom_update.py + - sub_port_interfaces/test_show_subinterface.py + - pc/test_lag_member.py + - platform_tests/test_link_down.py + - gnmi/test_gnmi_countersdb.py + - sub_port_interfaces/test_sub_port_l2_forwarding.py + - snmp/test_snmp_psu.py + - platform_tests/cli/test_show_platform.py + - snmp/test_snmp_queue_counters.py + - platform_tests/test_reload_config.py + - generic_config_updater/test_dynamic_acl.py + - sub_port_interfaces/test_sub_port_interfaces.py + - hash/test_generic_hash.py + - span/test_port_mirroring.py + - route/test_route_bgp_ecmp.py + - drop_packets/test_drop_counters.py + - copp/test_copp.py + - crm/test_crm.py + - test_nbr_health.py + - dhcp_server/test_dhcp_server.py + - dhcp_server/test_dhcp_server_multi_vlans.py + - dhcp_server/test_dhcp_server_stress.py + - platform_tests/test_idle_driver.py + - wol/test_wol.py + - dhcp_relay/test_dhcp_pkt_fwd.py + - telemetry/test_telemetry.py + - platform_tests/test_cont_warm_reboot.py t0-2vlans: - dhcp_relay/test_dhcp_relay.py + - dhcp_relay/test_dhcp_relay_stress.py - dhcp_relay/test_dhcpv6_relay.py + - vlan/test_host_vlan.py + - vlan/test_vlan_ping.py t0-sonic: - bgp/test_bgp_fact.py @@ -177,10 +238,26 @@ t0-sonic: - platform_tests/test_advanced_reboot.py::test_warm_reboot dualtor: + - arp/test_arp_dualtor.py - arp/test_arp_extended.py + - dualtor/test_ipinip.py + - dualtor/test_switchover_failure.py + - dualtor/test_tor_ecn.py + - dualtor/test_tunnel_memory_leak.py + - dualtor_io/test_heartbeat_failure.py + - dualtor_io/test_link_drop.py + - dualtor_io/test_link_failure.py + - dualtor_io/test_tor_bgp_failure.py + - dualtor_mgmt/test_grpc_periodical_sync.py + - dualtor_mgmt/test_ingress_drop.py + - dualtor_mgmt/test_server_failure.py + - dualtor_mgmt/test_toggle_mux.py t1-lag: + - acl/test_acl.py + - acl/test_stress_acl.py - arp/test_arpall.py + - arp/test_neighbor_mac.py - arp/test_neighbor_mac_noptf.py - bgp/test_bgp_allow_list.py - bgp/test_bgp_bbr.py @@ -189,14 +266,22 @@ t1-lag: - bgp/test_bgp_multipath_relax.py - bgp/test_bgp_update_timer.py - bgp/test_bgpmon.py + - bgp/test_bgp_session.py - bgp/test_traffic_shift.py - configlet/test_add_rack.py - container_checker/test_container_checker.py + - decap/test_decap.py - dhcp_relay/test_dhcp_pkt_fwd.py + - everflow/test_everflow_ipv6.py + - everflow/test_everflow_per_interface.py + - everflow/test_everflow_testbed.py - fdb/test_fdb_flush.py + - fib/test_fib.py - generic_config_updater/test_mmu_dynamic_threshold_config_update.py + - golden_config_infra/test_config_reload_with_rendered_golden_config.py - http/test_http_copy.py - iface_namingmode/test_iface_namingmode.py + - ip/test_ip_packet.py - ipfwd/test_dip_sip.py - ipfwd/test_mtu.py - lldp/test_lldp.py @@ -204,10 +289,14 @@ t1-lag: - override_config_table/test_override_config_table.py - pc/test_lag_2.py - pc/test_po_update.py + - pfcwd/test_pfc_config.py - platform_tests/test_cpu_memory_usage.py - process_monitoring/test_critical_process_monitoring.py - qos/test_buffer.py + - radv/test_radv_restart.py - route/test_default_route.py + - route/test_route_consistency.py + - route/test_route_flap.py - route/test_route_perf.py - scp/test_scp_copy.py - snmp/test_snmp_cpu.py @@ -219,7 +308,118 @@ t1-lag: - snmp/test_snmp_pfc_counters.py - snmp/test_snmp_queue.py - snmp/test_snmp_v2mib.py + - sub_port_interfaces/test_show_subinterface.py - test_interfaces.py + - test_pktgen.py + - vxlan/test_vxlan_crm.py + - vxlan/test_vxlan_ecmp.py + - autorestart/test_container_autorestart.py + - bgp/test_bgp_gr_helper.py + - bgp/test_bgp_peer_shutdown.py + - bgp/test_bgp_queue.py + - bgp/test_bgp_session_flap.py + - cacl/test_cacl_application.py + - clock/test_clock.py + - console/test_console_availability.py + - console/test_console_driver.py + - console/test_console_loopback.py + - console/test_console_reversessh.py + - console/test_console_udevrule.py + - container_hardening/test_container_hardening.py + - database/test_db_config.py + - database/test_db_scripts.py + - disk/test_disk_exhaustion.py + - dns/static_dns/test_static_dns.py + - dns/test_dns_resolv_conf.py + - dut_console/test_console_baud_rate.py + - dut_console/test_escape_character.py + - dut_console/test_idle_timeout.py + - generic_config_updater/test_aaa.py + - generic_config_updater/test_bgp_prefix.py + - generic_config_updater/test_bgp_sentinel.py + - generic_config_updater/test_cacl.py + - generic_config_updater/test_ecn_config_update.py + - generic_config_updater/test_eth_interface.py + - generic_config_updater/test_ipv6.py + - generic_config_updater/test_kubernetes_config.py + - generic_config_updater/test_monitor_config.py + - generic_config_updater/test_ntp.py + - generic_config_updater/test_pfcwd_status.py + - generic_config_updater/test_pg_headroom_update.py + - generic_config_updater/test_syslog.py + - gnmi/test_gnmi.py + - gnmi/test_gnmi_appldb.py + - gnmi/test_gnmi_configdb.py + - gnmi/test_gnmi_countersdb.py + - iface_loopback_action/test_iface_loopback_action.py + - log_fidelity/test_bgp_shutdown.py + - memory_checker/test_memory_checker.py + - minigraph/test_masked_services.py + - ntp/test_ntp.py + - override_config_table/test_override_config_table_masic.py + - passw_hardening/test_passw_hardening.py + - pc/test_po_cleanup.py + - pc/test_retry_count.py + - platform_tests/broadcom/test_ser.py + - platform_tests/cli/test_show_platform.py + - platform_tests/counterpoll/test_counterpoll_watermark.py + - platform_tests/fwutil/test_fwutil.py + - platform_tests/link_flap/test_cont_link_flap.py + - platform_tests/link_flap/test_link_flap.py + - platform_tests/sfp/test_sfpshow.py + - platform_tests/sfp/test_sfputil.py + - platform_tests/sfp/test_show_intf_xcvr.py + - platform_tests/test_auto_negotiation.py + - platform_tests/test_first_time_boot_password_change/test_first_time_boot_password_change.py + - platform_tests/test_kdump.py + - platform_tests/test_link_down.py + - platform_tests/test_memory_exhaustion.py + - platform_tests/test_platform_info.py + - platform_tests/test_port_toggle.py + - platform_tests/test_power_off_reboot.py + - platform_tests/test_reboot.py + - platform_tests/test_secure_upgrade.py + - platform_tests/test_sensors.py + - platform_tests/test_sequential_restart.py + - platform_tests/test_xcvr_info_in_db.py + - portstat/test_portstat.py + - reset_factory/test_reset_factory.py + - show_techsupport/test_techsupport.py + - show_techsupport/test_techsupport_no_secret.py + - snmp/test_snmp_psu.py + - snmp/test_snmp_queue_counters.py + - ssh/test_ssh_ciphers.py + - ssh/test_ssh_default_password.py + - ssh/test_ssh_limit.py + - ssh/test_ssh_stress.py + - syslog/test_logrotate.py + - syslog/test_syslog.py + - syslog/test_syslog_rate_limit.py + - syslog/test_syslog_source_ip.py + - system_health/test_system_status.py + - system_health/test_watchdog.py + - tacacs/test_ro_disk.py + - telemetry/test_telemetry_cert_rotation.py + - test_features.py + - test_procdockerstatsd.py + - platform_tests/test_reload_config.py + - hash/test_generic_hash.py + - sub_port_interfaces/test_sub_port_interfaces.py + - telemetry/test_events.py + - show_techsupport/test_auto_techsupport.py + - vxlan/test_vxlan_bfd_tsa.py + - vxlan/test_vxlan_ecmp_switchover.py + - copp/test_copp.py + - stress/test_stress_routes.py + - drop_packets/test_drop_counters.py + - crm/test_crm.py + - bgp/test_bgp_sentinel.py + - bgp/test_bgp_suppress_fib.py + - test_nbr_health.py + - platform_tests/test_link_down.py + - gnmi/test_gnmi_countersdb.py + - generic_config_updater/test_cacl.py + - telemetry/test_telemetry.py multi-asic-t1-lag: - bgp/test_bgp_bbr.py @@ -230,6 +430,7 @@ multi-asic-t1-lag: - snmp/test_snmp_loopback.py - snmp/test_snmp_pfc_counters.py - snmp/test_snmp_queue.py + - snmp/test_snmp_queue_counters.py - tacacs/test_accounting.py - tacacs/test_authorization.py - tacacs/test_jit_user.py @@ -244,36 +445,28 @@ multi-asic-t1-lag: - container_checker/test_container_checker.py - http/test_http_copy.py -t2: - - test_vs_chassis_setup.py - - voq/test_voq_init.py - -wan-pub: - - system_health/test_system_status.py - - snmp/test_snmp_cpu.py - - snmp/test_snmp_interfaces.py - - snmp/test_snmp_lldp.py - - snmp/test_snmp_pfc_counters.py - - snmp/test_snmp_queue.py - - snmp/test_snmp_v2mib.py - - arp/test_arpall.py - - telemetry/test_telemetry.py - - tacacs/test_jit_user.py - - tacacs/test_ro_user.py - - tacacs/test_rw_user.py - - tacacs/test_ro_disk.py - - tacacs/test_authorization.py - - tacacs/test_accounting.py - dpu: - dash/test_dash_vnet.py onboarding_t0: - - console/test_console_availability.py - - platform_tests/test_power_off_reboot.py - - platform_tests/test_sequential_restart.py - - platform_tests/test_xcvr_info_in_db.py + - bgp/test_bgp_stress_link_flap.py + # We will add a batch of T0 control plane cases and fix the failed cases later + - pfcwd/test_pfcwd_all_port_storm.py + - pfcwd/test_pfcwd_function.py + - pfcwd/test_pfcwd_timer_accuracy.py + - pfcwd/test_pfcwd_warm_reboot.py + # - platform_tests/test_advanced_reboot.py + - snmp/test_snmp_link_local.py + - lldp/test_lldp_syncd.py + + +onboarding_t1: + - bgp/test_bgp_stress_link_flap.py + - snmp/test_snmp_link_local.py + - lldp/test_lldp_syncd.py +onboarding_dualtor: + - dualtor_mgmt/test_dualtor_bgp_update_delay.py specific_param: t0-sonic: diff --git a/.azure-pipelines/pr_test_skip_scripts.yaml b/.azure-pipelines/pr_test_skip_scripts.yaml new file mode 100644 index 0000000000..e3aaa8fb50 --- /dev/null +++ b/.azure-pipelines/pr_test_skip_scripts.yaml @@ -0,0 +1,305 @@ +t0: + # KVM do not support drop reason in testcase, and testcase would set drop reason in setup stage, can't do more test + - drop_packets/test_configurable_drop_counters.py + # KVM do not support dualtor tunnel functionality, lower tor bgp verify would fail + - dualtor/test_orchagent_slb.py + # KVM do not support dualtor tunnel functionality, verify DB status would fail + - dualtor_io/test_normal_op.py + # This script would toggle PDU, which is not supported on KVM + - dualtor_io/test_tor_failure.py + # This script only supported on Mellanox + - generic_config_updater/test_pfcwd_interval.py + # There is no k8s in inventory file + - k8s/test_config_reload.py + - k8s/test_disable_flag.py + - k8s/test_join_available_master.py + # Mclag test only support on t0-mclag platform which is not in PR test + - mclag/test_mclag_l3.py + # Nat feature is default disabled on both KVM and physical platforms + - nat/test_dynamic_nat.py + - nat/test_static_nat.py + # Neighbor type must be sonic + - ospf/test_ospf.py + - ospf/test_ospf_bfd.py + # Test is not supported on vs testbed + - platform_tests/test_intf_fec.py + # Platform api needs the module `sonic_platform`, which is not included in vs + # So skip these scripts + - platform_tests/api/test_chassis.py + - platform_tests/api/test_chassis_fans.py + - platform_tests/api/test_component.py + - platform_tests/api/test_fan_drawer.py + - platform_tests/api/test_fan_drawer_fans.py + - platform_tests/api/test_module.py + - platform_tests/api/test_psu.py + - platform_tests/api/test_psu_fans.py + - platform_tests/api/test_sfp.py + - platform_tests/api/test_thermal.py + - platform_tests/api/test_watchdog.py + # These test scripts are aimed to test some daemons on physical testbed. + - platform_tests/daemon/test_fancontrol.py + - platform_tests/daemon/test_ledd.py + - platform_tests/daemon/test_pcied.py + - platform_tests/daemon/test_psud.py + - platform_tests/daemon/test_syseepromd.py + # These test scripts are aimed to run on mellanox platform + - platform_tests/mellanox/test_check_sfp_eeprom.py + - platform_tests/mellanox/test_check_sfp_presence.py + - platform_tests/mellanox/test_check_sfp_using_ethtool.py + - platform_tests/mellanox/test_check_sysfs.py + - platform_tests/mellanox/test_hw_management_service.py + - platform_tests/mellanox/test_psu_power_threshold.py + - platform_tests/mellanox/test_reboot_cause.py + # read_mac test needs specific variables and image urls, currently do not support on KVM and regular nightly test + - read_mac/test_read_mac_metadata.py + # This script only supported on Mellanox + - restapi/test_restapi.py + # Route flow counter is not supported on vs platform + - route/test_route_flow_counter.py + # Sflow feature is default disabled on vs platform + - sflow/test_sflow.py + - snmp/test_snmp_phy_entity.py + # Remove from PR test in https://github.com/sonic-net/sonic-mgmt/pull/6073 + - cacl/test_ebtables_application.py + # There is no table SYSTEM_HEALTH_INFO in STATE_DB on kvm testbed + # The tests in this script are all related to the above table + - system_health/test_system_health.py + # Vrf tests are also skipped in nightly test + - mvrf/test_mgmtvrf.py + - vrf/test_vrf.py + - vrf/test_vrf_attr.py + +t1-lag: + # KVM do not support bfd test + - bfd/test_bfd.py + # KVM do not support drop reason in testcase, and testcase would set drop reason in setup stage, can't do more test + - drop_packets/test_configurable_drop_counters.py + # This script only supported on Mellanox + - generic_config_updater/test_pfcwd_interval.py + # There is no k8s in inventory file + - k8s/test_config_reload.py + - k8s/test_disable_flag.py + - k8s/test_join_available_master.py + # Neighbor type must be sonic + - ospf/test_ospf_bfd.py + # Test is not supported on vs testbed + - platform_tests/test_intf_fec.py + # Platform api needs the module `sonic_platform`, which is not included in vs + # So skip these scripts + - platform_tests/api/test_chassis.py + - platform_tests/api/test_chassis_fans.py + - platform_tests/api/test_component.py + - platform_tests/api/test_fan_drawer.py + - platform_tests/api/test_fan_drawer_fans.py + - platform_tests/api/test_module.py + - platform_tests/api/test_psu.py + - platform_tests/api/test_psu_fans.py + - platform_tests/api/test_sfp.py + - platform_tests/api/test_thermal.py + - platform_tests/api/test_watchdog.py + # These test scripts are aimed to test some daemons on physical testbed. + - platform_tests/daemon/test_fancontrol.py + - platform_tests/daemon/test_ledd.py + - platform_tests/daemon/test_pcied.py + - platform_tests/daemon/test_psud.py + - platform_tests/daemon/test_syseepromd.py + # These test scripts are aimed to run on mellanox platform + - platform_tests/mellanox/test_check_sfp_eeprom.py + - platform_tests/mellanox/test_check_sfp_presence.py + - platform_tests/mellanox/test_check_sfp_using_ethtool.py + - platform_tests/mellanox/test_check_sysfs.py + - platform_tests/mellanox/test_hw_management_service.py + - platform_tests/mellanox/test_psu_power_threshold.py + - platform_tests/mellanox/test_reboot_cause.py + # read_mac test needs specific variables and image urls, currently do not support on KVM and regular nightly test + - read_mac/test_read_mac_metadata.py + # Route flow counter is not supported on vs platform + - route/test_route_flow_counter.py + - snmp/test_snmp_phy_entity.py + # Remove from PR test in https://github.com/sonic-net/sonic-mgmt/pull/6073 + - cacl/test_ebtables_application.py + # There is no table SYSTEM_HEALTH_INFO in STATE_DB on kvm testbed + # The tests in this script are all related to the above table + - system_health/test_system_health.py + # Vrf tests are also skipped in nightly test + - mvrf/test_mgmtvrf.py + +t2: + # KVM do not support bfd test + - bfd/test_bfd_static_route.py + - bfd/test_bfd_traffic.py + # This script only supported on Mellanox + - generic_config_updater/test_pfcwd_interval.py + # There is no k8s in inventory file + - k8s/test_config_reload.py + - k8s/test_disable_flag.py + - k8s/test_join_available_master.py + # Platform api needs the module `sonic_platform`, which is not included in vs + # So skip these scripts + - platform_tests/api/test_chassis.py + - platform_tests/api/test_chassis_fans.py + - platform_tests/api/test_component.py + - platform_tests/api/test_fan_drawer.py + - platform_tests/api/test_fan_drawer_fans.py + - platform_tests/api/test_module.py + - platform_tests/api/test_psu.py + - platform_tests/api/test_psu_fans.py + - platform_tests/api/test_sfp.py + - platform_tests/api/test_thermal.py + - platform_tests/api/test_watchdog.py + # Test is not supported on vs testbed + - platform_tests/test_intf_fec.py + # These test scripts are aimed to test some daemons on physical testbed. + - platform_tests/daemon/test_fancontrol.py + - platform_tests/daemon/test_ledd.py + - platform_tests/daemon/test_pcied.py + - platform_tests/daemon/test_psud.py + - platform_tests/daemon/test_syseepromd.py + # These test scripts are aimed to run on mellanox platform + - platform_tests/mellanox/test_check_sfp_eeprom.py + - platform_tests/mellanox/test_check_sfp_presence.py + - platform_tests/mellanox/test_check_sfp_using_ethtool.py + - platform_tests/mellanox/test_check_sysfs.py + - platform_tests/mellanox/test_hw_management_service.py + - platform_tests/mellanox/test_psu_power_threshold.py + - platform_tests/mellanox/test_reboot_cause.py + # read_mac test needs specific variables and image urls, currently do not support on KVM and regular nightly test + - read_mac/test_read_mac_metadata.py + - snmp/test_snmp_phy_entity.py + # Remove from PR test in https://github.com/sonic-net/sonic-mgmt/pull/6073 + - cacl/test_ebtables_application.py + # There is no table SYSTEM_HEALTH_INFO in STATE_DB on kvm testbed + # The tests in this script are all related to the above table + - system_health/test_system_health.py + # This script is also skipped in nightly test + - mvrf/test_mgmtvrf.py + # Voq test only support on T2, currently not available in PR test + - voq/test_fabric_cli_and_db.py + - voq/test_fabric_reach.py + - voq/test_voq_chassis_app_db_consistency.py + - voq/test_voq_disrupts.py + - voq/test_voq_fabric_isolation.py + - voq/test_voq_fabric_status_all.py + - voq/test_voq_intfs.py + - voq/test_voq_ipfwd.py + - voq/test_voq_nbr.py + - test_vs_chassis_setup.py + - voq/test_voq_init.py + +dualtor: + # This test only support on dualtor-aa testbed + - dualtor_io/test_grpc_server_failure.py + # This test is only for Nvidia platforms. + - dualtor_mgmt/test_egress_drop_nvidia.py + +tgen: + # Ixia test only support on physical ixia testbed + - ixia/ecn/test_dequeue_ecn.py + - ixia/ecn/test_red_accuracy.py + - ixia/ixanvl/test_bgp_conformance.py + - ixia/pfc/test_global_pause.py + - ixia/pfc/test_pfc_congestion.py + - ixia/pfc/test_pfc_pause_lossless.py + - ixia/pfc/test_pfc_pause_lossy.py + - ixia/pfcwd/test_pfcwd_a2a.py + - ixia/pfcwd/test_pfcwd_basic.py + - ixia/pfcwd/test_pfcwd_burst_storm.py + - ixia/pfcwd/test_pfcwd_m2o.py + - ixia/pfcwd/test_pfcwd_runtime_traffic.py + - ixia/test_ixia_traffic.py + - ixia/test_tgen.py + # Snappi test only support on physical tgen testbed + - snappi_tests/bgp/test_bgp_convergence_performance.py + - snappi_tests/bgp/test_bgp_local_link_failover.py + - snappi_tests/bgp/test_bgp_remote_link_failover.py + - snappi_tests/bgp/test_bgp_rib_in_capacity.py + - snappi_tests/bgp/test_bgp_rib_in_convergence.py + - snappi_tests/bgp/test_bgp_scalability.py + - snappi_tests/ecn/test_dequeue_ecn_with_snappi.py + - snappi_tests/ecn/test_red_accuracy_with_snappi.py + - snappi_tests/multidut/bgp/test_bgp_outbound_downlink_port_flap.py + - snappi_tests/multidut/bgp/test_bgp_outbound_downlink_process_crash.py + - snappi_tests/multidut/bgp/test_bgp_outbound_tsa.py + - snappi_tests/multidut/bgp/test_bgp_outbound_uplink_multi_po_flap.py + - snappi_tests/multidut/bgp/test_bgp_outbound_uplink_po_flap.py + - snappi_tests/multidut/bgp/test_bgp_outbound_uplink_po_member_flap.py + - snappi_tests/multidut/bgp/test_bgp_outbound_uplink_process_crash.py + - snappi_tests/multidut/ecn/test_multidut_dequeue_ecn_with_snappi.py + - snappi_tests/multidut/ecn/test_multidut_red_accuracy_with_snappi.py + - snappi_tests/multidut/pfc/test_lossless_response_to_external_pause_storms.py + - snappi_tests/multidut/pfc/test_lossless_response_to_throttling_pause_storms.py + - snappi_tests/multidut/pfc/test_m2o_fluctuating_lossless.py + - snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossless.py + - snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossless_lossy.py + - snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossy.py + - snappi_tests/multidut/pfc/test_multidut_global_pause_with_snappi.py + - snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossless_with_snappi.py + - snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossy_with_snappi.py + - snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py + - snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py + - snappi_tests/multidut/pfcwd/test_multidut_pfcwd_burst_storm_with_snappi.py + - snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py + - snappi_tests/lacp/test_add_remove_link_from_dut.py + - snappi_tests/lacp/test_add_remove_link_physically.py + - snappi_tests/lacp/test_lacp_timers_effect.py + - snappi_tests/pfc/test_global_pause_with_snappi.py + - snappi_tests/pfc/test_pfc_pause_lossless_with_snappi.py + - snappi_tests/pfc/test_pfc_pause_lossy_with_snappi.py + - snappi_tests/pfc/test_pfc_pause_response_with_snappi.py + - snappi_tests/pfc/test_pfc_pause_unset_bit_enable_vector.py + - snappi_tests/pfc/test_pfc_pause_zero_mac.py + - snappi_tests/pfc/test_valid_pfc_frame_with_snappi.py + - snappi_tests/pfcwd/test_pfcwd_a2a_with_snappi.py + - snappi_tests/pfcwd/test_pfcwd_basic_with_snappi.py + - snappi_tests/pfcwd/test_pfcwd_burst_storm_with_snappi.py + - snappi_tests/pfcwd/test_pfcwd_m2o_with_snappi.py + - snappi_tests/pfcwd/test_pfcwd_runtime_traffic_with_snappi.py + - snappi_tests/qos/test_ipip_packet_reorder_with_snappi.py + +snappi: + # Snappi test only support on physical snappi testbed + - snappi_tests/multidut/pfcwd/test_multidut_pfcwd_runtime_traffic_with_snappi.py + - snappi_tests/reboot/test_cold_reboot.py + - snappi_tests/reboot/test_fast_reboot.py + - snappi_tests/reboot/test_soft_reboot.py + - snappi_tests/reboot/test_warm_reboot.py + - snappi_tests/test_multidut_snappi.py + - snappi_tests/test_snappi.py + +wan-pub: + # Currently PR test will not test wan topo + - macsec/test_interop_wan_isis.py + - wan/isis/test_isis_authentication.py + - wan/isis/test_isis_csnp_interval.py + - wan/isis/test_isis_database.py + - wan/isis/test_isis_dynamic_hostname.py + - wan/isis/test_isis_ecmp.py + - wan/isis/test_isis_hello_interval.py + - wan/isis/test_isis_hello_pad.py + - wan/isis/test_isis_holdtime.py + - wan/isis/test_isis_intf_passive.py + - wan/isis/test_isis_level_capacity.py + - wan/isis/test_isis_log_adjacency_change.py + - wan/isis/test_isis_lsp_fragment.py + - wan/isis/test_isis_lsp_gen_interval.py + - wan/isis/test_isis_lsp_lifetime.py + - wan/isis/test_isis_lsp_refresh.py + - wan/isis/test_isis_metric_wide.py + - wan/isis/test_isis_neighbor.py + - wan/isis/test_isis_overload_bit.py + - wan/isis/test_isis_redistribute.py + - wan/isis/test_isis_spf_default_interval.py + - wan/isis/test_isis_spf_ietf_interval.py + - wan/lacp/test_wan_lacp.py + - wan/lacp/test_wan_lag_member.py + - wan/lacp/test_wan_lag_min_link.py + - wan/lldp/test_wan_lldp.py + - wan/traffic_test/test_traffic.py + +ptf: + # PTF test do not support on physical testbed + - sai_qualify/test_brcm_t0.py + - sai_qualify/test_community.py + - sai_qualify/test_sai_ptf.py + - sai_qualify/test_sai_ptf_warm_reboot.py + - sai_qualify/test_sai_t0_warm_reboot.py diff --git a/.azure-pipelines/recover_testbed/recover_testbed.py b/.azure-pipelines/recover_testbed/recover_testbed.py index a7f508e8ae..b4278b6408 100644 --- a/.azure-pipelines/recover_testbed/recover_testbed.py +++ b/.azure-pipelines/recover_testbed/recover_testbed.py @@ -7,7 +7,6 @@ import ipaddress import traceback from common import do_power_cycle, check_sonic_installer, posix_shell_aboot, posix_shell_onie -from constants import RC_SSH_FAILED _self_dir = os.path.dirname(os.path.abspath(__file__)) base_path = os.path.realpath(os.path.join(_self_dir, "../..")) @@ -106,11 +105,9 @@ def recover_testbed(sonichosts, conn_graph_facts, localhost, image_url, hwsku): except Exception as e: logger.info("Exception caught while executing cmd. Error message: {}".format(e)) need_to_recover = True - elif dut_ssh == RC_SSH_FAILED: + else: # Do power cycle need_to_recover = True - else: - raise Exception("Authentication failed. Passwords are incorrect.") if need_to_recover: recover_via_console(sonichost, conn_graph_facts, localhost, mgmt_ip, image_url, hwsku) diff --git a/.azure-pipelines/run-test-elastictest-template.yml b/.azure-pipelines/run-test-elastictest-template.yml index 639cdac720..1c846d3fb1 100644 --- a/.azure-pipelines/run-test-elastictest-template.yml +++ b/.azure-pipelines/run-test-elastictest-template.yml @@ -47,6 +47,10 @@ parameters: type: string default: "" + - name: PTF_IMAGE_TAG + type: string + default: "" + - name: IMAGE_URL type: string default: "" @@ -159,11 +163,7 @@ steps: # Else, internal build image repo, download from internal sonic-mgmt repo else - if [ -z "$(MSSONIC-TOKEN)" ]; then - curl -u $(AZP_REPO_ACCESS_TOKEN) "${{ parameters.MGMT_URL }}&commitOrBranch=${{ parameters.MGMT_BRANCH }}&api-version=5.0-preview.1&path=.azure-pipelines%2Fpr_test_scripts.yaml" -o ./.azure-pipelines/pr_test_scripts.yaml - else - curl -u :$(MSSONIC-TOKEN) "${{ parameters.MGMT_URL }}&commitOrBranch=${{ parameters.MGMT_BRANCH }}&api-version=5.0-preview.1&path=.azure-pipelines%2Fpr_test_scripts.yaml" -o ./.azure-pipelines/pr_test_scripts.yaml - fi + curl -u :$(MSSONIC-TOKEN) "${{ parameters.MGMT_URL }}&commitOrBranch=${{ parameters.MGMT_BRANCH }}&api-version=5.0-preview.1&path=.azure-pipelines%2Fpr_test_scripts.yaml" -o ./.azure-pipelines/pr_test_scripts.yaml fi displayName: "Download pr script" - ${{ else }}: @@ -175,174 +175,226 @@ steps: displayName: "Download test plan script" - script: | - set -e - - pip install PyYAML - - rm -f new_test_plan_id.txt - - python ./.azure-pipelines/test_plan.py create \ - -t ${{ parameters.TOPOLOGY }} \ - -o new_test_plan_id.txt \ - --min-worker ${{ parameters.MIN_WORKER }} \ - --max-worker ${{ parameters.MAX_WORKER }} \ - --lock-wait-timeout-seconds ${{ parameters.LOCK_WAIT_TIMEOUT_SECONDS }} \ - --test-set ${{ parameters.TEST_SET }} \ - --kvm-build-id $(KVM_BUILD_ID) \ - --kvm-image-branch "${{ parameters.KVM_IMAGE_BRANCH }}" \ - --deploy-mg-extra-params="${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" \ - --common-extra-params="${{ parameters.COMMON_EXTRA_PARAMS }}" \ - --vm-type ${{ parameters.VM_TYPE }} --num-asic ${{ parameters.NUM_ASIC }} \ - --image_url ${{ parameters.IMAGE_URL }} \ - --upgrade-image-param="${{ parameters.UPGRADE_IMAGE_PARAM }}" \ - --hwsku ${{ parameters.HWSKU }} \ - --test-plan-type ${{ parameters.TEST_PLAN_TYPE }} \ - --platform ${{ parameters.PLATFORM }} \ - --testbed-name "${{ parameters.TESTBED_NAME }}" \ - --scripts "${{ parameters.SCRIPTS }}" \ - --features "${{ parameters.FEATURES }}" \ - --scripts-exclude "${{ parameters.SCRIPTS_EXCLUDE }}" \ - --features-exclude "${{ parameters.FEATURES_EXCLUDE }}" \ - --specific-param='${{ parameters.SPECIFIC_PARAM }}' \ - --affinity='${{ parameters.AFFINITY }}' \ - --build-reason ${{ parameters.BUILD_REASON }} \ - --repo-name ${{ parameters.REPO_NAME }} \ - --mgmt-branch ${{ parameters.MGMT_BRANCH }} \ - --stop-on-failure ${{ parameters.STOP_ON_FAILURE }} \ - --retry-times ${{ parameters.RETRY_TIMES }} \ - --dump-kvm-if-fail ${{ parameters.DUMP_KVM_IF_FAIL }} \ - --requester "${{ parameters.REQUESTER }}" \ - --max-execute-seconds $((${{ parameters.MAX_RUN_TEST_MINUTES }} * 60)) \ - --test-plan-num ${{ parameters.TEST_PLAN_NUM }} - - TEST_PLAN_ID_LIST=( $(cat new_test_plan_id.txt) ) - echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - echo "Created test plan $TEST_PLAN_ID" - echo -e -n "\033[33mPlease visit Elastictest page \033[0m" - echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " - echo -e "\033[33mfor detailed test plan progress \033[0m" - done - TEST_PLAN_ID_LIST_STRING=$(printf "%s," "${TEST_PLAN_ID_LIST[@]}") - TEST_PLAN_ID_LIST_STRING=${TEST_PLAN_ID_LIST_STRING%,} - echo "##vso[task.setvariable variable=TEST_PLAN_ID_LIST_STRING]$TEST_PLAN_ID_LIST_STRING" + # Check if azure cli is installed. If not, try to install it + if ! command -v az; then + echo "Azure CLI is not installed. Trying to install it..." + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + else + echo "Azure CLI is already installed" + fi + displayName: "Install azure-cli" + + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -e + + pip install PyYAML + + rm -f new_test_plan_id.txt + + python ./.azure-pipelines/test_plan.py create \ + -t ${{ parameters.TOPOLOGY }} \ + -o new_test_plan_id.txt \ + --min-worker ${{ parameters.MIN_WORKER }} \ + --max-worker ${{ parameters.MAX_WORKER }} \ + --lock-wait-timeout-seconds ${{ parameters.LOCK_WAIT_TIMEOUT_SECONDS }} \ + --test-set ${{ parameters.TEST_SET }} \ + --kvm-build-id $(KVM_BUILD_ID) \ + --kvm-image-branch "${{ parameters.KVM_IMAGE_BRANCH }}" \ + --deploy-mg-extra-params="${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" \ + --common-extra-params="${{ parameters.COMMON_EXTRA_PARAMS }}" \ + --vm-type ${{ parameters.VM_TYPE }} --num-asic ${{ parameters.NUM_ASIC }} \ + --ptf_image_tag ${{ parameters.PTF_IMAGE_TAG }} \ + --image_url ${{ parameters.IMAGE_URL }} \ + --upgrade-image-param="${{ parameters.UPGRADE_IMAGE_PARAM }}" \ + --hwsku ${{ parameters.HWSKU }} \ + --test-plan-type ${{ parameters.TEST_PLAN_TYPE }} \ + --platform ${{ parameters.PLATFORM }} \ + --testbed-name "${{ parameters.TESTBED_NAME }}" \ + --scripts "${{ parameters.SCRIPTS }}" \ + --features "${{ parameters.FEATURES }}" \ + --scripts-exclude "${{ parameters.SCRIPTS_EXCLUDE }}" \ + --features-exclude "${{ parameters.FEATURES_EXCLUDE }}" \ + --specific-param='${{ parameters.SPECIFIC_PARAM }}' \ + --affinity='${{ parameters.AFFINITY }}' \ + --build-reason ${{ parameters.BUILD_REASON }} \ + --repo-name ${{ parameters.REPO_NAME }} \ + --mgmt-branch ${{ parameters.MGMT_BRANCH }} \ + --stop-on-failure ${{ parameters.STOP_ON_FAILURE }} \ + --retry-times ${{ parameters.RETRY_TIMES }} \ + --dump-kvm-if-fail ${{ parameters.DUMP_KVM_IF_FAIL }} \ + --requester "${{ parameters.REQUESTER }}" \ + --max-execute-seconds $((${{ parameters.MAX_RUN_TEST_MINUTES }} * 60)) \ + --test-plan-num ${{ parameters.TEST_PLAN_NUM }} + + TEST_PLAN_ID_LIST=( $(cat new_test_plan_id.txt) ) + echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + echo "Created test plan $TEST_PLAN_ID" + echo -e -n "\033[33mPlease visit Elastictest page \033[0m" + echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " + echo -e "\033[33mfor detailed test plan progress \033[0m" + done + TEST_PLAN_ID_LIST_STRING=$(printf "%s," "${TEST_PLAN_ID_LIST[@]}") + TEST_PLAN_ID_LIST_STRING=${TEST_PLAN_ID_LIST_STRING%,} + echo "##vso[task.setvariable variable=TEST_PLAN_ID_LIST_STRING]$TEST_PLAN_ID_LIST_STRING" displayName: "Trigger test" - - script: | - set -o - echo "Lock testbed" - - echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" - IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" - failure_count=0 - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - echo -e -n "\033[33mPlease visit Elastictest page \033[0m" - echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " - echo -e "\033[33mfor detailed test plan progress \033[0m" - # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED" - echo "[test_plan.py] poll LOCK_TESTBED status" - python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state LOCK_TESTBED - RET=$? - if [ $RET -ne 0 ]; then - ((failure_count++)) - fi - done + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -o + echo "Lock testbed" + + echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" + IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" + failure_count=0 + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + echo -e -n "\033[33mPlease visit Elastictest page \033[0m" + echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " + echo -e "\033[33mfor detailed test plan progress \033[0m" + # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED" + echo "[test_plan.py] poll LOCK_TESTBED status" + python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state LOCK_TESTBED + RET=$? + if [ $RET -ne 0 ]; then + ((failure_count++)) + fi + done - if [ $failure_count -eq ${#TEST_PLAN_ID_LIST[@]} ]; then - echo "All testplan failed, cancel following steps" - exit 3 - fi + if [ $failure_count -eq ${#TEST_PLAN_ID_LIST[@]} ]; then + echo "All testplan failed, cancel following steps" + exit 3 + fi displayName: "Lock testbed" - - script: | - set -o - echo "Prepare testbed" - echo "Preparing the testbed(add-topo, deploy-mg) may take 15-30 minutes. Before the testbed is ready, the progress of the test plan keeps displayed as 0, please be patient" - - echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" - IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" - failure_count=0 - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - echo -e -n "\033[33mPlease visit Elastictest page \033[0m" - echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " - echo -e "\033[33mfor detailed test plan progress \033[0m" - # When "PREPARE_TESTBED" finish, it changes into "EXECUTING" - echo "[test_plan.py] poll PREPARE_TESTBED status" - python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state PREPARE_TESTBED - RET=$? - if [ $RET -ne 0 ]; then - ((failure_count++)) - fi - done + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -o + echo "Prepare testbed" + echo "Preparing the testbed(add-topo, deploy-mg) may take 15-30 minutes. Before the testbed is ready, the progress of the test plan keeps displayed as 0, please be patient" + + echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" + IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" + failure_count=0 + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + echo -e -n "\033[33mPlease visit Elastictest page \033[0m" + echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " + echo -e "\033[33mfor detailed test plan progress \033[0m" + # When "PREPARE_TESTBED" finish, it changes into "EXECUTING" + echo "[test_plan.py] poll PREPARE_TESTBED status" + python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state PREPARE_TESTBED + RET=$? + if [ $RET -ne 0 ]; then + ((failure_count++)) + fi + done - if [ "$failure_count" -eq ${#TEST_PLAN_ID_LIST[@]} ]; then - echo "All testplan failed, cancel following steps" - exit 3 - fi + if [ "$failure_count" -eq ${#TEST_PLAN_ID_LIST[@]} ]; then + echo "All testplan failed, cancel following steps" + exit 3 + fi displayName: "Prepare testbed" - - script: | - set -o - echo "Run test" - - echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" - IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" - failure_count=0 - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - echo -e -n "\033[33mPlease visit Elastictest page \033[0m" - echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " - echo -e "\033[33mfor detailed test plan progress \033[0m" - # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED" - echo "[test_plan.py] poll EXECUTING status" - python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state EXECUTING --expected-result ${{ parameters.EXPECTED_RESULT }} - RET=$? - if [ $RET -ne 0 ]; then - ((failure_count++)) - fi - done + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -o + echo "Run test" + + echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" + IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" + failure_count=0 + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + echo -e -n "\033[33mPlease visit Elastictest page \033[0m" + echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " + echo -e "\033[33mfor detailed test plan progress \033[0m" + # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED" + echo "[test_plan.py] poll EXECUTING status, timeout 22 hours" + python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state EXECUTING --expected-result ${{ parameters.EXPECTED_RESULT }} --timeout 79200 + RET=$? + # RC==2 means polling test plan timeout, do not consider it as failure so far + if [ $RET -ne 0 ] && [ $RET -ne 2 ]; then + echo "Test plan $TEST_PLAN_ID failed with RC $RET" + ((failure_count++)) + fi + done - if [ $failure_count -eq ${#TEST_PLAN_ID_LIST[@]} ]; then - echo "All testplan failed, cancel following steps" - exit 3 - fi + if [ $failure_count -eq ${#TEST_PLAN_ID_LIST[@]} ]; then + echo "All testplan failed, cancel following steps" + exit 3 + fi displayName: "Run test" timeoutInMinutes: ${{ parameters.MAX_RUN_TEST_MINUTES }} - ${{ if eq(parameters.DUMP_KVM_IF_FAIL, 'True') }}: - - script: | - set -e - echo "KVM dump" - - echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" - IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - echo -e -n "\033[33mPlease visit Elastictest page \033[0m" - echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " - echo -e "\033[33mfor detailed test plan progress \033[0m" - # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED" - echo "##[group][test_plan.py] poll KVMDUMP status" - python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state KVMDUMP - done + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -e + echo "KVM dump" + + echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" + IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + echo -e -n "\033[33mPlease visit Elastictest page \033[0m" + echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " + echo -e "\033[33mfor detailed test plan progress \033[0m" + # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED" + echo "##[group][test_plan.py] poll KVMDUMP status" + python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state KVMDUMP + done condition: succeededOrFailed() displayName: "KVM dump" - - script: | - set -e - echo "Try to cancel test plan $TEST_PLAN_ID, cancelling finished test plan has no effect." - IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - python ./.azure-pipelines/test_plan.py cancel -i $TEST_PLAN_ID - done + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -e + echo "Try to cancel test plan $TEST_PLAN_ID, cancelling finished test plan has no effect." + + # If TEST_PLAN_TYPE is NIGHTLY, skip the cancel step + test_plan_type=${{ parameters.TEST_PLAN_TYPE }} + echo "TEST_PLAN_TYPE is $test_plan_type" + if [ "$test_plan_type" == "NIGHTLY" ]; then + echo "TEST_PLAN_TYPE is NIGHTLY, skip the cancel step as a dirty workaround for az login timeout issue" + exit 0 + fi + + IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + python ./.azure-pipelines/test_plan.py cancel -i $TEST_PLAN_ID + done condition: always() displayName: "Finalize running test plan" diff --git a/.azure-pipelines/test_plan.py b/.azure-pipelines/test_plan.py index c238ddd550..1d32bc5ec1 100644 --- a/.azure-pipelines/test_plan.py +++ b/.azure-pipelines/test_plan.py @@ -5,6 +5,7 @@ import json import os import sys +import subprocess import copy import time from datetime import datetime, timedelta @@ -22,8 +23,14 @@ PR_TEST_SCRIPTS_FILE = "pr_test_scripts.yaml" SPECIFIC_PARAM_KEYWORD = "specific_param" TOLERATE_HTTP_EXCEPTION_TIMES = 20 -TOKEN_EXPIRE_HOURS = 6 +TOKEN_EXPIRE_HOURS = 1 MAX_GET_TOKEN_RETRY_TIMES = 3 +TEST_PLAN_STATUS_UNSUCCESSFUL_FINISHED = ["FAILED", "CANCELLED"] +TEST_PLAN_STEP_STATUS_UNFINISHED = ["EXECUTING", None] + + +class PollTimeoutException(Exception): + pass class TestPlanStatus(Enum): @@ -69,7 +76,7 @@ def test_plan_status_factory(status): raise Exception("The status is not correct.") -class AbstractStatus(): +class AbstractStatus: def __init__(self, status): self.status = status @@ -130,18 +137,6 @@ def __init__(self): super(FinishStatus, self).__init__(TestPlanStatus.FINISHED) -def get_scope(elastictest_url): - scope = "api://sonic-testbed-tools-dev/.default" - if elastictest_url in [ - "http://sonic-testbed2-scheduler-backend.azurewebsites.net", - "https://sonic-testbed2-scheduler-backend.azurewebsites.net", - "http://sonic-elastictest-prod-scheduler-backend-webapp.azurewebsites.net", - "https://sonic-elastictest-prod-scheduler-backend-webapp.azurewebsites.net" - ]: - scope = "api://sonic-testbed-tools-prod/.default" - return scope - - def parse_list_from_str(s): # Since Azure Pipeline doesn't support to receive an empty parameter, # We use ' ' as a magic code for empty parameter. @@ -157,49 +152,65 @@ def parse_list_from_str(s): class TestPlanManager(object): - def __init__(self, url, frontend_url, tenant_id=None, client_id=None, client_secret=None, ): + def __init__(self, url, frontend_url, client_id=None): self.url = url self.frontend_url = frontend_url - self.tenant_id = tenant_id self.client_id = client_id - self.client_secret = client_secret self.with_auth = False self._token = None - self._token_generate_time = None - if self.tenant_id and self.client_id and self.client_secret: + self._token_expires_on = None + if self.client_id: self.with_auth = True self.get_token() + def cmd(self, cmds): + process = subprocess.Popen( + cmds, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout, stderr = process.communicate() + return_code = process.returncode + + return stdout, stderr, return_code + + def az_run(self, cmd): + stdout, stderr, retcode = self.cmd(cmd.split()) + if retcode != 0: + raise Exception(f'Command {cmd} execution failed, rc={retcode}, error={stderr}') + return stdout, stderr, retcode + def get_token(self): - token_generate_time_valid = \ - self._token_generate_time is not None and \ - (datetime.utcnow() - self._token_generate_time) < timedelta(hours=TOKEN_EXPIRE_HOURS) - if self._token is not None and token_generate_time_valid: - return self._token + token_is_valid = \ + self._token_expires_on is not None and \ + (self._token_expires_on - datetime.now()) > timedelta(hours=TOKEN_EXPIRE_HOURS) - token_url = "https://login.microsoftonline.com/{}/oauth2/v2.0/token".format(self.tenant_id) - headers = { - "Content-Type": "application/x-www-form-urlencoded" - } + if self._token is not None and token_is_valid: + return self._token - payload = { - "grant_type": "client_credentials", - "client_id": self.client_id, - "client_secret": self.client_secret, - "scope": get_scope(self.url) - } + cmd = 'az account get-access-token --resource {}'.format(self.client_id) attempt = 0 while (attempt < MAX_GET_TOKEN_RETRY_TIMES): try: - resp = requests.post(token_url, headers=headers, data=payload, timeout=10).json() - self._token = resp["access_token"] - self._token_generate_time = datetime.utcnow() + stdout, _, _ = self.az_run(cmd) + + token = json.loads(stdout.decode("utf-8")) + self._token = token.get("accessToken", None) + if not self._token: + raise Exception("Parse token from stdout failed") + + # Parse token expires time from string + token_expires_on = token.get("expiresOn", "") + self._token_expires_on = datetime.strptime(token_expires_on, "%Y-%m-%d %H:%M:%S.%f") + print("Get token successfully.") return self._token + except Exception as exception: attempt += 1 - print("Get token failed with exception: {}. Retry {} times to get token." - .format(repr(exception), MAX_GET_TOKEN_RETRY_TIMES - attempt)) + print("Failed to get token with exception: {}".format(repr(exception))) + raise Exception("Failed to get token after {} attempts".format(MAX_GET_TOKEN_RETRY_TIMES)) def create(self, topology, test_plan_name="my_test_plan", deploy_mg_extra_params="", kvm_build_id="", @@ -215,13 +226,14 @@ def create(self, topology, test_plan_name="my_test_plan", deploy_mg_extra_params features = parse_list_from_str(kwargs.get("features", None)) scripts_exclude = parse_list_from_str(kwargs.get("scripts_exclude", None)) features_exclude = parse_list_from_str(kwargs.get("features_exclude", None)) + ptf_image_tag = kwargs.get("ptf_image_tag", None) print("Creating test plan, topology: {}, name: {}, build info:{} {} {}".format(topology, test_plan_name, repo_name, pr_id, build_id)) print("Test scripts to be covered in this test plan:") print(json.dumps(scripts, indent=4)) - common_extra_params = common_extra_params + " --completeness_level=confident --allow_recover" + common_extra_params = common_extra_params + " --allow_recover" # Add topo and device type args for PR test if test_plan_type == "PR": @@ -277,6 +289,7 @@ def create(self, topology, test_plan_name="my_test_plan", deploy_mg_extra_params "features_exclude": features_exclude, "scripts_exclude": scripts_exclude }, + "ptf_image_tag": ptf_image_tag, "image": { "url": image_url, "upgrade_image_param": kwargs.get("upgrade_image_param", None), @@ -371,7 +384,7 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte } start_time = time.time() http_exception_times = 0 - while (timeout < 0 or (time.time() - start_time) < timeout): + while timeout < 0 or (time.time() - start_time) < timeout: try: if self.with_auth: headers["Authorization"] = "Bearer {}".format(self.get_token()) @@ -393,13 +406,15 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte if not resp_data: raise Exception("No valid data in response: {}".format(str(resp))) - status = resp_data.get("status", None) - result = resp_data.get("result", None) + current_tp_status = resp_data.get("status", None) + current_tp_result = resp_data.get("result", None) if expected_state: - current_status = test_plan_status_factory(status) + current_status = test_plan_status_factory(current_tp_status) expected_status = test_plan_status_factory(expected_state) + print("current test plan status: {}, expected status: {}".format(current_tp_status, expected_state)) + if expected_status.get_status() == current_status.get_status(): current_status.print_logs(test_plan_id, resp_data, start_time) elif expected_status.get_status() < current_status.get_status(): @@ -413,34 +428,60 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte if step.get("step") == expected_state: step_status = step.get("status") break - # We fail the step only if the step_status is "FAILED". - # Other status such as "SKIPPED", "CANCELED" are considered successful. - if step_status == "FAILED": + + # Print test summary + test_summary = resp_data.get("runtime", {}).get("test_summary", None) + if test_summary: + print("Test summary:\n{}".format(json.dumps(test_summary, indent=4))) + + """ + In below scenarios, need to return false to pipeline. + 1. If step status is {FAILED}, exactly need to return false to pipeline. + 2. If current test plan status finished but unsuccessful, need to check if current step status + executed successfully, if not, return false to pipeline. + """ + current_step_unsuccessful = (step_status == "FAILED" + or (current_tp_status in TEST_PLAN_STATUS_UNSUCCESSFUL_FINISHED + and step_status in TEST_PLAN_STEP_STATUS_UNFINISHED)) + + if current_step_unsuccessful: + + # Print error type and message + err_code = resp_data.get("runtime", {}).get("err_code", None) + if err_code: + print("Error type: {}".format(err_code)) + + err_msg = resp_data.get("runtime", {}).get("message", None) + if err_msg: + print("Error message: {}".format(err_msg)) + raise Exception("Test plan id: {}, status: {}, result: {}, Elapsed {:.0f} seconds. " "Check {}/scheduler/testplan/{} for test plan status" - .format(test_plan_id, step_status, result, time.time() - start_time, + .format(test_plan_id, step_status, current_tp_result, time.time() - start_time, self.frontend_url, test_plan_id)) if expected_result: - if result != expected_result: + if current_tp_result != expected_result: raise Exception("Test plan id: {}, status: {}, result: {} not match expected result: {}, " "Elapsed {:.0f} seconds. " "Check {}/scheduler/testplan/{} for test plan status" - .format(test_plan_id, step_status, result, + .format(test_plan_id, step_status, current_tp_result, expected_result, time.time() - start_time, self.frontend_url, test_plan_id)) - print("Current status is {}".format(step_status)) + print("Current step status is {}".format(step_status)) return else: - print("Current state is {}, waiting for the state {}".format(status, expected_state)) + print("Current test plan state is {}, waiting for the expected state {}".format(current_tp_status, + expected_state)) time.sleep(interval) else: - raise Exception("Max polling time reached, test plan at {} is not successfully finished or cancelled" - .format(poll_url)) + raise PollTimeoutException( + "Max polling time reached, test plan at {} is not successfully finished or cancelled".format(poll_url) + ) if __name__ == "__main__": @@ -616,6 +657,16 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte required=False, help="Testbed name, Split by ',', like: 'testbed1, testbed2'" ) + parser_create.add_argument( + "--ptf_image_tag", + type=str, + dest="ptf_image_tag", + nargs='?', + const=None, + default=None, + required=False, + help="PTF image tag" + ) parser_create.add_argument( "--image_url", type=str, @@ -837,7 +888,7 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte required=False, default=-1, dest="timeout", - help="Max polling time. Default 36000 seconds (10 hours)." + help="Max polling time in seconds. Default -1, no timeout." ) if len(sys.argv) == 1: @@ -852,7 +903,7 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte args.test_plan_id = args.test_plan_id.replace("'", "") print("Test plan utils parameters: {}".format(args)) - auth_env = ["TENANT_ID", "CLIENT_ID", "CLIENT_SECRET"] + auth_env = ["CLIENT_ID"] required_env = ["ELASTICTEST_SCHEDULER_BACKEND_URL"] if args.action in ["create", "cancel"]: @@ -860,9 +911,7 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte env = { "elastictest_scheduler_backend_url": os.environ.get("ELASTICTEST_SCHEDULER_BACKEND_URL"), - "tenant_id": os.environ.get("ELASTICTEST_MSAL_TENANT_ID"), "client_id": os.environ.get("ELASTICTEST_MSAL_CLIENT_ID"), - "client_secret": os.environ.get("ELASTICTEST_MSAL_CLIENT_SECRET"), "frontend_url": os.environ.get("ELASTICTEST_FRONTEND_URL", "https://elastictest.org"), } env_missing = [k.upper() for k, v in env.items() if k.upper() in required_env and not v] @@ -874,9 +923,7 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte tp = TestPlanManager( env["elastictest_scheduler_backend_url"], env["frontend_url"], - env["tenant_id"], - env["client_id"], - env["client_secret"]) + env["client_id"]) if args.action == "create": pr_id = os.environ.get("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER") or os.environ.get( @@ -937,6 +984,7 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte affinity=args.affinity, vm_type=args.vm_type, testbed_name=args.testbed_name, + ptf_image_tag=args.ptf_image_tag, image_url=args.image_url, upgrade_image_param=args.upgrade_image_param, hwsku=args.hwsku, @@ -954,6 +1002,9 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte elif args.action == "cancel": tp.cancel(args.test_plan_id) sys.exit(0) + except PollTimeoutException as e: + print("Polling test plan failed with exception: {}".format(repr(e))) + sys.exit(2) except Exception as e: print("Operation failed with exception: {}".format(repr(e))) sys.exit(3) diff --git a/.azure-pipelines/testscripts_analyse/analyse_testscripts.py b/.azure-pipelines/testscripts_analyse/analyse_testscripts.py new file mode 100644 index 0000000000..3e5f06b1d1 --- /dev/null +++ b/.azure-pipelines/testscripts_analyse/analyse_testscripts.py @@ -0,0 +1,219 @@ +""" +To ensure that a test script is included in the PR checker for the corresponding topology type +and if the script is skipped in PR checker, this script will perform a validation check. +Additionally, the return value will be enhanced to include more detailed information, such as the category of the test. +Post-execution, the script will also append the scan time and track ID to the results. + +The return value is formatted as below: +[ + { + 'testscript': 'acl/custom_acl_table/test_custom_acl_table.py', + 'topology': 't0', + 'trackid': '3aa57f0f-8f18-4cf7-ae1e-0a18973a0b86', + 'scantime': '2024-05-31 06:53:40.826349', + 'category': 'data', + 'covered': False, + 'skipped': False + }, + { + 'testscript': 'bgp/test_bgp_allow_list.py', + 'topology': 't1', + 'trackid': '3aa57f0f-8f18-4cf7-ae1e-0a18973a0b86', + 'scantime': '2024-05-31 06:53:40.826349', + 'category': 'control', + 'covered': False, + 'skipped': False + } +] +And finally, we will upload the results to Kusto table `TestScripts` +""" + +import yaml +import re +import os +import sys +import uuid +import logging + +from natsort import natsorted +from datetime import datetime +from constant import DATAPLANE_FEATURES, PR_TOPOLOGY_TYPE, PR_TOPOLOGY_MAPPING +from report_data_storage import KustoConnector + + +def topo_name_to_type(topo_name): + pattern = re.compile(r'^(wan|t0|t1|ptf|fullmesh|dualtor|t2|tgen|multidut-tgen|mgmttor|m0|mc0|mx|dpu|any|snappi)') + match = pattern.match(topo_name) + if match is None: + logging.warning("Unsupported testbed type - {}".format(topo_name)) + return topo_name + + topo_type = match.group() + if topo_type in ['mgmttor', 'dualtor', 'm0', 'mc0', 'mx']: + # certain testbed types are in 't0' category with different names. + topo_type = 't0' + if topo_type in ['multidut-tgen']: + topo_type = 'tgen' + return topo_type + + +def collect_all_scripts(): + ''' + This function collects all test scripts under the folder 'tests/' + and get the topology type marked in the script + + The return value is a dict contains the script name and topology type + [{ + 'testscript': 'acl/custom_acl_table/test_custom_acl_table.py', + 'topology': 't0' + }] + ''' + location = sys.argv[1] + + # Recursively find all files starting with "test_" and ending with ".py" + # Note: The full path and name of files are stored in a list named "files" + files = [] + for root, dirs, file in os.walk(location): + for f in file: + if f.startswith("test_") and f.endswith(".py"): + files.append(os.path.join(root, f)) + files = natsorted(files) + + # Open each file and search for regex pattern + pattern = re.compile(r"[^@]pytest\.mark\.topology\(([^\)]*)\)") + test_scripts = [] + + for f in files: + # Remove prefix from file name: + filename = f[len(location) + 1:] + try: + with open(f, 'r') as file: + for line in file: + # Get topology type of script from mark `pytest.mark.topology` + match = pattern.search(line) + if match: + for topology in match.group(1).split(","): + topology_mark = topology.strip().strip('"').strip('\'') + result = { + "testscript": filename, + "topology": topo_name_to_type(topology_mark) + } + if result not in test_scripts: + test_scripts.append(result) + except Exception as e: + logging.error('Failed to load file {}, error {}'.format(f, e)) + + return test_scripts + + +def get_pr_checker_scripts(): + ''' + Check if a script is included in the PR checker for the corresponding topology type + ''' + pr_test_scripts_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../", "pr_test_scripts.yaml") + + # Get all the scripts included in different PR checker + pr_test_scripts = {} + try: + with open(pr_test_scripts_file) as f: + pr_test_scripts = yaml.safe_load(f) + except Exception as e: + logging.error('Failed to load file {}, error {}'.format(f, e)) + + # Get all the skip scripts + pr_test_skip_scripts_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../", + "pr_test_skip_scripts.yaml") + pr_test_skip_scripts = {} + try: + with open(pr_test_skip_scripts_file) as f: + pr_test_skip_scripts = yaml.safe_load(f) + except Exception as e: + logging.error('Failed to load file {}, error {}'.format(f, e)) + + test_scripts_per_topology_type = {} + skipped_scripts_per_topology_type = {} + + for key, value in pr_test_skip_scripts.items(): + topology_type = PR_TOPOLOGY_MAPPING.get(key, "") + if topology_type: + if skipped_scripts_per_topology_type.get(topology_type, ""): + skipped_scripts_per_topology_type[topology_type].update(value) + else: + skipped_scripts_per_topology_type[topology_type] = set(value) + + if key in pr_test_scripts: + pr_test_scripts[key].extend(value) + else: + pr_test_scripts[key] = value + + for key, value in pr_test_scripts.items(): + topology_type = PR_TOPOLOGY_MAPPING.get(key, "") + if topology_type: + if test_scripts_per_topology_type.get(topology_type, ""): + test_scripts_per_topology_type[topology_type].update(value) + else: + test_scripts_per_topology_type[topology_type] = set(value) + + return test_scripts_per_topology_type, skipped_scripts_per_topology_type + + +def expand_test_scripts(test_scripts, test_scripts_per_topology_type, skipped_scripts_per_topology_type): + # Expand the test scripts list here. + # If the topology mark is "any", we will add all topology types in PR checker on this script. + expanded_test_scripts = [] + for test_script in test_scripts: + topology_mark = test_script["topology"] + + if topology_mark == "any": + for topology in PR_TOPOLOGY_TYPE: + expanded_test_scripts.append({ + "testscript": test_script["testscript"], + "topology": topology + }) + else: + expanded_test_scripts.append(test_script) + + # Check if a script is included in the PR checker for the corresponding topology type + # And if this script is skipped in PR checker + for test_script in expanded_test_scripts: + topology_type = topo_name_to_type(test_script["topology"]) + + test_script["skipped"] = test_script["testscript"] in skipped_scripts_per_topology_type.get(topology_type, "") + + if test_script["testscript"] == "test_posttest.py" or test_script["testscript"] == "test_pretest.py": + test_script["covered"] = True + else: + test_script["covered"] = test_script["testscript"] in test_scripts_per_topology_type.get(topology_type, "") + return expanded_test_scripts + + +def upload_results(test_scripts): + database = sys.argv[2] + kusto_db = KustoConnector(database) + kusto_db.upload_testscripts(test_scripts) + + +def main(): + test_scripts = collect_all_scripts() + test_scripts_per_topology_type, skipped_scripts_per_topology_type = get_pr_checker_scripts() + expanded_test_scripts = expand_test_scripts(test_scripts, test_scripts_per_topology_type, + skipped_scripts_per_topology_type) + + # Add additionally field to mark one running + trackid = str(uuid.uuid4()) + scantime = str(datetime.now()) + print(trackid) + + # Also, we will specify if the script belongs to data plane or control plane + for script in expanded_test_scripts: + script["trackid"] = trackid + script["scantime"] = scantime + if script["testscript"].split("/")[0] in DATAPLANE_FEATURES: + script["category"] = "data" + else: + script["category"] = "control" + upload_results(expanded_test_scripts) + + +if __name__ == '__main__': + main() diff --git a/.azure-pipelines/testscripts_analyse/constant.py b/.azure-pipelines/testscripts_analyse/constant.py new file mode 100644 index 0000000000..72538c5808 --- /dev/null +++ b/.azure-pipelines/testscripts_analyse/constant.py @@ -0,0 +1,27 @@ +DATAPLANE_FEATURES = { + "acl", "arp", "bfd", "copp", "crm", "dash", "decap", "drop_packets", "dualtor", "dualtor_io", + "ecmp", "everflow", "fdb", "fib", "flow_counter", "ip", "ipfwd", "ixia", "macsec", "mclag", + "mpls", "nat", "pfc_asym", "pfcwd", "qos", "radv", "read_mac", "route", "sai_qualify", "sflow", + "snappi_tests", "span", "stress", "upgrade_path", "vlan", "voq", "vrf", "vs_voq_cfgs", "vxlan", "wan" +} + +# We temporarily set four types of PR checker here +PR_TOPOLOGY_TYPE = ["t0", "t1", "t2", "dpu", "tgen", "snappi", "ptf"] + +# Map the topology name and topology type in pr_test_scripts.yaml +# Key is the topology name in pr_test_scripts.yaml and the value is topology type +PR_TOPOLOGY_MAPPING = { + "t0": "t0", + "t0-2vlans": "t0", + "t0-sonic": "t0", + "dualtor": "t0", + "t1-lag": "t1", + "multi-asic-t1-lag": "t1", + "t2": "t2", + "wan-pub": "wan", + "dpu": "dpu", + "tgen": "tgen", + "multidut-tgen": "tgen", + "snappi": "snappi", + "ptf": "ptf" +} diff --git a/.azure-pipelines/testscripts_analyse/report_data_storage.py b/.azure-pipelines/testscripts_analyse/report_data_storage.py new file mode 100644 index 0000000000..64bf111e94 --- /dev/null +++ b/.azure-pipelines/testscripts_analyse/report_data_storage.py @@ -0,0 +1,76 @@ +"""Wrappers and utilities for storing test reports.""" +import json +import os +import tempfile + +from azure.kusto.data import KustoConnectionStringBuilder +from azure.kusto.ingest import QueuedIngestClient as KustoIngestClient +from azure.kusto.ingest import IngestionProperties +from azure.kusto.data.data_format import DataFormat +from datetime import datetime + + +class KustoConnector(): + """KustoReportDB is a wrapper for storing test reports in Kusto/Azure Data Explorer.""" + + TESTSCRIPT_TABLE = "TestScripts" + + TABLE_FORMAT_LOOKUP = { + TESTSCRIPT_TABLE: DataFormat.JSON, + } + + TABLE_MAPPING_LOOKUP = { + TESTSCRIPT_TABLE: "TestScriptsMapping" + } + + def __init__(self, db_name: str): + """Initialize a Kusto report DB connector. + + Args: + db_name: The Kusto database to connect to. + """ + self.db_name = db_name + + """ + Kusto performance depends on the work load of cluster, + to improve the high availability of test result data service + by hosting a backup cluster, which is optional. + """ + ingest_cluster = os.getenv("TEST_REPORT_INGEST_KUSTO_CLUSTER_BACKUP") + access_token = os.getenv('ACCESS_TOKEN', None) + + if not ingest_cluster or not access_token: + raise RuntimeError( + "Could not load Kusto Credentials from environment") + else: + kcsb = KustoConnectionStringBuilder.with_aad_application_token_authentication(ingest_cluster, access_token) + self._ingestion_client_backup = KustoIngestClient(kcsb) + + def upload_testscripts(self, test_scripts): + uploadtime = str(datetime.now()) + + for script in test_scripts: + script["uploadtime"] = uploadtime + + print("Upload test scripts") + self._ingest_data(self.TESTSCRIPT_TABLE, test_scripts) + + def _ingest_data(self, table, data): + props = IngestionProperties( + database=self.db_name, + table=table, + data_format=self.TABLE_FORMAT_LOOKUP[table], + ingestion_mapping_reference=self.TABLE_MAPPING_LOOKUP[table] + ) + + with tempfile.NamedTemporaryFile(mode="w+") as temp: + if isinstance(data, list): + temp.writelines( + '\n'.join([json.dumps(entry) for entry in data])) + else: + temp.write(json.dumps(data)) + temp.seek(0) + if self._ingestion_client_backup: + print("Ingest to backup cluster...") + self._ingestion_client_backup.ingest_from_file( + temp.name, ingestion_properties=props) diff --git a/.azure-pipelines/testscripts_analyse/requirements.txt b/.azure-pipelines/testscripts_analyse/requirements.txt new file mode 100644 index 0000000000..29bfbc0db2 --- /dev/null +++ b/.azure-pipelines/testscripts_analyse/requirements.txt @@ -0,0 +1,3 @@ +azure-kusto-data==3.1.3 +azure-kusto-ingest==3.1.3 +natsort diff --git a/.azure-pipelines/testscripts_analyse/setup.kql b/.azure-pipelines/testscripts_analyse/setup.kql index 1155a159f9..dd548c222b 100644 --- a/.azure-pipelines/testscripts_analyse/setup.kql +++ b/.azure-pipelines/testscripts_analyse/setup.kql @@ -5,13 +5,14 @@ # 2. Add a JSON mapping for the table # ############################################################################### .create table TestScripts (TestScript: string, Topology: string, Category: string, - PRCheckCovered: bool, ScanTime: datetime, UploadTime: datetime, + PRCheckCovered: bool, IsSkipped: bool, ScanTime: datetime, UploadTime: datetime, TrackId: string) .create table TestScripts ingestion json mapping 'TestScriptsMapping' @'[{"column":"TestScript","Properties":{"path":"$.testscript"}},' '{"column":"Topology","Properties":{"path":"$.topology"}},' '{"column":"Category","Properties":{"path":"$.category"}},' '{"column":"PRCheckCovered","Properties":{"path":"$.covered"}},' + '{"column":"IsSkipped","Properties":{"path":"$.skipped"}},' '{"column":"ScanTime","Properties":{"path":"$.scantime"}},' '{"column":"UploadTime","Properties":{"path":"$.uploadtime"}},' '{"column":"TrackId","Properties":{"path":"$.trackid"}}]' diff --git a/.azure-pipelines/testscripts_analyse/testscripts.collection.yml b/.azure-pipelines/testscripts_analyse/testscripts.collection.yml new file mode 100644 index 0000000000..f9ba317ad8 --- /dev/null +++ b/.azure-pipelines/testscripts_analyse/testscripts.collection.yml @@ -0,0 +1,39 @@ +name: TestscriptsCollection_$(Build.DefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) + +trigger: none +pr: none + +schedules: + # Run at GME+4 everyday + - cron: "0 4 * * *" + displayName: Testscripts Collection + branches: + include: + - master + always: true + +stages: + +- stage: TestscriptsCollection + jobs: + - job: TestscriptsCollection + variables: + - group: KUSTO_SECRETS + + steps: + - task: Bash@3 + displayName: Testscripts Collection + inputs: + targetType: 'inline' + script: | + set -x + + cd .azure-pipelines/testscripts_analyse + pip install -r requirements.txt + python analyse_testscripts.py "../../tests" SonicTestData + + env: + TEST_REPORT_INGEST_KUSTO_CLUSTER_BACKUP: $(TEST_REPORT_INGEST_KUSTO_CLUSTER_BACKUP) + TEST_REPORT_AAD_TENANT_ID_BACKUP: $(TEST_REPORT_AAD_TENANT_ID_BACKUP) + TEST_REPORT_AAD_CLIENT_ID_BACKUP: $(TEST_REPORT_AAD_CLIENT_ID_BACKUP) + TEST_REPORT_AAD_CLIENT_KEY_BACKUP: $(TEST_REPORT_AAD_CLIENT_KEY_BACKUP) diff --git a/.github/ISSUE_TEMPLATE/02-testgap.yml b/.github/ISSUE_TEMPLATE/02-testgap.yml new file mode 100644 index 0000000000..253e80c316 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/02-testgap.yml @@ -0,0 +1,63 @@ +name: Test Gap Issue Template +description: File a test gap issue. +title: "[Test Gap][][]" +labels: ["Test Gap"] +projects: ["sonic-mgmt"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this test gap issue! + + If you are reporting a new test gap, make sure that we do not have any duplicates already open. You can ensure this by searching the issue list in this repository. If there is a duplicate, please close your issue and add a comment to the existing issue instead. + + - type: markdown + id: tc-contact-links + attributes: + value: | + Please ask and answer questions here at the [SONiC Support Forums](https://groups.google.com/forum/#!forum/sonicproject) + + - type: textarea + id: tc-description + attributes: + label: Test Gap Description + description: Describe the test gap in the test coverage. + placeholder: | + Why do we need to address this test gap? + What part of the functionality is not covered by existing tests? + Explain the impact of this test gap. What potential issues or risks could introduce from not having this test? + validations: + required: true + + - type: textarea + id: tc-test-plan + attributes: + label: Test Plan + description: Describe the test plan to address this test gap. + placeholder: | + What test cases need to be added or modified? + What is the test coverage goal? + What is the expected outcome? + Attach any files or PR links if applicable. + validations: + required: false + + - type: textarea + id: tc-test-environment + attributes: + label: Test environment + description: Provide details about the test environment where the test gap was identified. + placeholder: | + SONiC version + Platform + Topology + HardwareSKU + Testbed setup details + validations: + required: false + + - type: textarea + id: tc-attachments + attributes: + label: Attach files (if any) + description: If applicable, add logs or screenshots to help explain this test gap issue. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 42ad004577..11cd58d3ce 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -28,11 +28,11 @@ Fixes # (issue) ### Back port request -- [ ] 201911 - [ ] 202012 - [ ] 202205 - [ ] 202305 - [ ] 202311 +- [ ] 202405 ### Approach #### What is the motivation for this PR? diff --git a/.gitignore b/.gitignore index 0af6e40757..128f40544b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ ansible.log tests/logs tests/_cache tests/metadata +ansible/plugins/*/*.pyc +# Temporary trimmed inventory file +ansible/*_tmp # Dev tools .vscode/ diff --git a/.hooks/pre_commit_hooks/check_conditional_mark_sort.py b/.hooks/pre_commit_hooks/check_conditional_mark_sort.py index fac50d255f..2c1f480b72 100755 --- a/.hooks/pre_commit_hooks/check_conditional_mark_sort.py +++ b/.hooks/pre_commit_hooks/check_conditional_mark_sort.py @@ -4,7 +4,6 @@ def main(): stage_files = sys.argv[1:] - retval = 0 for stage_file in stage_files: if "tests_mark_conditions" in stage_file: conditions = [] @@ -17,11 +16,21 @@ def main(): conditions.append(line.strip().rstrip(":")) sorted_conditions = conditions[:] sorted_conditions.sort() - if conditions != sorted_conditions: - print("The entries in tests/common/plugins/conditional_mark/tests_mark_conditions*.yaml " - "are not sorted in alphabetic order.") - retval = 1 - return retval + for i in range(len(conditions)): + if conditions[i] != sorted_conditions[i]: + print("The entries in tests/common/plugins/conditional_mark/tests_mark_conditions*.yaml " + "are not sorted in alphabetic order, please adjust the order before commit") + print("===========================================================================") + print("File: {}".format(stage_file)) + print("===========================================================================") + print("Conditional marks before sort: {}".format(conditions)) + print("Conditional marks after sort: {}".format(sorted_conditions)) + print("===========================================================================") + print("Mismatch item, before sort: {}, after sort: {}".format(conditions[i], + sorted_conditions[i])) + print("===========================================================================") + return 1 + return 0 if __name__ == "__main__": diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c35c738df3..3ba3117260 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,6 +22,6 @@ repos: args: ["--max-line-length=120", "--ignore=E1,E2,E3,E5,E7,W5"] - repo: https://github.com/sonic-net/sonic-mgmt - rev: 1.0.0+pre_commit + rev: 1.0.1+pre_commit hooks: - id: check-conditional-mark-sort diff --git a/ansible/TestbedProcessing.py b/ansible/TestbedProcessing.py index a61f069c37..0ae253ce39 100755 --- a/ansible/TestbedProcessing.py +++ b/ansible/TestbedProcessing.py @@ -392,10 +392,12 @@ def makeFanoutSecrets(data, outfile): for key, value in devices.items(): if "fanout" in value.get("device_type").lower(): - result.update({"ansible_ssh_user": value.get( - "ansible").get("ansible_ssh_user")}) - result.update({"ansible_ssh_pass": value.get( - "ansible").get("ansible_ssh_pass")}) + result.update({"ansible_ssh_user": value.get("ansible").get("ansible_ssh_user")}) + result.update({"ansible_ssh_pass": value.get("ansible").get("ansible_ssh_pass")}) + result.update({"fanout_network_user": value.get("ansible").get("fanout_network_user")}) + result.update({"fanout_network_password": value.get("ansible").get("fanout_network_password")}) + result.update({"fanout_shell_user": value.get("ansible").get("ansible_ssh_user")}) + result.update({"fanout_shell_password": value.get("ansible").get("ansible_ssh_pass")}) with open(outfile, "w") as toWrite: yaml.dump(result, stream=toWrite, default_flow_style=False) @@ -466,6 +468,14 @@ def makeLab(data, devices, testbed, outfile): else: dev = None + try: + ansible_hostv6 = dev.get( + "ansible").get("ansible_hostv6") + entry += "\tansible_hostv6=" + \ + ansible_hostv6.split("/")[0] + except Exception: + print("\t\t" + host + ": ansible_hostv6 not found") + if "ptf" in key: try: # get ansible host ansible_host = dev.get( @@ -475,14 +485,6 @@ def makeLab(data, devices, testbed, outfile): except Exception: print("\t\t" + host + ": ansible_host not found") - try: - ansible_hostv6 = dev.get( - "ansible").get("ansible_hostv6") - entry += "\tansible_hostv6=" + \ - ansible_hostv6.split("/")[0] - except Exception: - print("\t\t" + host + ": ansible_hostv6 not found") - if ansible_host: try: # get ansible ssh username ansible_ssh_user = dev.get( diff --git a/ansible/config_sonic_basedon_testbed.yml b/ansible/config_sonic_basedon_testbed.yml index 4f94bd7506..0ea25ed16a 100644 --- a/ansible/config_sonic_basedon_testbed.yml +++ b/ansible/config_sonic_basedon_testbed.yml @@ -205,7 +205,7 @@ - name: gather hwsku that supports ComputeAI deployment set_fact: - hwsku_list_compute_ai: "['Cisco-8111-O64']" + hwsku_list_compute_ai: "['Cisco-8111-O64', 'Cisco-8111-O32', 'Cisco-8122-O64', 'Cisco-8122-O128']" - name: enable ComputeAI deployment set_fact: @@ -413,12 +413,27 @@ become: true when: proxy_env is defined and deploy is defined and deploy|bool == true + - name: Enable PTF tacacs server by default + set_fact: + use_ptf_tacacs_server: true + tacacs_enabled_by_default: true + when: use_ptf_tacacs_server is not defined + + - debug: msg="use_ptf_tacacs_server {{ use_ptf_tacacs_server }}" + - block: - name: saved original minigraph file in SONiC DUT(ignore errors when file does not exist) shell: mv /etc/sonic/minigraph.xml /etc/sonic/minigraph.xml.orig become: true ignore_errors: true + - name: Update TACACS server address to PTF IP + set_fact: + tacacs_servers: ["{{ testbed_facts['ptf_ip'] }}"] + when: use_ptf_tacacs_server is defined and use_ptf_tacacs_server|bool == true + + - debug: msg="tacacs_servers {{ tacacs_servers }}" + - name: create new minigraph file for SONiC device template: src=templates/minigraph_template.j2 dest=/etc/sonic/minigraph.xml @@ -523,9 +538,35 @@ path: /etc/sonic/port_config.json state: absent + - name: find interface name mapping + port_alias: + hwsku: "{{ hwsku }}" + when: topo == "mx" + + - name: Copy dhcp_server config hwsku {{ hwsku }} + copy: src=golden_config_db/dhcp_server_mx.json + dest=/tmp/dhcp_server.json + become: true + when: topo == "mx" + + - name: Generate golden_config_db.json + generate_golden_config_db: + topo_name: "{{ topo }}" + port_index_map: "{{ port_index_map | default({}) }}" + become: true + + - name: execute cli "config load_minigraph --override_config -y" to apply new minigraph + become: true + shell: config load_minigraph --override_config -y + register: load_minigraph_result + failed_when: + - load_minigraph_result.rc != 0 + - "'no such option: --override_config' not in load_minigraph_result.stderr" + - name: execute cli "config load_minigraph -y" to apply new minigraph become: true shell: config load_minigraph -y + when: "'no such option: --override_config' in load_minigraph_result.stderr" - name: Wait for switch to become reachable again become: false @@ -589,6 +630,24 @@ ignore_errors: true when: tacacs_enabled_by_default is defined and tacacs_enabled_by_default|bool == true + - block: + - name: Configure TACACS with PTF TACACS server + become: true + shell: "{{ tacacs_config_cmd }}" + loop: + - config tacacs authtype login + - config aaa authorization tacacs+ + - config aaa accounting "tacacs+ local" + loop_control: + loop_var: tacacs_config_cmd + ignore_errors: true + + - name: Configure TACACS with PTF TACACS server for MX + become: true + shell: config aaa authorization "tacacs+ local" + when: topo == "mx" + when: use_ptf_tacacs_server is defined and use_ptf_tacacs_server|bool == true + - name: execute configlet application script, which applies configlets in strict order. become: true shell: bash "/etc/sonic/apply_clet.sh" diff --git a/ansible/devutil/device_inventory.py b/ansible/devutil/device_inventory.py index b9daf58125..c890e86830 100644 --- a/ansible/devutil/device_inventory.py +++ b/ansible/devutil/device_inventory.py @@ -22,6 +22,8 @@ def __init__( self.device_type = device_type self.protocol = protocol self.os = os + self.console_device = None + self.console_port = 0 @staticmethod def from_csv_row(row: List[str]) -> "DeviceInfo": @@ -63,6 +65,9 @@ def from_device_files(device_file_pattern: str) -> "List[DeviceInventory]": inv: List[DeviceInventory] = [] for file_path in glob.glob(device_file_pattern): device_inventory = DeviceInventory.from_device_file(file_path) + console_links_file_path = file_path.replace("_devices", "_console_links") + if os.path.exists(console_links_file_path): + device_inventory.load_console_links_info(console_links_file_path) inv.append(device_inventory) return inv @@ -89,5 +94,30 @@ def from_device_file(file_path: str) -> "DeviceInventory": return DeviceInventory(inv_name, file_path, devices) + def load_console_links_info(self, file_path: str): + print(f"Loading console links inventory: {file_path}") + + with open(file_path, newline="") as file: + reader = csv.reader(file) + + # Skip the header line + next(reader) + + for row in reader: + if row: + console_hostname = row[0] + console_port = int(row[1]) + device_hostname = row[2] + console_device_info = self.get_device(console_hostname) + device_info = self.get_device(device_hostname) + if not console_device_info: + print(f"Unknown console hostname {console_hostname}, skipping") + continue + if not device_info: + print(f"Unknown device hostname {device_hostname}, skipping") + continue + device_info.console_device = console_device_info + device_info.console_port = console_port + def get_device(self, hostname: str) -> Optional[DeviceInfo]: return self.devices.get(hostname) diff --git a/ansible/devutil/inv_helpers.py b/ansible/devutil/inv_helpers.py index e91131496c..e6accc827f 100644 --- a/ansible/devutil/inv_helpers.py +++ b/ansible/devutil/inv_helpers.py @@ -66,6 +66,8 @@ def get_host_vars(self, hostname): @return: A dict of hostvars """ host = self._inv_mgr.get_host(hostname) + if not host: + raise Exception("Host not found in inventory files") vars = self._var_mgr.get_vars(host=host) vars['creds'] = self.get_host_creds(hostname) vars.update(host.vars) diff --git a/ansible/devutil/ssh_session_repo.py b/ansible/devutil/ssh_session_repo.py index da95716b0a..8324c660c3 100644 --- a/ansible/devutil/ssh_session_repo.py +++ b/ansible/devutil/ssh_session_repo.py @@ -34,7 +34,8 @@ def _load_template(self, template_file): """ raise NotImplementedError - def generate(self, session_path, ssh_ip, ssh_user, ssh_pass): + def generate(self, session_path, ssh_ip, ssh_ipv6, ssh_user, ssh_pass, + console_ssh_ip, console_ssh_port, console_ssh_user, console_ssh_pass): """Generate SSH session for a node. This is a virtual method that should be implemented by child class. @@ -102,30 +103,40 @@ def _load_template(self, template_file): return template - def generate(self, session_path, ssh_ip, ssh_user, ssh_pass): + def generate(self, session_path, ssh_ip, ssh_ipv6, ssh_user, ssh_pass, + console_ssh_ip, console_ssh_port, console_ssh_user, console_ssh_pass): """Generate SSH session for a testbed node.""" - # In SecureCRT, every SSH session is stored in a ini file separately, - # hence we add .ini extension to the session path in order to generate individual SSH session file. - ssh_session_file_path = os.path.join(self.target, session_path + ".ini") - - # Recursively create SSH session file directory - ssh_session_folder = os.path.dirname(ssh_session_file_path) - self._create_ssh_session_folder(ssh_session_folder) - - # Generate SSH session file - ssh_session_name = os.path.basename(session_path) - ssh_session_file_content = self._generate_ssh_session_file_content( - ssh_session_name, ssh_ip, ssh_user, ssh_pass - ) - with open(ssh_session_file_path, "w") as ssh_session_file: - ssh_session_file.write(ssh_session_file_content) + session_file_matrix = [ + (session_path, ssh_ip, ssh_user, ssh_pass), + (session_path + "-v6", ssh_ipv6, ssh_user, ssh_pass), + (session_path + "-console", console_ssh_ip, f"{console_ssh_user}:{console_ssh_port}", console_ssh_pass), + ] + + for (session_name, ip, user, password) in session_file_matrix: + if not ip or not user: + continue + + # In SecureCRT, every SSH session is stored in a ini file separately, + # hence we add .ini extension to the session path in order to generate individual SSH session file. + ssh_session_file_path = os.path.join(self.target, session_name + ".ini") + + # Recursively create SSH session file directory + ssh_session_folder = os.path.dirname(ssh_session_file_path) + self._create_ssh_session_folder(ssh_session_folder) + + # Generate SSH session file + ssh_session_file_content = self._generate_ssh_session_file_content( + session_name, ip, user, password + ) + with open(ssh_session_file_path, "w") as ssh_session_file: + ssh_session_file.write(ssh_session_file_content) - # Add newly created session file into current folder data - ssh_session_folder_data = SecureCRTRepoFolderData.from_folder( - ssh_session_folder, create_if_not_exist=True - ) - ssh_session_folder_data.add_session(ssh_session_name) - ssh_session_folder_data.save() + # Add newly created session file into current folder data + ssh_session_folder_data = SecureCRTRepoFolderData.from_folder( + ssh_session_folder, create_if_not_exist=True + ) + ssh_session_folder_data.add_session(session_name) + ssh_session_folder_data.save() def _create_ssh_session_folder(self, ssh_session_file_dir): """Recursively create SSH session file directory level by level if it does not exist, @@ -326,7 +337,7 @@ class SshConfigSshSessionRepoGenerator(SshSessionRepoGenerator): It derives from SshSessionRepoGenerator and implements the generate method. """ - def __init__(self, target, ssh_config_params): + def __init__(self, target, ssh_config_params, console_ssh_config_params): super().__init__(target, "") # Load SSH config file from target file path @@ -338,6 +349,7 @@ def __init__(self, target, ssh_config_params): # Add SSH config parameters self.ssh_config_params = ssh_config_params + self.console_ssh_config_params = console_ssh_config_params def _load_template(self, template_file): """Load SSH session template file. @@ -346,19 +358,41 @@ def _load_template(self, template_file): """ pass - def generate(self, session_path, ssh_ip, ssh_user, ssh_pass): + def generate(self, session_path, ssh_ip, ssh_ipv6, ssh_user, ssh_pass, + console_ssh_ip, console_ssh_port, console_ssh_user, console_ssh_pass): """Generate SSH session for a testbed node.""" ssh_session_name = os.path.basename(session_path) - # Remove existing host config if it exists - try: - self.ssh_config.remove(ssh_session_name) - except ValueError: - pass + current_hosts = self.ssh_config.hosts() + ssh_config = {} + if ssh_user: + ssh_config["User"] = ssh_user # Add new host config - self.ssh_config.add(ssh_session_name, Hostname=ssh_ip, User=ssh_user, - PasswordAuthentication="yes", **self.ssh_config_params) + if ssh_ip: + session_name = ssh_session_name + ssh_config["Hostname"] = ssh_ip + if session_name in current_hosts: + self.ssh_config.set(session_name, **ssh_config, **self.ssh_config_params) + else: + self.ssh_config.add(session_name, **ssh_config, **self.ssh_config_params) + if ssh_ipv6: + session_name = ssh_session_name + "-v6" + ssh_config["Hostname"] = ssh_ipv6 + if session_name in current_hosts: + self.ssh_config.set(session_name, **ssh_config, **self.ssh_config_params) + else: + self.ssh_config.add(session_name, **ssh_config, **self.ssh_config_params) + if console_ssh_ip: + session_name = ssh_session_name + "-console" + ssh_config["User"] = f"{console_ssh_user}:{console_ssh_port}" + ssh_config["Hostname"] = console_ssh_ip + if session_name in current_hosts: + self.ssh_config.set(session_name, **ssh_config, **self.ssh_config_params, + **self.console_ssh_config_params) + else: + self.ssh_config.add(session_name, **ssh_config, **self.ssh_config_params, + **self.console_ssh_config_params) def finish(self): """Finish SSH session generation.""" diff --git a/ansible/files/graph_groups.yml b/ansible/files/graph_groups.yml index b3ef72b824..4bfdc2f9ab 100644 --- a/ansible/files/graph_groups.yml +++ b/ansible/files/graph_groups.yml @@ -1,3 +1,3 @@ --- - lab - - example_ixia + - snappi-sonic diff --git a/ansible/files/sonic_example_ixia_devices.csv b/ansible/files/sonic_example_ixia_devices.csv deleted file mode 100644 index d057c88a98..0000000000 --- a/ansible/files/sonic_example_ixia_devices.csv +++ /dev/null @@ -1,4 +0,0 @@ -Hostname,ManagementIp,HwSku,Type -example-ixia-1,10.0.0.20/32,IXIA-tester,DevIxiaChassis -example-s6100-dut-1,10.0.0.10/32,Dell-S6100,DevSonic -example-ixia-api-serv-1,10.0.0.30/32,IXIA-tester,DevIxiaApiServ diff --git a/ansible/files/sonic_example_ixia_link.csv b/ansible/files/sonic_example_ixia_link.csv deleted file mode 100644 index 5d0ca2f34b..0000000000 --- a/ansible/files/sonic_example_ixia_link.csv +++ /dev/null @@ -1,4 +0,0 @@ -StartDevice,StartPort,EndDevice,EndPort,BandWidth,VlanID,VlanMode -example-s6100-dut-1,Ethernet0,example-ixia-1,Card1/Port1,40000,,Access -example-s6100-dut-1,Ethernet1,example-ixia-1,Card1/Port2,40000,,Access -example-s6100-dut-1,Ethernet2,example-ixia-1,Card1/Port3,40000,,Access diff --git a/ansible/files/sonic_snappi-sonic_devices.csv b/ansible/files/sonic_snappi-sonic_devices.csv new file mode 100644 index 0000000000..08ed6cf2b3 --- /dev/null +++ b/ansible/files/sonic_snappi-sonic_devices.csv @@ -0,0 +1,5 @@ +Hostname,ManagementIp,HwSku,Type +snappi-sonic,10.251.0.234/32,SNAPPI-tester,DevSnappiChassis +sonic-s6100-dut1,10.251.0.233/32,Arista-7060CX-32S-C32,DevSonic +sonic-s6100-dut2,10.251.0.234/32,Arista-7060CX-32S-C32,DevSonic +snappi-sonic-api-serv,10.251.0.232/32,SNAPPI-tester,DevSnappiApiServ diff --git a/ansible/files/sonic_snappi-sonic_links.csv b/ansible/files/sonic_snappi-sonic_links.csv new file mode 100644 index 0000000000..756f55fbe0 --- /dev/null +++ b/ansible/files/sonic_snappi-sonic_links.csv @@ -0,0 +1,9 @@ +StartDevice,StartPort,EndDevice,EndPort,BandWidth,VlanID,VlanMode +sonic-s6100-dut1,Ethernet0,snappi-sonic,Card1/Port1,100000,2,Access +sonic-s6100-dut1,Ethernet4,snappi-sonic,Card1/Port2,100000,2,Access +sonic-s6100-dut1,Ethernet8,snappi-sonic,Card1/Port3,100000,2,Access +sonic-s6100-dut1,Ethernet12,snappi-sonic,Card1/Port4,100000,2,Access +sonic-s6100-dut2,Ethernet0,snappi-sonic,Card1/Port5,100000,2,Access +sonic-s6100-dut2,Ethernet4,snappi-sonic,Card1/Port6,100000,2,Access +sonic-s6100-dut2,Ethernet8,snappi-sonic,Card1/Port7,100000,2,Access +sonic-s6100-dut2,Ethernet12,snappi-sonic,Card1/Port8,100000,2,Access diff --git a/ansible/golden_config_db/dhcp_server_mx.json b/ansible/golden_config_db/dhcp_server_mx.json new file mode 100644 index 0000000000..0524978d77 --- /dev/null +++ b/ansible/golden_config_db/dhcp_server_mx.json @@ -0,0 +1,243 @@ +{ + "DHCP_SERVER_IPV4": { + "Vlan1000": { + "gateway": "192.168.0.1", + "lease_time": "900", + "mode": "PORT", + "netmask": "255.255.255.0", + "state": "enabled" + } + }, + "DHCP_SERVER_IPV4_PORT": { + "Vlan1000|1": { + "ips": [ + "192.168.0.3" + ] + }, + "Vlan1000|2": { + "ips": [ + "192.168.0.4" + ] + }, + "Vlan1000|3": { + "ips": [ + "192.168.0.5" + ] + }, + "Vlan1000|4": { + "ips": [ + "192.168.0.6" + ] + }, + "Vlan1000|5": { + "ips": [ + "192.168.0.7" + ] + }, + "Vlan1000|6": { + "ips": [ + "192.168.0.8" + ] + }, + "Vlan1000|7": { + "ips": [ + "192.168.0.9" + ] + }, + "Vlan1000|8": { + "ips": [ + "192.168.0.10" + ] + }, + "Vlan1000|9": { + "ips": [ + "192.168.0.11" + ] + }, + "Vlan1000|10": { + "ips": [ + "192.168.0.12" + ] + }, + "Vlan1000|11": { + "ips": [ + "192.168.0.13" + ] + }, + "Vlan1000|12": { + "ips": [ + "192.168.0.14" + ] + }, + "Vlan1000|13": { + "ips": [ + "192.168.0.15" + ] + }, + "Vlan1000|14": { + "ips": [ + "192.168.0.16" + ] + }, + "Vlan1000|15": { + "ips": [ + "192.168.0.17" + ] + }, + "Vlan1000|16": { + "ips": [ + "192.168.0.18" + ] + }, + "Vlan1000|17": { + "ips": [ + "192.168.0.19" + ] + }, + "Vlan1000|18": { + "ips": [ + "192.168.0.20" + ] + }, + "Vlan1000|19": { + "ips": [ + "192.168.0.21" + ] + }, + "Vlan1000|20": { + "ips": [ + "192.168.0.22" + ] + }, + "Vlan1000|21": { + "ips": [ + "192.168.0.23" + ] + }, + "Vlan1000|22": { + "ips": [ + "192.168.0.24" + ] + }, + "Vlan1000|23": { + "ips": [ + "192.168.0.25" + ] + }, + "Vlan1000|24": { + "ips": [ + "192.168.0.26" + ] + }, + "Vlan1000|25": { + "ips": [ + "192.168.0.27" + ] + }, + "Vlan1000|26": { + "ips": [ + "192.168.0.28" + ] + }, + "Vlan1000|27": { + "ips": [ + "192.168.0.29" + ] + }, + "Vlan1000|28": { + "ips": [ + "192.168.0.30" + ] + }, + "Vlan1000|29": { + "ips": [ + "192.168.0.31" + ] + }, + "Vlan1000|30": { + "ips": [ + "192.168.0.32" + ] + }, + "Vlan1000|31": { + "ips": [ + "192.168.0.33" + ] + }, + "Vlan1000|32": { + "ips": [ + "192.168.0.34" + ] + }, + "Vlan1000|33": { + "ips": [ + "192.168.0.35" + ] + }, + "Vlan1000|34": { + "ips": [ + "192.168.0.36" + ] + }, + "Vlan1000|35": { + "ips": [ + "192.168.0.37" + ] + }, + "Vlan1000|36": { + "ips": [ + "192.168.0.38" + ] + }, + "Vlan1000|37": { + "ips": [ + "192.168.0.39" + ] + }, + "Vlan1000|38": { + "ips": [ + "192.168.0.40" + ] + }, + "Vlan1000|39": { + "ips": [ + "192.168.0.41" + ] + }, + "Vlan1000|40": { + "ips": [ + "192.168.0.42" + ] + }, + "Vlan1000|41": { + "ips": [ + "192.168.0.43" + ] + }, + "Vlan1000|42": { + "ips": [ + "192.168.0.44" + ] + }, + "Vlan1000|43": { + "ips": [ + "192.168.0.45" + ] + }, + "Vlan1000|44": { + "ips": [ + "192.168.0.46" + ] + }, + "Vlan1000|45": { + "ips": [ + "192.168.0.47" + ] + }, + "Vlan1000|46": { + "ips": [ + "192.168.0.48" + ] + } + } +} diff --git a/ansible/group_vars/all/ceos.yml b/ansible/group_vars/all/ceos.yml deleted file mode 100644 index 61559a09f8..0000000000 --- a/ansible/group_vars/all/ceos.yml +++ /dev/null @@ -1,12 +0,0 @@ -# upgrade cEOS to 4.25.5.1M from 4.23.2F at 2021/09/24 - -#ceos_image_filename: cEOS64-lab-4.23.2F.tar.xz -#ceos_image_orig: ceosimage:4.23.2F -#ceos_image: ceosimage:4.23.2F-1 -#ceos_image_filename: cEOS64-lab-4.25.5.1M.tar -#ceos_image_orig: ceosimage:4.25.5.1M -#ceos_image: ceosimage:4.25.5.1M-1 -ceos_image_filename: cEOS64-lab-4.29.3M.tar -ceos_image_orig: ceosimage:4.29.3M -ceos_image: ceosimage:4.29.3M-1 -skip_ceos_image_downloading: false diff --git a/ansible/group_vars/snappi-sonic/secrets.yml b/ansible/group_vars/snappi-sonic/secrets.yml new file mode 100644 index 0000000000..2b0b6b2fe6 --- /dev/null +++ b/ansible/group_vars/snappi-sonic/secrets.yml @@ -0,0 +1,5 @@ +ansible_ssh_pass: YourPaSsWoRd +ansible_become_pass: YourPaSsWoRd +sonicadmin_user: admin +sonicadmin_password: YourPaSsWoRd +sonicadmin_initial_password: YourPaSsWoRd diff --git a/ansible/group_vars/snappi-sonic/snappi-sonic.yml b/ansible/group_vars/snappi-sonic/snappi-sonic.yml new file mode 100644 index 0000000000..a8e7914a33 --- /dev/null +++ b/ansible/group_vars/snappi-sonic/snappi-sonic.yml @@ -0,0 +1,50 @@ +--- +#testlab (lab) group variables +# file: group_vars/lab.yml + +# ntp variables +ntp_servers: ['10.0.0.1', '10.0.0.2'] + +# syslog variables +syslog_servers: ['10.0.0.5', '10.0.0.6'] + +# dns variables +dns_servers: ['10.0.0.5', '10.0.0.6'] + +# forced_mgmt_routes +forced_mgmt_routes: [] + +# ErspanDestinationIpv4 +erspan_dest: ['10.0.0.7'] + +radius_servers: [] + +tacacs_servers: ['10.0.0.9', '10.0.0.8'] + +tacacs_passkey: testing123 +tacacs_rw_user: test_rwuser +tacacs_rw_user_passwd: '123456' +tacacs_ro_user: test_rouser +tacacs_ro_user_passwd: '123456' + +# tacacs grous +tacacs_group: 'testlab' + +# snmp servers +snmp_servers: ['10.0.0.9'] + +# dhcp relay servers +dhcp_servers: ['192.0.0.1', '192.0.0.2', '192.0.0.3', '192.0.0.4'] + +# snmp variables +snmp_rocommunity: public +snmp_location: testlab + +# bgp slb passive range +bgp_slb_passive_range: 10.255.0.0/25 + +#For Arista fanout switch deployment only +fanout_admin_user: "fanoutadminuser" +fanout_admin_password: "fanoutadminpassword" + +secret_group_vars: {snappi_api_server: {user: admin, password: admin, rest_port: 443, session_id: "None"}} diff --git a/ansible/group_vars/sonic/sku-sensors-data.yml b/ansible/group_vars/sonic/sku-sensors-data.yml index ba67a910de..58c5578476 100644 --- a/ansible/group_vars/sonic/sku-sensors-data.yml +++ b/ansible/group_vars/sonic/sku-sensors-data.yml @@ -2413,6 +2413,254 @@ sensors_checks: - dps460-i2c-4-59 sensor_skip_per_version: {} + x86_64-nvidia_sn4280-r0: + alarms: + fan: + + - dps460-i2c-4-58/PSU-2(R) Fan 1/fan1_alarm + - dps460-i2c-4-58/PSU-2(R) Fan 1/fan1_fault + + - mlxreg_fan-isa-0000/Chassis Fan Drawer-1 Tach 1/fan1_fault + - mlxreg_fan-isa-0000/Chassis Fan Drawer-2 Tach 1/fan2_fault + - mlxreg_fan-isa-0000/Chassis Fan Drawer-3 Tach 1/fan3_fault + - mlxreg_fan-isa-0000/Chassis Fan Drawer-4 Tach 1/fan4_fault + + - dps460-i2c-4-59/PSU-1(L) Fan 1/fan1_alarm + - dps460-i2c-4-59/PSU-1(L) Fan 1/fan1_fault + + power: + + - mp2975-i2c-5-6a/PMIC-4 12V ASIC VDD_T4_T7 VDDI_T4_T7 (in1)/in1_crit_alarm + - mp2975-i2c-5-6a/PMIC-4 12V ASIC VDD_T4_T7 VDDI_T4_T7 (in2)/in2_lcrit_alarm + - mp2975-i2c-5-6a/PMIC-4 12V ASIC VDD_T4_T7 VDDI_T4_T7 (in2)/in2_crit_alarm + - mp2975-i2c-5-6a/PMIC-4 ASIC VCORE_T2_3 Rail (out1)/in3_lcrit_alarm + - mp2975-i2c-5-6a/PMIC-4 ASIC VCORE_T2_3 Rail (out1)/in3_crit_alarm + - mp2975-i2c-5-6a/PMIC-4 12V ASIC VCORE_1.8V_T2_3 Rail Pwr (in) /power1_alarm + - mp2975-i2c-5-6a/PMIC-4 12V ASIC VCORE_T2_3 Rail Curr (in1)/curr1_alarm + - mp2975-i2c-5-6a/PMIC-4 12V ASIC 1.8V_T2_3 Rail Curr (in2)/curr2_alarm + - mp2975-i2c-5-6a/iout2/curr6_alarm + + - mp2975-i2c-5-64/PMIC-2 12V ASIC VDDI_M VDDHS_M (in)/in1_crit_alarm + - mp2975-i2c-5-64/PMIC-2 ASIC 1.8V_MAIN Rail (out1)/in2_lcrit_alarm + - mp2975-i2c-5-64/PMIC-2 ASIC 1.8V_MAIN Rail (out1)/in2_crit_alarm + - mp2975-i2c-5-64/PMIC-2 ASIC 1.2V_MAIN Rail (out2)/in3_lcrit_alarm + - mp2975-i2c-5-64/PMIC-2 ASIC 1.2V_MAIN Rail (out2)/in3_crit_alarm + - mp2975-i2c-5-64/PMIC-2 12V ASIC 1.8V_1.2V_MAIN Rail Pwr (in)/power1_alarm + - mp2975-i2c-5-64/PMIC-2 12V ASIC 1.8V_1.2V_MAIN Rail Curr (in)/curr1_alarm + - mp2975-i2c-5-64/PMIC-2 ASIC 1.8V_MAIN Rail Curr (out1)/curr2_alarm + + - mp2975-i2c-39-6a/PMIC-15 COMEX VDD_MEM INPUT VOLT/in1_crit_alarm + - mp2975-i2c-39-6a/PMIC-15 COMEX VDD_MEM OUTPUT VOLT/in2_lcrit_alarm + - mp2975-i2c-39-6a/PMIC-15 COMEX VDD_MEM OUTPUT VOLT/in2_crit_alarm + - mp2975-i2c-39-6a/PMIC-15 COMEX VDD_MEM INPUT POWER/power1_alarm + + - dps460-i2c-4-58/PSU-2(R) 220V Rail (in)/in1_min_alarm + - dps460-i2c-4-58/PSU-2(R) 220V Rail (in)/in1_max_alarm + - dps460-i2c-4-58/PSU-2(R) 220V Rail (in)/in1_lcrit_alarm + - dps460-i2c-4-58/PSU-2(R) 220V Rail (in)/in1_crit_alarm + - dps460-i2c-4-58/PSU-2(R) 12V Rail (out)/in3_min_alarm + - dps460-i2c-4-58/PSU-2(R) 12V Rail (out)/in3_max_alarm + - dps460-i2c-4-58/PSU-2(R) 12V Rail (out)/in3_lcrit_alarm + - dps460-i2c-4-58/PSU-2(R) 12V Rail (out)/in3_crit_alarm + - dps460-i2c-4-58/PSU-2(R) 220V Rail Pwr (in)/power1_alarm + - dps460-i2c-4-58/PSU-2(R) 12V Rail Pwr (out)/power2_max_alarm + - dps460-i2c-4-58/PSU-2(R) 12V Rail Pwr (out)/power2_crit_alarm + - dps460-i2c-4-58/PSU-2(R) 220V Rail Curr (in)/curr1_max_alarm + - dps460-i2c-4-58/PSU-2(R) 220V Rail Curr (in)/curr1_crit_alarm + - dps460-i2c-4-58/PSU-2(R) 12V Rail Curr (out)/curr2_max_alarm + - dps460-i2c-4-58/PSU-2(R) 12V Rail Curr (out)/curr2_crit_alarm + + - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_CPU VOLT (out1)/in2_lcrit_alarm + - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_CPU VOLT (out1)/in2_crit_alarm + - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_SOC VOLT (out2)/in3_lcrit_alarm + - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_SOC VOLT (out2)/in3_crit_alarm + - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_CPU CURR/curr1_alarm + - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_SOC CURR/curr2_alarm + + - dps460-i2c-4-59/PSU-1(L) 220V Rail (in)/in1_min_alarm + - dps460-i2c-4-59/PSU-1(L) 220V Rail (in)/in1_max_alarm + - dps460-i2c-4-59/PSU-1(L) 220V Rail (in)/in1_lcrit_alarm + - dps460-i2c-4-59/PSU-1(L) 220V Rail (in)/in1_crit_alarm + - dps460-i2c-4-59/PSU-1(L) 12V Rail (out)/in3_min_alarm + - dps460-i2c-4-59/PSU-1(L) 12V Rail (out)/in3_max_alarm + - dps460-i2c-4-59/PSU-1(L) 12V Rail (out)/in3_lcrit_alarm + - dps460-i2c-4-59/PSU-1(L) 12V Rail (out)/in3_crit_alarm + - dps460-i2c-4-59/PSU-1(L) 220V Rail Pwr (in)/power1_alarm + - dps460-i2c-4-59/PSU-1(L) 12V Rail Pwr (out)/power2_max_alarm + - dps460-i2c-4-59/PSU-1(L) 12V Rail Pwr (out)/power2_crit_alarm + - dps460-i2c-4-59/PSU-1(L) 220V Rail Curr (in)/curr1_max_alarm + - dps460-i2c-4-59/PSU-1(L) 220V Rail Curr (in)/curr1_crit_alarm + - dps460-i2c-4-59/PSU-1(L) 12V Rail Curr (out)/curr2_max_alarm + - dps460-i2c-4-59/PSU-1(L) 12V Rail Curr (out)/curr2_crit_alarm + + - mp2975-i2c-5-66/PMIC-3 12V ASIC VDD_T0_T3 VDDI_T0_T3 (in)/in1_crit_alarm + - mp2975-i2c-5-66/PMIC-3 ASIC VCORE_T0_3 Rail (out1)/in2_lcrit_alarm + - mp2975-i2c-5-66/PMIC-3 ASIC VCORE_T0_3 Rail (out1)/in2_crit_alarm + - mp2975-i2c-5-66/PMIC-3 ASIC 1.8V_T0_3 Rail (out2)/in3_lcrit_alarm + - mp2975-i2c-5-66/PMIC-3 ASIC 1.8V_T0_3 Rail (out2)/in3_crit_alarm + - mp2975-i2c-5-66/PMIC-3 12V ASIC VCORE_1.8V_T0_3 Rail Pwr (in) /power1_alarm + - mp2975-i2c-5-66/PMIC-3 12V ASIC VCORE_1.8V_T0_3 Rail Curr (in)/curr1_alarm + - mp2975-i2c-5-66/PMIC-3 ASIC VCORE_T0_3 Rail Curr (out1)/curr2_alarm + - mp2975-i2c-5-66/iout2/curr6_alarm + + - mp2975-i2c-5-62/PMIC-1 12V ASIC VDD_M (in)/in1_crit_alarm + - mp2975-i2c-5-62/PMIC-1 ASIC VCORE_MAIN Rail (out)/in2_lcrit_alarm + - mp2975-i2c-5-62/PMIC-1 ASIC VCORE_MAIN Rail (out)/in2_crit_alarm + - mp2975-i2c-5-62/PMIC-1 12V ASIC VCORE_MAIN Rail Pwr (in)/power1_alarm + - mp2975-i2c-5-62/PMIC-1 12V ASIC VCORE_MAIN Rail Curr (in)/curr1_alarm + - mp2975-i2c-5-62/PMIC-1 ASIC VCORE_MAIN Rail Curr (out)/curr2_alarm + + - mp2975-i2c-5-6e/PMIC-5 12V ASIC VDDHS_T0_T3 VDDHS_T4_T7 (in)/in1_crit_alarm + - mp2975-i2c-5-6e/PMIC-5 12V ASIC VDDHS_T0_T3 VDDHS_T4_T7 Rail (out1)/in2_lcrit_alarm + - mp2975-i2c-5-6e/PMIC-5 12V ASIC VDDHS_T0_T3 VDDHS_T4_T7 Rail (out1)/in2_crit_alarm + - mp2975-i2c-5-6e/PMIC-5 ASIC 1.2V_T4_7 Rail (out2)/in3_lcrit_alarm + - mp2975-i2c-5-6e/PMIC-5 ASIC 1.2V_T4_7 Rail (out2)/in3_crit_alarm + - mp2975-i2c-5-6e/PMIC-5 12V ASIC 1.2V_T0_3_T4_7 Rail Pwr (in)/power1_alarm + - mp2975-i2c-5-6e/PMIC-5 12V ASIC 1.2V_T0_3_T4_7 Rail Curr (in)/curr1_alarm + - mp2975-i2c-5-6e/PMIC-5 ASIC 1.2V_T0_3 Rail Curr (out1)/curr2_alarm + - mp2975-i2c-5-6e/iout2/curr5_alarm + + + temp: + + - jc42-i2c-43-1b/SODIMM2 Temp/temp1_max_alarm + - jc42-i2c-43-1b/SODIMM2 Temp/temp1_min_alarm + - jc42-i2c-43-1b/SODIMM2 Temp/temp1_crit_alarm + + - mp2975-i2c-5-6a/PMIC-4 ASIC VCORE_T2_3 Temp 1/temp1_max_alarm + - mp2975-i2c-5-6a/PMIC-4 ASIC VCORE_T2_3 Temp 1/temp1_crit_alarm + + - mp2975-i2c-5-64/PMIC-2 ASIC 1.8V_1.2V_MAIN Temp 1/temp1_max_alarm + - mp2975-i2c-5-64/PMIC-2 ASIC 1.8V_1.2V_MAIN Temp 1/temp1_crit_alarm + + - mp2975-i2c-39-6a/PMIC-15 COMEX VDD_MEM PHASE TEMP/temp1_max_alarm + - mp2975-i2c-39-6a/PMIC-15 COMEX VDD_MEM PHASE TEMP/temp1_crit_alarm + + - dps460-i2c-4-58/PSU-2(R) Temp 1/temp1_max_alarm + - dps460-i2c-4-58/PSU-2(R) Temp 1/temp1_min_alarm + - dps460-i2c-4-58/PSU-2(R) Temp 1/temp1_crit_alarm + - dps460-i2c-4-58/PSU-2(R) Temp 2/temp2_max_alarm + - dps460-i2c-4-58/PSU-2(R) Temp 2/temp2_min_alarm + - dps460-i2c-4-58/PSU-2(R) Temp 2/temp2_crit_alarm + - dps460-i2c-4-58/PSU-2(R) Temp 3/temp3_max_alarm + - dps460-i2c-4-58/PSU-2(R) Temp 3/temp3_min_alarm + - dps460-i2c-4-58/PSU-2(R) Temp 3/temp3_crit_alarm + + - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_CPU PHASE TEMP/temp1_crit_alarm + - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_SOC PHASE TEMP/temp2_crit_alarm + + - 00000800400-mdio-8/temp1/temp1_max_alarm + + - jc42-i2c-43-1e/SODIMM3 Temp/temp1_max_alarm + - jc42-i2c-43-1e/SODIMM3 Temp/temp1_min_alarm + - jc42-i2c-43-1e/SODIMM3 Temp/temp1_crit_alarm + + - jc42-i2c-43-1a/SODIMM1 Temp/temp1_max_alarm + - jc42-i2c-43-1a/SODIMM1 Temp/temp1_min_alarm + - jc42-i2c-43-1a/SODIMM1 Temp/temp1_crit_alarm + + - dps460-i2c-4-59/PSU-1(L) Temp 1/temp1_max_alarm + - dps460-i2c-4-59/PSU-1(L) Temp 1/temp1_min_alarm + - dps460-i2c-4-59/PSU-1(L) Temp 1/temp1_crit_alarm + - dps460-i2c-4-59/PSU-1(L) Temp 2/temp2_max_alarm + - dps460-i2c-4-59/PSU-1(L) Temp 2/temp2_min_alarm + - dps460-i2c-4-59/PSU-1(L) Temp 2/temp2_crit_alarm + - dps460-i2c-4-59/PSU-1(L) Temp 3/temp3_max_alarm + - dps460-i2c-4-59/PSU-1(L) Temp 3/temp3_min_alarm + - dps460-i2c-4-59/PSU-1(L) Temp 3/temp3_crit_alarm + + - mp2975-i2c-5-66/PMIC-3 ASIC VCORE_1.8V_T0_3 Temp 1/temp1_max_alarm + - mp2975-i2c-5-66/PMIC-3 ASIC VCORE_1.8V_T0_3 Temp 1/temp1_crit_alarm + + - mp2975-i2c-5-62/PMIC-1 ASIC VCORE_MAIN Temp 1/temp1_max_alarm + - mp2975-i2c-5-62/PMIC-1 ASIC VCORE_MAIN Temp 1/temp1_crit_alarm + + - mp2975-i2c-5-6e/PMIC-5 ASIC 1.2V_T0_3_T4_7 Temp 1/temp1_max_alarm + - mp2975-i2c-5-6e/PMIC-5 ASIC 1.2V_T0_3_T4_7 Temp 1/temp1_crit_alarm + + - jc42-i2c-43-1f/SODIMM4 Temp/temp1_max_alarm + - jc42-i2c-43-1f/SODIMM4 Temp/temp1_min_alarm + - jc42-i2c-43-1f/SODIMM4 Temp/temp1_crit_alarm + + + compares: + power: [ ] + temp: + + - - jc42-i2c-43-1b/SODIMM2 Temp/temp1_input + - jc42-i2c-43-1b/SODIMM2 Temp/temp1_crit + + - - mp2975-i2c-5-6a/PMIC-4 ASIC VCORE_T2_3 Temp 1/temp1_input + - mp2975-i2c-5-6a/PMIC-4 ASIC VCORE_T2_3 Temp 1/temp1_crit + + - - mp2975-i2c-5-64/PMIC-2 ASIC 1.8V_1.2V_MAIN Temp 1/temp1_input + - mp2975-i2c-5-64/PMIC-2 ASIC 1.8V_1.2V_MAIN Temp 1/temp1_crit + + - - mp2975-i2c-39-6a/PMIC-15 COMEX VDD_MEM PHASE TEMP/temp1_input + - mp2975-i2c-39-6a/PMIC-15 COMEX VDD_MEM PHASE TEMP/temp1_crit + + - - dps460-i2c-4-58/PSU-2(R) Temp 1/temp1_input + - dps460-i2c-4-58/PSU-2(R) Temp 1/temp1_crit + + - - dps460-i2c-4-58/PSU-2(R) Temp 2/temp2_input + - dps460-i2c-4-58/PSU-2(R) Temp 2/temp2_crit + + - - dps460-i2c-4-58/PSU-2(R) Temp 3/temp3_input + - dps460-i2c-4-58/PSU-2(R) Temp 3/temp3_crit + + - - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_CPU PHASE TEMP/temp1_input + - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_CPU PHASE TEMP/temp1_crit + + - - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_SOC PHASE TEMP/temp2_input + - mp2855-i2c-39-69/PMIC-14 COMEX VDDCR_SOC PHASE TEMP/temp2_crit + + - - 00000800400-mdio-8/temp1/temp1_input + - 00000800400-mdio-8/temp1/temp1_crit + + - - jc42-i2c-43-1e/SODIMM3 Temp/temp1_input + - jc42-i2c-43-1e/SODIMM3 Temp/temp1_crit + + - - nvme-pci-0300/SSD Temp/temp1_input + - nvme-pci-0300/SSD Temp/temp1_crit + + - - adt75-i2c-7-4a/Ambient Port Side Temp (air exhaust)/temp1_input + - adt75-i2c-7-4a/Ambient Port Side Temp (air exhaust)/temp1_max + + - - jc42-i2c-43-1a/SODIMM1 Temp/temp1_input + - jc42-i2c-43-1a/SODIMM1 Temp/temp1_crit + + - - dps460-i2c-4-59/PSU-1(L) Temp 1/temp1_input + - dps460-i2c-4-59/PSU-1(L) Temp 1/temp1_crit + + - - dps460-i2c-4-59/PSU-1(L) Temp 2/temp2_input + - dps460-i2c-4-59/PSU-1(L) Temp 2/temp2_crit + + - - dps460-i2c-4-59/PSU-1(L) Temp 3/temp3_input + - dps460-i2c-4-59/PSU-1(L) Temp 3/temp3_crit + + - - mp2975-i2c-5-66/PMIC-3 ASIC VCORE_1.8V_T0_3 Temp 1/temp1_input + - mp2975-i2c-5-66/PMIC-3 ASIC VCORE_1.8V_T0_3 Temp 1/temp1_crit + + - - mp2975-i2c-5-62/PMIC-1 ASIC VCORE_MAIN Temp 1/temp1_input + - mp2975-i2c-5-62/PMIC-1 ASIC VCORE_MAIN Temp 1/temp1_crit + + - - mp2975-i2c-5-6e/PMIC-5 ASIC 1.2V_T0_3_T4_7 Temp 1/temp1_input + - mp2975-i2c-5-6e/PMIC-5 ASIC 1.2V_T0_3_T4_7 Temp 1/temp1_crit + + - - jc42-i2c-43-1f/SODIMM4 Temp/temp1_input + - jc42-i2c-43-1f/SODIMM4 Temp/temp1_crit + + - - adt75-i2c-6-49/Ambient Fan Side Temp (air intake)/temp1_input + - adt75-i2c-6-49/Ambient Fan Side Temp (air intake)/temp1_max + + + non_zero: + fan: [ ] + power: [ ] + temp: [ ] + psu_skips: { } + sensor_skip_per_version: { } + x86_64-mlnx_msn4700-r0: alarms: fan: diff --git a/ansible/group_vars/sonic/variables b/ansible/group_vars/sonic/variables index 663caf222d..f253d1d8d0 100644 --- a/ansible/group_vars/sonic/variables +++ b/ansible/group_vars/sonic/variables @@ -15,13 +15,14 @@ broadcom_th_hwskus: ['Force10-S6100', 'Arista-7060CX-32S-C32', 'Arista-7060CX-32 broadcom_th2_hwskus: ['Arista-7260CX3-D108C8', 'Arista-7260CX3-C64', 'Arista-7260CX3-Q64'] broadcom_th3_hwskus: ['DellEMC-Z9332f-M-O16C64', 'DellEMC-Z9332f-O32'] broadcom_th4_hwskus: ['Arista-7060DX5-32', 'Arista-7060DX5-64S'] +broadcom_th5_hwskus: ['Arista-7060X6-64DE', 'Arista-7060X6-64DE-64x400G', 'Arista-7060X6-64DE-256x200G', 'Arista-7060X6-64PE', 'Arista-7060X6-64PE-64x400G', 'Arista-7060X6-64PE-128x400G', 'Arista-7060X6-64PE-256x200G'] broadcom_j2c+_hwskus: ['Nokia-IXR7250E-36x100G', 'Nokia-IXR7250E-36x400G', 'Arista-7800R3A-36DM2-C36', 'Arista-7800R3A-36DM2-D36', 'Arista-7800R3AK-36DM2-C36', 'Arista-7800R3AK-36DM2-D36'] broadcom_jr2_hwskus: ['Arista-7800R3-48CQ2-C48', 'Arista-7800R3-48CQM2-C48'] mellanox_spc1_hwskus: [ 'ACS-MSN2700', 'ACS-MSN2740', 'ACS-MSN2100', 'ACS-MSN2410', 'ACS-MSN2010', 'Mellanox-SN2700', 'Mellanox-SN2700-A1', 'Mellanox-SN2700-D48C8','Mellanox-SN2700-D40C8S8', 'Mellanox-SN2700-A1-D48C8'] mellanox_spc2_hwskus: [ 'ACS-MSN3700', 'ACS-MSN3700C', 'ACS-MSN3800', 'Mellanox-SN3800-D112C8' , 'ACS-MSN3420'] -mellanox_spc3_hwskus: [ 'ACS-MSN4700', 'Mellanox-SN4700-O28', 'ACS-MSN4600', 'ACS-MSN4600C', 'ACS-MSN4410', 'Mellanox-SN4600C-D112C8', 'Mellanox-SN4600C-C64', 'Mellanox-SN4700-O8C48', 'Mellanox-SN4700-O8V48', 'ACS-SN4280'] -mellanox_spc4_hwskus: [ 'ACS-SN5600' ] +mellanox_spc3_hwskus: [ 'ACS-MSN4700', 'Mellanox-SN4700-O28', 'ACS-MSN4600', 'ACS-MSN4600C', 'ACS-MSN4410', 'Mellanox-SN4600C-D112C8', 'Mellanox-SN4600C-C64', 'Mellanox-SN4700-O8C48', 'Mellanox-SN4700-O8V48', 'ACS-SN4280', 'Mellanox-SN4700-V64'] +mellanox_spc4_hwskus: [ 'ACS-SN5600' , 'Mellanox-SN5600-V256'] mellanox_hwskus: "{{ mellanox_spc1_hwskus + mellanox_spc2_hwskus + mellanox_spc3_hwskus + mellanox_spc4_hwskus }}" mellanox_dualtor_hwskus: [ 'Mellanox-SN4600C-C64' ] @@ -32,9 +33,10 @@ barefoot_hwskus: [ "montara", "mavericks", "Arista-7170-64C", "newport", "Arista marvell_hwskus: [ "et6448m" ] innovium_tl7_hwskus: ["Wistron_sw_to3200k_32x100" , "Wistron_sw_to3200k"] -cisco_hwskus: ["Cisco-8102-C64", "Cisco-8111-O32", "Cisco-8111-O64", "Cisco-8800-LC-48H-C48"] +cisco_hwskus: ["Cisco-8102-C64", "Cisco-8111-O32", "Cisco-8111-O64", "Cisco-8122-O64", "Cisco-8800-LC-48H-C48"] cisco-8000_gb_hwskus: ["Cisco-8102-C64", "Cisco-88-LC0-36FH-M-O36", "Cisco-8101-O8C48", "Cisco-8101-O32", "Cisco-88-LC0-36FH-O36"] cisco-8000_gr_hwskus: ["Cisco-8111-O32", "Cisco-8111-O64"] +cisco-8000_gr2_hwskus: ["Cisco-8122-O64"] cisco-8000_pac_hwskus: ["Cisco-8800-LC-48H-C48"] ## Note: diff --git a/ansible/group_vars/vm_host/ceos.yml b/ansible/group_vars/vm_host/ceos.yml new file mode 100644 index 0000000000..f6be555170 --- /dev/null +++ b/ansible/group_vars/vm_host/ceos.yml @@ -0,0 +1,11 @@ +ceos_image_filename: cEOS64-lab-4.29.3M.tar +ceos_image_orig: ceosimage:4.29.3M +ceos_image: ceosimage:4.29.3M-1 +# Please update ceos_image_url to the actual URL of the cEOS image file in your environment. If the cEOS image file +# is not available on test server, the cEOS image file will be downloaded from this URL. +# The ceos_image_url can be a string as single URL or a list of strings as multiple URLs. If it is a list, the code +# logic will automatically try each URL in the list +ceos_image_url: + - "http://example1.com/cEOS64-lab-4.29.3M.tar" + - "http://example2.com/cEOS64-lab-4.29.3M.tar" +skip_ceos_image_downloading: false diff --git a/ansible/group_vars/vm_host/main.yml b/ansible/group_vars/vm_host/main.yml index fbb59eee1f..b2e348e2c2 100644 --- a/ansible/group_vars/vm_host/main.yml +++ b/ansible/group_vars/vm_host/main.yml @@ -1,11 +1,5 @@ supported_vm_types: [ "veos", "ceos", "vsonic", "vcisco" ] root_path: veos-vm -vm_images_url: https://acsbe.blob.core.windows.net/vmimages -cd_image_filename: Aboot-veos-serial-8.0.0.iso -hdd_image_filename: vEOS-lab-4.20.15M.vmdk -sonic_image_filename: sonic-vs.img -cisco_image_filename: vIOS-xrv9k-goldenk9-x-7.3.4-20.qcow2 -skip_image_downloading: false vm_console_base: 7000 memory: 2097152 diff --git a/ansible/group_vars/vm_host/vcisco.yml b/ansible/group_vars/vm_host/vcisco.yml new file mode 100644 index 0000000000..726728a5fa --- /dev/null +++ b/ansible/group_vars/vm_host/vcisco.yml @@ -0,0 +1,10 @@ +vcisco_image_filename: vIOS-xrv9k-goldenk9-x-7.3.4-20.qcow2 +skip_vcisco_image_downloading: false + +# Please update url to the actual URL of the image file in your environment. If the image file +# is not available on test server, the file will be downloaded from the URLs. +# The url can be a string as single URL or a list of strings as multiple URLs. If it is a list, the code +# logic will automatically try each URL in the list +vcisco_image_url: + - http://example1.com/vIOS-xrv9k-goldenk9-x-7.3.4-20.qcow2 + - http://example2.com/vIOS-xrv9k-goldenk9-x-7.3.4-20.qcow2 diff --git a/ansible/group_vars/vm_host/veos.yml b/ansible/group_vars/vm_host/veos.yml new file mode 100644 index 0000000000..423522d3e4 --- /dev/null +++ b/ansible/group_vars/vm_host/veos.yml @@ -0,0 +1,20 @@ +# Two image files required for vEOS VMs: +# 1. cd file. +# 2. hdd file. +veos_cd_image_filename: Aboot-veos-serial-8.0.0.iso +veos_hdd_image_filename: vEOS-lab-4.20.15M.vmdk + +# Please update url to the actual URL of the veos image files in your environment. If the image files +# are not available on test server, the files will be downloaded from the URLs. +# The url can be a string as single URL or a list of strings as multiple URLs. If it is a list, the code +# logic will automatically try each URL in the list +veos_cd_image_url: + - http://example1.com/Aboot-veos-serial-8.0.0.iso + - http://example2.com/Aboot-veos-serial-8.0.0.iso +veos_hdd_image_url: + - http://example1.com/vEOS-lab-4.20.15M.vmdk + - http://example2.com/vEOS-lab-4.20.15M.vmdk + +# If the variable is set to true, the code logic will not try to download the image files from the URLs when the files +# are not available on test server +skip_veos_image_downloading: false diff --git a/ansible/group_vars/vm_host/vsonic.yml b/ansible/group_vars/vm_host/vsonic.yml new file mode 100644 index 0000000000..c796f21c7d --- /dev/null +++ b/ansible/group_vars/vm_host/vsonic.yml @@ -0,0 +1,10 @@ +vsonic_image_filename: sonic-vs.img +skip_vsonic_image_downloading: false + +# Please update url to the actual URL of the image file in your environment. If the image file +# is not available on test server, the file will be downloaded from the URLs. +# The url can be a string as single URL or a list of strings as multiple URLs. If it is a list, the code +# logic will automatically try each URL in the list +vsonic_image_url: + - http://example1.com/sonic-vs.img + - http://example2.com/sonic-vs.img diff --git a/ansible/lab b/ansible/lab index b226b28386..17096c265c 100644 --- a/ansible/lab +++ b/ansible/lab @@ -34,20 +34,35 @@ all: management-1: ansible_host: 192.168.10.3 os: sonic + server: + hosts: + server_1: + ansible_host: 10.254.0.1 + ansible_hostv6: 2001:db7:1::1/64 + pdu: + hosts: + pdu-1: + ansible_host: 192.168.9.2 + protocol: snmp + pdu-2: + ansible_host: 192.168.9.3 + protocol: snmp ptf: + vars: + ansible_ssh_user: root + ansible_ssh_pass: root hosts: ptf_ptf1: ansible_host: 10.255.0.188 - ansible_ssh_user: root - ansible_ssh_pass: root + ansible_hostv6: 2001:db8:1::1/64 ptf_vms1-1: ansible_host: 10.255.0.178 - ansible_ssh_user: root - ansible_ssh_pass: root - ptf_vms6-1: - ansible_host: 10.250.0.100 - ansible_ssh_user: root - ansible_ssh_pass: root + ptf_vms1-2: + ansible_host: 10.255.0.179 + ptf_vms1-3: + ansible_host: 10.255.0.180 + ptf_vms5-1: + ansible_host: 10.255.0.183 sonic_sn2700_40: vars: diff --git a/ansible/library/announce_routes.py b/ansible/library/announce_routes.py index aed26bc9bd..3128110b7e 100644 --- a/ansible/library/announce_routes.py +++ b/ansible/library/announce_routes.py @@ -10,9 +10,11 @@ import sys import socket import random +import logging import time from multiprocessing.pool import ThreadPool from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.debug_utils import config_module_logging if sys.version_info.major == 3: UNICODE_TYPE = str @@ -160,7 +162,8 @@ def change_routes(action, ptf_ip, port, routes): try: r = requests.post(url, data=data, timeout=360, proxies={"http": None, "https": None}) break - except ConnectionError as e: + except Exception as e: + logging.debug("Got exception {}, will try to connect again".format(e)) time.sleep(0.01 * (i+1)) if i == 4: raise e @@ -1037,10 +1040,14 @@ def main(): ptf_ip=dict(required=True, type='str'), action=dict(required=False, type='str', default='announce', choices=["announce", "withdraw"]), - path=dict(required=False, type='str', default='') + path=dict(required=False, type='str', default=''), + log_path=dict(required=False, type='str', default='') ), supports_check_mode=False) + if module.params['log_path']: + config_module_logging("announce_routes", log_path=module.params['log_path']) + topo_name = module.params['topo_name'] ptf_ip = module.params['ptf_ip'] action = module.params['action'] diff --git a/ansible/library/exabgp.py b/ansible/library/exabgp.py index 8159d583db..3a8b41de91 100644 --- a/ansible/library/exabgp.py +++ b/ansible/library/exabgp.py @@ -6,6 +6,7 @@ import os import re import time +import six DOCUMENTATION = ''' module: exabgp @@ -48,6 +49,7 @@ http_api_py = '''\ from flask import Flask, request import sys +import six #Disable banner msg from app.run, or the output might be caught by exabgp and run as command cli = sys.modules['flask.cli'] @@ -58,7 +60,15 @@ # Setup a command route to listen for prefix advertisements @app.route('/', methods=['POST']) def run_command(): - if request.form.has_key('commands'): + # code made compatible to run in Py2 or Py3 environment + # to support back-porting + request_has_commands = False + if six.PY2: + request_has_commands = request.form.has_key('commands') + else: + request_has_commands = 'commands' in request.form + + if request_has_commands: cmds = request.form['commands'].split(';') else: cmds = [ request.form['command'] ] @@ -82,7 +92,8 @@ def run_command(): } ''' -exabgp_conf_tmpl = '''\ +# ExaBGP Version 3 configuration file format +exabgp3_config_template = '''\ group exabgp { {{ dump_config }} @@ -105,6 +116,35 @@ def run_command(): } ''' +# ExaBGP Version 4 uses a different configuration file +# format. The dump_config would come from the user. The caller +# must pass Version 4 compatible configuration. +# Example configs are available here +# https://github.com/Exa-Networks/exabgp/tree/master/etc/exabgp +# Look for sample for a given section for details +exabgp4_config_template = '''\ +{{ dump_config }} +process http-api { + run /usr/bin/python /usr/share/exabgp/http_api.py {{ port }}; + encoder json; +} +neighbor {{ peer_ip }} { + router-id {{ router_id }}; + local-address {{ local_ip }}; + peer-as {{ peer_asn }}; + local-as {{ local_asn }}; + auto-flush {{ auto_flush }}; + group-updates {{ group_updates }}; + {%- if passive %} + passive; + listen {{ listen_port }}; + {%- endif %} + api { + processes [ http-api ]; + } +} +''' + exabgp_supervisord_conf_tmpl = '''\ [program:exabgp-{{ name }}] command=/usr/local/bin/exabgp /etc/exabgp/{{ name }}.conf @@ -132,7 +172,12 @@ def exec_command(module, cmd, ignore_error=False, msg="executing command"): def get_exabgp_status(module, name): output = exec_command(module, cmd="supervisorctl status exabgp-%s" % name) - m = re.search(r'^([\w|-]*)\s+(\w*).*$', output.decode("utf-8")) + m = None + if six.PY2: + m = re.search(r'^([\w|-]*)\s+(\w*).*$', output.decode("utf-8")) + else: + # For PY3 module.run_command encoding is "utf-8" by default + m = re.search(r'^([\w|-]*)\s+(\w*).*$', output) return m.group(2) @@ -182,7 +227,12 @@ def setup_exabgp_conf(name, router_id, local_ip, peer_ip, local_asn, peer_asn, p dump_config = jinja2.Template( dump_config_tmpl).render(dump_script=dump_script) - t = jinja2.Template(exabgp_conf_tmpl) + # backport friendly checking; not required if everything is Py3 + t = None + if six.PY2: + t = jinja2.Template(exabgp3_config_template) + else: + t = jinja2.Template(exabgp4_config_template) data = t.render(name=name, router_id=router_id, local_ip=local_ip, diff --git a/ansible/library/extract_log.py b/ansible/library/extract_log.py index d80bcb8d52..124f89cf56 100644 --- a/ansible/library/extract_log.py +++ b/ansible/library/extract_log.py @@ -151,6 +151,14 @@ def convert_date(fct, s): if len(re_result) > 0: str_date = re_result[0] dt = datetime.datetime.strptime(str_date, '%Y-%m-%d.%X.%f') + + else: + re_result = re.findall( + r'^\d{4} \w{3} \d{2} \d{2}:\d{2}:\d{2}\.\d{6}', s) + if len(re_result) > 0: + str_date = re_result[0] + dt = datetime.datetime.strptime(str_date, '%Y %b %d %H:%M:%S.%f') + locale.setlocale(locale.LC_ALL, loc) return dt diff --git a/ansible/library/generate_golden_config_db.py b/ansible/library/generate_golden_config_db.py new file mode 100644 index 0000000000..4341147896 --- /dev/null +++ b/ansible/library/generate_golden_config_db.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +# This ansible module is for generate golden_config_db.json +# Currently, only enable dhcp_server feature and generated related configuration in MX device +# which has dhcp_server feature. + + +import copy +import json + +from ansible.module_utils.basic import AnsibleModule + +DOCUMENTATION = ''' +module: generate_golden_config_db.py +author: Yaqiang Zhu (yaqiangzhu@microsoft.com) +short_description: Generate golden_config_db.json +Description: + When load_minigraph, SONiC support to use parameter --override_config to add configuration via + golden_config_db.json. This module is to generate required /etc/sonic/golden_config_db.json + Input: + topo_name: Name of current topo +''' + +GOLDEN_CONFIG_DB_PATH = "/etc/sonic/golden_config_db.json" +TEMP_DHCP_SERVER_CONFIG_PATH = "/tmp/dhcp_server.json" +DUMMY_QUOTA = "dummy_single_quota" + + +class GenerateGoldenConfigDBModule(object): + def __init__(self): + self.module = AnsibleModule(argument_spec=dict(topo_name=dict(required=True, type='str'), + port_index_map=dict(require=False, type='dict', default=None)), + supports_check_mode=True) + self.topo_name = self.module.params['topo_name'] + self.port_index_map = self.module.params['port_index_map'] + + def generate_mx_golden_config_db(self): + """ + If FEATURE table in init_cfg.json contains dhcp_server, enable it. + And add dhcp_server related configuration + """ + rc, out, err = self.module.run_command("sonic-cfggen -H -m -j /etc/sonic/init_cfg.json --print-data") + if rc != 0: + self.module.fail_json(msg="Failed to get config from minigraph: {}".format(err)) + + # Generate FEATURE table from init_cfg.ini + ori_config_db = json.loads(out) + if "FEATURE" not in ori_config_db or "dhcp_server" not in ori_config_db["FEATURE"]: + return "{}" + + ori_config_db["FEATURE"]["dhcp_server"]["state"] = "enabled" + gold_config_db = { + "FEATURE": copy.deepcopy(ori_config_db["FEATURE"]), + "PORT": copy.deepcopy(ori_config_db["PORT"]) + } + + # Generate dhcp_server related configuration + rc, out, err = self.module.run_command("cat {}".format(TEMP_DHCP_SERVER_CONFIG_PATH)) + if rc != 0: + self.module.fail_json(msg="Failed to get dhcp_server config: {}".format(err)) + if self.port_index_map is None or self.port_index_map == {}: + self.module.fail_json(msg="port_index_map is missing") + dhcp_server_config_obj = json.loads(out) + # Update DHCP_SERVER_IPV4_PORT based on port index map + dhcp_server_port_config = {} + for key, value in dhcp_server_config_obj["DHCP_SERVER_IPV4_PORT"].items(): + splits = key.split("|") + new_key = "{}|{}".format(splits[0], self.port_index_map[splits[1]]) + dhcp_server_port_config[new_key] = value + dhcp_server_config_obj["DHCP_SERVER_IPV4_PORT"] = dhcp_server_port_config + + gold_config_db.update(dhcp_server_config_obj) + return json.dumps(gold_config_db, indent=4) + + def generate(self): + if self.topo_name == "mx": + config = self.generate_mx_golden_config_db() + else: + config = "{}" + + with open(GOLDEN_CONFIG_DB_PATH, "w") as temp_file: + temp_file.write(config) + self.module.run_command("sudo rm -f {}".format(TEMP_DHCP_SERVER_CONFIG_PATH)) + self.module.exit_json(change=True, msg="Success to generate golden_config_db.json") + + +def main(): + generate_golden_config_db = GenerateGoldenConfigDBModule() + generate_golden_config_db.generate() + + +if __name__ == '__main__': + main() diff --git a/ansible/library/port_alias.py b/ansible/library/port_alias.py index 073c8c122d..0535d8e5f4 100755 --- a/ansible/library/port_alias.py +++ b/ansible/library/port_alias.py @@ -104,6 +104,7 @@ def get_portmap(self, asic_id=None, include_internal=False, portmap = {} aliasmap = {} portspeed = {} + indexmap = {} # Front end interface asic names front_panel_asic_ifnames = {} front_panel_asic_id = {} @@ -205,6 +206,8 @@ def get_portmap(self, asic_id=None, include_internal=False, sysport['asic_name'] = asic_name sysport['switchid'] = switchid sysports.append(sysport) + if port_index != -1 and len(mapping) > port_index: + indexmap[mapping[port_index]] = name if len(sysports) > 0: sysport = {} sysport['name'] = 'Cpu0' @@ -218,7 +221,7 @@ def get_portmap(self, asic_id=None, include_internal=False, sysports.insert(0, sysport) return (aliases, portmap, aliasmap, portspeed, front_panel_asic_ifnames, front_panel_asic_id, asic_if_names, - sysports) + sysports, indexmap) def main(): @@ -241,6 +244,7 @@ def main(): aliasmap = {} portspeed = {} sysports = [] + indexmap = {} # Map of ASIC interface names to front panel interfaces front_panel_asic_ifnames = {} front_panel_asic_ifs_asic_id = {} @@ -296,7 +300,7 @@ def main(): if num_asic == 1: asic_id = None (aliases_asic, portmap_asic, aliasmap_asic, portspeed_asic, front_panel_asic, front_panel_asic_ids, - asicifnames_asic, sysport_asic) = allmap.get_portmap( + asicifnames_asic, sysport_asic, index_name) = allmap.get_portmap( asic_id, include_internal, hostname, switchid, slotid) if aliases_asic is not None: aliases.extend(aliases_asic) @@ -315,6 +319,8 @@ def main(): asic_if_names[asic] = asicifnames_asic if sysport_asic is not None: sysports.extend(sysport_asic) + if index_name is not None: + indexmap.update(index_name) # Sort the Interface Name needed in multi-asic aliases.sort(key=lambda x: int(x[1])) @@ -335,7 +341,8 @@ def main(): 'front_panel_asic_ifnames': front_panel_asic_ifnames_list, 'front_panel_asic_ifs_asic_id': front_panel_asic_ifs_asic_id_list, 'asic_if_names': asic_if_names, - 'sysports': sysports}) + 'sysports': sysports, + 'port_index_map': indexmap}) except (IOError, OSError) as e: fail_msg = "IO error" + str(e) diff --git a/ansible/library/reduce_and_add_sonic_images.py b/ansible/library/reduce_and_add_sonic_images.py index 7d09b4354e..d86661ec3f 100644 --- a/ansible/library/reduce_and_add_sonic_images.py +++ b/ansible/library/reduce_and_add_sonic_images.py @@ -208,6 +208,11 @@ def install_new_sonic_image(module, new_image_url, save_as=None, required_space= ) return + skip_package_migrate_param = "" + _, output, _ = exec_command(module, cmd="sonic_installer install --help", ignore_error=True) + if "skip-package-migration" in output: + skip_package_migrate_param = "--skip-package-migration" + if save_as.startswith("/tmp/tmpfs"): log("Create a tmpfs partition to download image to install") exec_command(module, cmd="mkdir -p /tmp/tmpfs", ignore_error=True) @@ -222,7 +227,7 @@ def install_new_sonic_image(module, new_image_url, save_as=None, required_space= log("Running sonic_installer to install image at {}".format(save_as)) rc, out, err = exec_command( module, - cmd="sonic_installer install {} -y".format(save_as), + cmd="sonic_installer install {} {} -y".format(save_as, skip_package_migrate_param), msg="installing new image", ignore_error=True ) log("Done running sonic_installer to install image") @@ -241,8 +246,8 @@ def install_new_sonic_image(module, new_image_url, save_as=None, required_space= log("Running sonic_installer to install image at {}".format(save_as)) rc, out, err = exec_command( module, - cmd="sonic_installer install {} -y".format( - save_as), + cmd="sonic_installer install {} {} -y".format( + save_as, skip_package_migrate_param), msg="installing new image", ignore_error=True ) log("Always remove the downloaded temp image inside /host/ before proceeding") diff --git a/ansible/module_utils/port_utils.py b/ansible/module_utils/port_utils.py index 3abdbd0241..f2ea424499 100644 --- a/ansible/module_utils/port_utils.py +++ b/ansible/module_utils/port_utils.py @@ -103,9 +103,27 @@ def get_port_alias_to_name_map(hwsku, asic_name=None): or hwsku == "Arista-7050CX3-32S-C32": for i in range(1, 33): port_alias_to_name_map["Ethernet%d/1" % i] = "Ethernet%d" % ((i - 1) * 4) - elif hwsku in ["Arista-7060DX5-64S", "Arista-7060X6-64DE-64x400G"]: + elif hwsku in ["Arista-7060DX5-64S"]: for i in range(1, 65): port_alias_to_name_map["Ethernet%d/1" % i] = "Ethernet%d" % ((i - 1) * 8) + elif hwsku in ["Arista-7060X6-64DE", "Arista-7060X6-64DE-64x400G", + "Arista-7060X6-64PE", "Arista-7060X6-64PE-64x400G"]: + for i in range(1, 65): + port_alias_to_name_map["Ethernet%d/1" % i] = "Ethernet%d" % ((i - 1) * 8) + port_alias_to_name_map["Ethernet65"] = "Ethernet512" + port_alias_to_name_map["Ethernet66"] = "Ethernet513" + elif hwsku == "Arista-7060X6-64PE-128x400G": + for i in range(1, 65): + for j in [1, 5]: + port_alias_to_name_map["Ethernet%d/%d" % (i, j)] = "Ethernet%d" % ((i - 1) * 8 + j - 1) + port_alias_to_name_map["Ethernet65"] = "Ethernet512" + port_alias_to_name_map["Ethernet66"] = "Ethernet513" + elif hwsku == "Arista-7060X6-64PE-256x200G": + for i in range(1, 65): + for j in [1, 3, 5, 7]: + port_alias_to_name_map["Ethernet%d/%d" % (i, j)] = "Ethernet%d" % ((i - 1) * 8 + j - 1) + port_alias_to_name_map["Ethernet65"] = "Ethernet512" + port_alias_to_name_map["Ethernet66"] = "Ethernet513" elif hwsku == "Arista-7050QX32S-Q32": for i in range(5, 29): port_alias_to_name_map["Ethernet%d/1" % i] = "Ethernet%d" % ((i - 5) * 4) @@ -186,7 +204,7 @@ def get_port_alias_to_name_map(hwsku, asic_name=None): alias = "etp%d" % (i / 4 + 1) + ("a" if i % 4 == 0 else "b") # print alias, "Ethernet%d" % i port_alias_to_name_map[alias] = "Ethernet%d" % i - elif hwsku in ["ACS-MSN3800", "ACS-MSN4600C"]: + elif hwsku in ["ACS-MSN3800", "ACS-MSN4600C", 'Mellanox-SN4700-V64']: for i in range(1, 65): port_alias_to_name_map["etp%d" % i] = "Ethernet%d" % ((i - 1) * 4) elif hwsku == "Mellanox-SN2700" or hwsku == "ACS-MSN2700": @@ -215,8 +233,8 @@ def get_port_alias_to_name_map(hwsku, asic_name=None): s100G_ports = [x for x in range(45, 53)] + [64] port_alias_to_name_map = _port_alias_to_name_map_50G(all_ports, s100G_ports) - elif hwsku in ["Arista-7800R3-48CQ-LC", "Arista-7800R3K-48CQM2-LC", "Arista-7800R3K-48CQ-LC"]: - for i in range(1, 48): + elif hwsku in ["Arista-7800R3-48CQ2-C48", "Arista-7800R3-48CQM2-C48"]: + for i in range(1, 49): port_alias_to_name_map["Ethernet%d/1" % i] = "Ethernet%d" % ((i - 1) * 4) elif hwsku == "Arista-7800R3A-36DM2-C36" or hwsku == "Arista-7800R3A-36DM2-D36": for i in range(1, 36): @@ -267,12 +285,19 @@ def get_port_alias_to_name_map(hwsku, asic_name=None): elif hwsku == "Seastone-DX010": for i in range(1, 33): port_alias_to_name_map["Eth%d" % i] = "Ethernet%d" % ((i - 1) * 4) - elif hwsku in ["Celestica-E1031-T48S4", "Nokia-7215", "Nokia-M0-7215"]: + elif hwsku in ["Celestica-E1031-T48S4", "Nokia-7215", "Nokia-M0-7215", "Nokia-7215-A1"]: for i in range(1, 53): port_alias_to_name_map["etp%d" % i] = "Ethernet%d" % ((i - 1)) elif hwsku == "et6448m": for i in range(0, 52): port_alias_to_name_map["Ethernet%d" % i] = "Ethernet%d" % i + elif hwsku in ["rd98DX35xx_cn9131", "rd98DX35xx"]: + for i in range(0, 32): + port_alias_to_name_map["oneGigE%d" % i] = "Ethernet%d" % i + for i in range(32, 48): + port_alias_to_name_map["twod5GigE%d" % i] = "Ethernet%d" % i + for i in range(48, 54): + port_alias_to_name_map["twenty5GigE%d" % i] = "Ethernet%d" % i elif hwsku == "Nokia-IXR7250E-36x400G" or hwsku == "Nokia-IXR7250E-36x100G": for i in range(1, 37): sonic_name = "Ethernet%d" % ((i - 1) * 8) @@ -291,19 +316,10 @@ def get_port_alias_to_name_map(hwsku, asic_name=None): elif hwsku == "Cisco-8102-C64": for i in range(0, 64): port_alias_to_name_map["etp%d" % i] = "Ethernet%d" % (i * 4) - elif hwsku in ["Cisco-8101-T32", "Cisco-8101-O32", "Cisco-8111-C32", "Cisco-8111-O32"]: + elif hwsku in ["Cisco-8101-O32", "Cisco-8111-C32", "Cisco-8111-O32"]: for i in range(0, 32): port_alias_to_name_map["etp%d" % i] = "Ethernet%d" % (i * 8) - elif hwsku in ["Cisco-8101-C48T8", "Cisco-8101-O8C48"]: - for i in range(0, 24, 2): - port_alias_to_name_map["etp%d" % i] = "Ethernet%d" % (i * 4) - port_alias_to_name_map["etp%d" % (i+1)] = "Ethernet%d" % ((i+1) * 4) - for i in range(0, 8): - port_alias_to_name_map["etp%d" % (i+24)] = "Ethernet%d" % ((i+12) * 8) - for i in range(0, 24, 2): - port_alias_to_name_map["etp%d" % (i+32)] = "Ethernet%d" % ((i+40) * 4) - port_alias_to_name_map["etp%d" % (i+33)] = "Ethernet%d" % ((i+41) * 4) - elif hwsku in ["Cisco-8101-C64", "Cisco-8111-O64"]: + elif hwsku in ["Cisco-8111-O64"]: for i in range(0, 64): port_alias_to_name_map["etp%d" % i] = "Ethernet%d" % (i * 4) elif hwsku == "Cisco-8101-O8C48": @@ -319,6 +335,9 @@ def get_port_alias_to_name_map(hwsku, asic_name=None): for i in range(0, 32): port_alias_to_name_map["etp%da" % i] = "Ethernet%d" % (i * 4 * 2) port_alias_to_name_map["etp%db" % i] = "Ethernet%d" % ((i * 4 * 2) + 4) + elif hwsku in ["Cisco-8122-O64"]: + for i in range(0, 64): + port_alias_to_name_map["etp%d" % i] = "Ethernet%d" % (i * 8) elif hwsku in ["Cisco-8800-LC-48H-C48"]: for i in range(0, 48, 1): port_alias_to_name_map["Ethernet%d" % i] = "Ethernet%d" % (i * 4) @@ -354,6 +373,9 @@ def get_port_alias_to_name_map(hwsku, asic_name=None): elif hwsku in ["Wistron_sw_to3200k_32x100", "Wistron_sw_to3200k"]: for i in range(0, 256, 8): port_alias_to_name_map["Ethernet%d" % i] = "Ethernet%d" % i + elif hwsku in ["dbmvtx9180_64x100G"]: + for i in range(0, 505, 8): + port_alias_to_name_map["Ethernet%d" % i] = "Ethernet%d" % i elif hwsku == "Arista-720DT-48S" or hwsku == "Arista-720DT-G48S4": for i in range(1, 53): port_alias_to_name_map["etp%d" % i] = "Ethernet%d" % (i - 1) @@ -377,6 +399,13 @@ def get_port_alias_to_name_map(hwsku, asic_name=None): for i in range(1, 33): port_alias_to_name_map["etp%d" % i] = "Ethernet%d" % idx idx += 8 + elif hwsku == "Mellanox-SN5600-V256": + split_alias_list = ["a", "b", "c", "d"] + for i in range(1, 65): + for idx, split_alias in enumerate(split_alias_list): + alias = "etp{}{}".format(i, split_alias) + eth_name = "Ethernet{}".format((i - 1) * 8 + idx * 2) + port_alias_to_name_map[alias] = eth_name elif hwsku == "Arista-7060DX5-32": for i in range(1, 33): port_alias_to_name_map["Ethernet%d/1" % i] = "Ethernet%d" % ((i - 1) * 8) diff --git a/ansible/roles/eos/tasks/ceos.yml b/ansible/roles/eos/tasks/ceos.yml index 1ab7b0311a..99067b79f5 100644 --- a/ansible/roles/eos/tasks/ceos.yml +++ b/ansible/roles/eos/tasks/ceos.yml @@ -11,6 +11,7 @@ when: snmp_data.ansible_facts.ansible_sysname is defined - include_tasks: ceos_config.yml +- include_vars: group_vars/vm_host/ceos.yml - name: Create cEOS container become: yes diff --git a/ansible/roles/eos/templates/dualtor-64-breakout-leaf.j2 b/ansible/roles/eos/templates/dualtor-64-breakout-leaf.j2 new file mode 120000 index 0000000000..0312b84bdd --- /dev/null +++ b/ansible/roles/eos/templates/dualtor-64-breakout-leaf.j2 @@ -0,0 +1 @@ +dualtor-leaf.j2 \ No newline at end of file diff --git a/ansible/roles/eos/templates/dualtor-aa-64-breakout-leaf.j2 b/ansible/roles/eos/templates/dualtor-aa-64-breakout-leaf.j2 new file mode 120000 index 0000000000..0312b84bdd --- /dev/null +++ b/ansible/roles/eos/templates/dualtor-aa-64-breakout-leaf.j2 @@ -0,0 +1 @@ +dualtor-leaf.j2 \ No newline at end of file diff --git a/ansible/roles/eos/templates/t0-28-leaf.j2 b/ansible/roles/eos/templates/t0-28-leaf.j2 new file mode 120000 index 0000000000..8430cb1deb --- /dev/null +++ b/ansible/roles/eos/templates/t0-28-leaf.j2 @@ -0,0 +1 @@ +t0-leaf.j2 \ No newline at end of file diff --git a/ansible/roles/fanout/library/port_config_gen.py b/ansible/roles/fanout/library/port_config_gen.py index 5840859216..b3bc373035 100644 --- a/ansible/roles/fanout/library/port_config_gen.py +++ b/ansible/roles/fanout/library/port_config_gen.py @@ -151,13 +151,13 @@ def _read_from_port_config(filepath): with open(filepath) as fd: lines = fd.readlines() data_index = 0 - while lines[data_index].startswith("#"): + while not lines[data_index].strip() or lines[data_index].startswith("#"): data_index = data_index + 1 header = lines[data_index-1].strip("#\n ") keys = header.split() alias_index = keys.index("alias") for line in lines[data_index:]: - if not line: + if not line.strip() or line.startswith("#"): continue values = line.split() # port alias as the key diff --git a/ansible/roles/fanout/lookup_plugins/cisco_8101_port_convert.py b/ansible/roles/fanout/lookup_plugins/cisco_8101_port_convert.py new file mode 100644 index 0000000000..d8a6f0fdd1 --- /dev/null +++ b/ansible/roles/fanout/lookup_plugins/cisco_8101_port_convert.py @@ -0,0 +1,150 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.utils.display import Display +from ansible.plugins.lookup import LookupBase +from ansible.errors import AnsibleError + +import re + +DOCUMENTATION = """ + lookup: cisco_8101_port_convert + version_added: "1.0" + short_description: find port name and alias for Cisco-8101 devices + description: + - This lookup returns [port name, port alias] of the given port. + options: + _terms: + description: list of port name + required: True + output: + description: output port type, could only be sonic or alias + required: True + speed: + description: speed of input port + required: False +""" + +display = Display() + +# name lanes alias index speed subport +port_config = """ +Ethernet0 2304,2305,2306,2307 etp0a 0 100000 1 +Ethernet4 2308,2309,2310,2311 etp0b 0 100000 2 +Ethernet8 2320,2321,2322,2323 etp1a 1 100000 1 +Ethernet12 2324,2325,2326,2327 etp1b 1 100000 2 +Ethernet16 2312,2313,2314,2315 etp2a 2 100000 1 +Ethernet20 2316,2317,2318,2319 etp2b 2 100000 2 +Ethernet24 2056,2057,2058,2059 etp3a 3 100000 1 +Ethernet28 2060,2061,2062,2063 etp3b 3 100000 2 +Ethernet32 1792,1793,1794,1795 etp4a 4 100000 1 +Ethernet36 1796,1797,1798,1799 etp4b 4 100000 2 +Ethernet40 2048,2049,2050,2051 etp5a 5 100000 1 +Ethernet44 2052,2053,2054,2055 etp5b 5 100000 2 +Ethernet48 2560,2561,2562,2563 etp6a 6 100000 1 +Ethernet52 2564,2565,2566,2567 etp6b 6 100000 2 +Ethernet56 2824,2825,2826,2827 etp7a 7 100000 1 +Ethernet60 2828,2829,2830,2831 etp7b 7 100000 2 +Ethernet64 2832,2833,2834,2835 etp8a 8 100000 1 +Ethernet68 2836,2837,2838,2839 etp8b 8 100000 2 +Ethernet72 2816,2817,2818,2819 etp9a 9 100000 1 +Ethernet76 2820,2821,2822,2823 etp9b 9 100000 2 +Ethernet80 2568,2569,2570,2571 etp10a 10 100000 1 +Ethernet84 2572,2573,2574,2575 etp10b 10 100000 2 +Ethernet88 2576,2577,2578,2579 etp11a 11 100000 1 +Ethernet92 2580,2581,2582,2583 etp11b 11 100000 2 +Ethernet96 1536,1537,1538,1539 etp12a 12 100000 1 +Ethernet100 1540,1541,1542,1543 etp12b 12 100000 2 +Ethernet104 1800,1801,1802,1803 etp13a 13 100000 1 +Ethernet108 1804,1805,1806,1807 etp13b 13 100000 2 +Ethernet112 1552,1553,1554,1555 etp14a 14 100000 1 +Ethernet116 1556,1557,1558,1559 etp14b 14 100000 2 +Ethernet120 1544,1545,1546,1547 etp15a 15 100000 1 +Ethernet124 1548,1549,1550,1551 etp15b 15 100000 2 +Ethernet128 1296,1297,1298,1299 etp16a 16 100000 1 +Ethernet132 1300,1301,1302,1303 etp16b 16 100000 2 +Ethernet136 1288,1289,1290,1291 etp17a 17 100000 1 +Ethernet140 1292,1293,1294,1295 etp17b 17 100000 2 +Ethernet144 1280,1281,1282,1283 etp18a 18 100000 1 +Ethernet148 1284,1285,1286,1287 etp18b 18 100000 2 +Ethernet152 1032,1033,1034,1035 etp19a 19 100000 1 +Ethernet156 1036,1037,1038,1039 etp19b 19 100000 2 +Ethernet160 264,265,266,267 etp20a 20 100000 1 +Ethernet164 268,269,270,271 etp20b 20 100000 2 +Ethernet168 272,273,274,275 etp21a 21 100000 1 +Ethernet172 276,277,278,279 etp21b 21 100000 2 +Ethernet176 16,17,18,19 etp22a 22 100000 1 +Ethernet180 20,21,22,23 etp22b 22 100000 2 +Ethernet184 0,1,2,3 etp23a 23 100000 1 +Ethernet188 4,5,6,7 etp23b 23 100000 2 +Ethernet192 256,257,258,259 etp24a 24 100000 1 +Ethernet196 260,261,262,263 etp24b 24 100000 2 +Ethernet200 8,9,10,11 etp25a 25 100000 1 +Ethernet204 12,13,14,15 etp25b 25 100000 2 +Ethernet208 1024,1025,1026,1027 etp26a 26 100000 1 +Ethernet212 1028,1029,1030,1031 etp26b 26 100000 2 +Ethernet216 768,769,770,771 etp27a 27 100000 1 +Ethernet220 772,773,774,775 etp27b 27 100000 2 +Ethernet224 524,525,526,527 etp28a 28 100000 1 +Ethernet228 520,521,522,523 etp28b 28 100000 2 +Ethernet232 776,777,778,779 etp29a 29 100000 1 +Ethernet236 780,781,782,783 etp29b 29 100000 2 +Ethernet240 516,517,518,519 etp30a 30 100000 1 +Ethernet244 512,513,514,515 etp30b 30 100000 2 +Ethernet248 528,529,530,531 etp31a 31 100000 1 +Ethernet252 532,533,534,535 etp31b 31 100000 2 +""" + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + display.debug("Cisco-8101 port lookup: {}".format(terms)) + self.set_options(var_options=variables, direct=kwargs) + output = self.get_option("output") + speed = self.get_option("speed") + ret = [] + for port in terms: + match_result = re.findall(r"(^etp([0-9]+)([ab]?)$)|(^Ethernet([0-9]+)$)", port) + if len(match_result) != 1: + raise AnsibleError("port {} is illegal".format(port)) + match_result = match_result[0] + if match_result[0]: + alias = match_result[0] + sonic_port_index = int(match_result[1]) * 8 + 4 * (match_result[2] == "b") + sonic_name = "Ethernet{}".format(sonic_port_index) + elif match_result[3]: + sonic_name = match_result[3] + sonic_port_index = int(match_result[4]) + if speed == "400000": + if sonic_port_index % 8: + raise AnsibleError("port {} is not legal 400G port".format(port)) + alias = "etp{}".format(sonic_port_index // 8) + elif speed == "100000": + if sonic_port_index % 4: + raise AnsibleError("port {} is not legal 100G port".format(port)) + alias = "etp{}{}".format(sonic_port_index // 8, "b" if sonic_port_index // 4 % 2 else "a") + else: + raise AnsibleError("speed {} is illegal".format(speed)) + if output == "alias": + ret.append(alias) + elif output == "sonic": + ret.append(sonic_name) + elif output == "lanes": + lines = port_config.splitlines() + regex = re.compile(r"^\S+\s+(\S+)") + line_index = sonic_port_index // 4 + 1 + if speed == "100000": + ret.append(regex.findall(lines[line_index])[0]) + elif speed == "400000": + ret.append(regex.findall(lines[line_index])[0] + "," + regex.findall(lines[line_index + 1])[0]) + elif output == "index": + ret.append(str(sonic_port_index // 8)) + elif output == "subport": + if speed == "100000": + ret.append(str(1 + sonic_port_index // 4 % 2)) + elif speed == "100000": + ret.append("0") + else: + raise AnsibleError("output parameter must be provided (sonic, alias, lanes, index, or subport)") + return ret diff --git a/ansible/roles/fanout/tasks/fanout_sonic.yml b/ansible/roles/fanout/tasks/fanout_sonic.yml index ddba057215..cf0a1e161f 100644 --- a/ansible/roles/fanout/tasks/fanout_sonic.yml +++ b/ansible/roles/fanout/tasks/fanout_sonic.yml @@ -39,7 +39,12 @@ - name: deploy SONiC fanout not incremental and not dry_run include_tasks: sonic/fanout_sonic_202205.yml - when: dry_run is not defined and incremental is not defined + when: "dry_run is not defined and incremental is not defined and 'Cisco-8101' not in device_info[inventory_hostname]['HwSku']" + + - name: deploy SONiC Cisco 8101 fanout not incremental and not dry_run + include_tasks: + sonic/fanout_sonic_cisco_8101_202205.yml + when: "dry_run is not defined and incremental is not defined and 'Cisco-8101' in device_info[inventory_hostname]['HwSku']" - name: deploy SONiC fanout incremental and not dry_run include_tasks: @@ -52,10 +57,10 @@ when: dry_run is defined when: "'20220531' in fanout_sonic_version['build_version'] or 'internal' in fanout_sonic_version['build_version']" -- name: deploy SONiC fanout with image version 202305 +- name: deploy SONiC fanout with image version 202311 block: - name: deploy SONiC fanout not incremental and not dry_run include_tasks: - sonic/fanout_sonic_202305.yml + sonic/fanout_sonic_202311.yml when: dry_run is not defined and incremental is not defined when: "'2023' in fanout_sonic_version['build_version']" diff --git a/ansible/roles/fanout/tasks/sonic/fanout_sonic_202311.yml b/ansible/roles/fanout/tasks/sonic/fanout_sonic_202311.yml new file mode 100644 index 0000000000..4ae67fe3b1 --- /dev/null +++ b/ansible/roles/fanout/tasks/sonic/fanout_sonic_202311.yml @@ -0,0 +1,109 @@ +- name: collect fanout port config + port_config_gen: + hwsku: "{{ device_info[inventory_hostname]['HwSku'] }}" + hwsku_type: "{{ device_info[inventory_hostname]['HwSkuType'] | default('predefined') }}" + device_conn: "{{ device_conn[inventory_hostname] }}" + become: yes + +- name: Ensure TPID enabled in SAI + lineinfile: + path: /usr/share/sonic/device/{{ fanout_platform }}/{{ fanout_hwsku }}/sai.profile + line: 'SAI_USER_DEFINED_TPID=1' + state: present + become: yes + when: "'mellanox' in fanout_sonic_version.asic_type" + +- name: Disable software control module function in SAI + lineinfile: + path: /usr/share/sonic/device/{{ fanout_platform }}/{{ fanout_hwsku }}/sai.profile + line: 'SAI_INDEPENDENT_MODULE_MODE=1' + state: absent + become: yes + when: "'mellanox' in fanout_sonic_version.asic_type" + +- name: build fanout startup config + template: + src: "sonic_deploy_202311.j2" + dest: "/tmp/base_config.json" + +- name: generate config_db.json + shell: sonic-cfggen -H -j /tmp/base_config.json --print-data > /etc/sonic/config_db.json + become: yes + +- name: disable all copp rules + copy: + content: "{}" + dest: "/usr/share/sonic/templates/copp_cfg.j2" + become: yes + when: "'mellanox' not in fanout_sonic_version.asic_type" + +- name: Overwrite copp rules for Mellanox FanoutLeaf + copy: + src: "copp_cfg_mlnx.j2" + dest: "/usr/share/sonic/templates/copp_cfg.j2" + become: yes + when: "'mellanox' in fanout_sonic_version.asic_type" + +- name: Disable feature teamd and remove teamd container (avoid swss crash after config reload) + block: + - name: Check if teamd container exists + shell: "docker ps -a -q -f name=teamd" + register: teamd_container + + - name: disable feature teamd and remove container + block: + - name: disable feature teamd + shell: config feature state teamd disabled + become: true + - name: ensure teamd container is stopped + docker_container: + name: teamd + state: stopped + become: true + ignore_errors: yes + - name: remove teamd container + docker_container: + name: teamd + state: absent + become: true + when: teamd_container.stdout != "" + +- name: SONiC update config db + shell: config reload -y -f + become: true + +- name: wait for SONiC update config db finish + pause: + seconds: 180 + +- name: Shutdown arp_update process in swss (avoid fanout broadcasting it's MAC address) + shell: docker exec -i swss supervisorctl stop arp_update + become: yes + +- name: Setup broadcom based fanouts + block: + - name: reinit fp entries + shell: "bcmcmd 'fp detach' && bcmcmd 'fp init'" + become: yes + when: "'broadcom' in fanout_sonic_version.asic_type" + +- block: + - name: Remove ipv6 parameter + lineinfile: + path: /etc/sysctl.conf + line: 'net.ipv6.conf.all.disable_ipv6 = 0' + state: absent + become: yes + + - name: Update IPv6 parameter + lineinfile: + path: /etc/sysctl.conf + line: 'net.ipv6.conf.all.disable_ipv6 = 1' + state: present + become: yes + + - name: Apply parameter to disable IPv6 function + shell: "sysctl -p" + become: yes + + when: "'mellanox' in fanout_sonic_version.asic_type" diff --git a/ansible/roles/fanout/tasks/sonic/fanout_sonic_cisco_8101_202205.yml b/ansible/roles/fanout/tasks/sonic/fanout_sonic_cisco_8101_202205.yml new file mode 100644 index 0000000000..3e450f5dd3 --- /dev/null +++ b/ansible/roles/fanout/tasks/sonic/fanout_sonic_cisco_8101_202205.yml @@ -0,0 +1,76 @@ +- fail: msg="Cisco fanout do not support sonic version other than 20220531.45" + when: "'20220531.45' not in fanout_sonic_version['build_version']" + +- name: set device_conn + set_fact: + device_vlan_list: "{{ device_vlan_list[inventory_hostname] }}" + device_port_vlans: "{{ device_port_vlans[inventory_hostname] }}" + device_info: "{{ device_info[inventory_hostname] }}" + device_conn: "{{ device_conn[inventory_hostname] }}" + +- name: build fanout startup config + template: + src: "sonic_deploy_cisco_8101_202205.j2" + dest: "/tmp/base_config.json" + +- name: backup config_db.json + shell: cp /etc/sonic/config_db.json /etc/sonic/config_db.json.bak + +- name: generate config_db.json + shell: sonic-cfggen -H -j /tmp/base_config.json --print-data > /etc/sonic/config_db.json + become: yes + +- name: copy Cisco 8101 fanout command file to fanout + copy: + src: "templates/cisco_8101_commands.txt" + dest: "/tmp/cisco_8101_commands.txt" + +- name: disable feature teamd and remove teamd container (avoid swss crash after config reload) + block: + - name: check if teamd container exists + shell: "docker ps -a -q -f name=teamd" + register: teamd_container + + - name: disable feature teamd and remove container + block: + - name: disable feature teamd + shell: config feature state teamd disabled + become: true + - name: ensure teamd container is stopped + docker_container: + name: teamd + state: stopped + become: true + ignore_errors: yes + - name: remove teamd container + docker_container: + name: teamd + state: absent + become: true + when: teamd_container.stdout != "" + +- name: SONiC update config db + shell: config reload -y -f + become: true + +- name: wait for SONiC update config db finish + pause: + seconds: 180 + +- name: Shutdown arp_update process in swss (avoid fanout broadcasting it's MAC address) + shell: docker exec -i swss supervisorctl stop arp_update + become: yes + +- name: Disable LLDP on Cisco 8101 fanout + shell: | + config feature autorestart lldp disabled + config feature state lldp disabled + docker exec -i syncd supervisorctl start dshell_client + become: yes + +- name: wait for dshell_client start + pause: + seconds: 300 + +- name: Clear L2 trap configuration on Cisco 8101 fanout + shell: cat /tmp/cisco_8101_commands.txt | docker exec -i syncd sh -c "/usr/bin/dshell_client.py -i" diff --git a/ansible/roles/fanout/templates/arista_7260cx3_deploy.j2 b/ansible/roles/fanout/templates/arista_7260cx3_deploy.j2 index 894140c562..4986afd290 100644 --- a/ansible/roles/fanout/templates/arista_7260cx3_deploy.j2 +++ b/ansible/roles/fanout/templates/arista_7260cx3_deploy.j2 @@ -51,17 +51,21 @@ vrf definition management speed forced 40gfull no shutdown {% elif device_conn[inventory_hostname][intf]['speed'] == "10000" %} -{% for sub in range (1,5) %} +{% for sub in range (1,5) %} {% set subintf = 'Ethernet' + i|string + '/' + sub|string %} - interface {{ subintf }} - description {{ device_conn[inventory_hostname][intf]['peerdevice'] }}-{{ device_conn[inventory_hostname][intf]['peerport'] }} - switchport access vlan {{ device_port_vlans[inventory_hostname][intf]['vlanids'] }} +interface {{ subintf }} +{% if subintf in device_port_vlans[inventory_hostname] and device_port_vlans[inventory_hostname][subintf]['mode'] != "Trunk" %} + description {{ device_conn[inventory_hostname][subintf]['peerdevice'] }}-{{ device_conn[inventory_hostname][subintf]['peerport'] }} + switchport access vlan {{ device_port_vlans[inventory_hostname][subintf]['vlanids'] }} switchport mode dot1q-tunnel spanning-tree portfast speed forced 10gfull no shutdown ! -{% endfor %} +{% else %} + shutdown +{% endif %} +{% endfor %} {% elif device_conn[inventory_hostname][intf]['speed'] == "50000" %} interface {{ intf }} description {{ device_conn[inventory_hostname][intf]['peerdevice'] }}-{{ device_conn[inventory_hostname][intf]['peerport'] }} diff --git a/ansible/roles/fanout/templates/cisco_8101_commands.txt b/ansible/roles/fanout/templates/cisco_8101_commands.txt new file mode 100644 index 0000000000..7cc9f717ca --- /dev/null +++ b/ansible/roles/fanout/templates/cisco_8101_commands.txt @@ -0,0 +1,23 @@ +d0=sdk.la_get_device(0) + +d0.clear_trap_configuration(sdk.LA_EVENT_ETHERNET_L2CP0) + +d0.clear_trap_configuration(sdk.LA_EVENT_ETHERNET_LACP) + +d0.clear_trap_configuration(sdk.LA_EVENT_ETHERNET_ARP) + +d0.clear_trap_configuration(sdk.LA_EVENT_ETHERNET_L2CP2) + +d0.clear_trap_configuration(sdk.LA_EVENT_ETHERNET_DHCPV4_SERVER) + +d0.clear_trap_configuration(sdk.LA_EVENT_ETHERNET_DHCPV4_CLIENT) + +d0.clear_trap_configuration(sdk.LA_EVENT_ETHERNET_DHCPV6_SERVER) + +d0.clear_trap_configuration(sdk.LA_EVENT_ETHERNET_DHCPV6_CLIENT) + +d0.clear_trap_configuration(sdk.LA_EVENT_ETHERNET_CISCO_PROTOCOLS) + +d0.clear_trap_configuration(sdk.LA_EVENT_L3_ISIS_OVER_L3) + +quit() diff --git a/ansible/roles/fanout/templates/sonic_deploy_202305.j2 b/ansible/roles/fanout/templates/sonic_deploy_202305.j2 index 2580d45c30..efbf4836e2 100644 --- a/ansible/roles/fanout/templates/sonic_deploy_202305.j2 +++ b/ansible/roles/fanout/templates/sonic_deploy_202305.j2 @@ -166,15 +166,15 @@ }, "BUFFER_PORT_EGRESS_PROFILE_LIST": { {% for port_name in fanout_port_config %} - "{{ port_name }}|": { - "profile_list": "egress_lossless_profile,egress_lossy_profile" + "{{ port_name }}": { + "profile_list": "egress_lossy_profile" }{% if not loop.last %},{% endif %} {% endfor %} }, "BUFFER_PORT_INGRESS_PROFILE_LIST": { {% for port_name in fanout_port_config %} - "{{ port_name }}|": { - "profile_list": "ingress_lossless_profile,ingress_lossy_profile" + "{{ port_name }}": { + "profile_list": "ingress_lossy_profile" }{% if not loop.last %},{% endif %} {% endfor %} }, diff --git a/ansible/roles/fanout/templates/sonic_deploy_202311.j2 b/ansible/roles/fanout/templates/sonic_deploy_202311.j2 new file mode 100644 index 0000000000..137b4493fa --- /dev/null +++ b/ansible/roles/fanout/templates/sonic_deploy_202311.j2 @@ -0,0 +1,455 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "default_pfcwd_status": "disable", + "hwsku": "{{ fanout_hwsku }}", + "hostname": "{{ inventory_hostname }}" + } + }, + + "PORT": { + {% for port_name in fanout_port_config %} + "{{ port_name }}": { + "alias": "{{ fanout_port_config[port_name]['alias'] }}", + "speed" : "{{ fanout_port_config[port_name]['speed'] | default('100000') }}", + "index": "{{ fanout_port_config[port_name]['index'] }}", + "lanes": "{{ fanout_port_config[port_name]['lanes'] }}", + "pfc_asym": "off", + "mtu": "9100", + {% if fanout_hwsku == "Arista-720DT-G48S4" and (fanout_port_config[port_name]['lanes'] | int) <= 24 %} + "autoneg": "on", + {% endif %} + {% if fanout_port_config[port_name]['speed'] | default('100000') == "100000" %} + "fec" : "rs", + {% endif %} + {% if 'broadcom' in fanout_sonic_version["asic_type"] or 'marvell' in fanout_sonic_version["asic_type"] or 'mellanox' in fanout_sonic_version["asic_type"] %} + {% if port_name in device_port_vlans[inventory_hostname] and device_port_vlans[inventory_hostname][port_name]["mode"].lower() == "access" %} + "tpid": "0x9100", + {% endif %} + {% endif %} + {% if 'peerdevice' in fanout_port_config[port_name] %} + "admin_status": "up" + {% else %} + "admin_status": "down" + {% endif %} + }{% if not loop.last %},{% endif %} + {% endfor %} + }, + + "VLAN": { + {% for vlanid in device_vlan_list[inventory_hostname] | unique %} + "Vlan{{ vlanid }}": { + "vlanid": "{{ vlanid }}" + }{% if not loop.last %},{% endif %} + {% endfor %} + }, + + {% set ns = {'firstPrinted': False} %} + "VLAN_MEMBER": { + {% for port_name in device_port_vlans[inventory_hostname] %} + {% if device_port_vlans[inventory_hostname][port_name]['mode'].lower() == 'access' %} + {% if ns.firstPrinted %},{% endif %} + "Vlan{{ device_port_vlans[inventory_hostname][port_name]['vlanids'] }}|{{ fanout_port_config[port_name]['name'] }}": { + "tagging_mode" : "untagged" + } + {% if ns.update({'firstPrinted': True}) %} {% endif %} + {% elif device_port_vlans[inventory_hostname][port_name]['mode'].lower() == 'trunk' %} + {% for vlanid in device_port_vlans[inventory_hostname][port_name]['vlanlist'] %} + {% if ns.firstPrinted %},{% endif %} + "Vlan{{ vlanid }}|{{ fanout_port_config[port_name]['name'] }}": { + "tagging_mode" : "tagged" + } + {% if ns.update({'firstPrinted': True}) %} {% endif %} + {% endfor %} + {% endif %} + {% endfor %} + }, + + "MGMT_INTERFACE": { + "eth0|{{ device_info[inventory_hostname]["ManagementIp"] }}": { + "gwaddr": "{{ device_info[inventory_hostname]["ManagementGw"] }}" + } + }, + "MGMT_PORT": { + "eth0": { + "admin_status": "up", + "alias": "eth0" + } + }, + + "VERSIONS": { + "DATABASE": { + "VERSION": "version_1_0_1" + } + }, +{% if 'mellanox' in fanout_sonic_version["asic_type"] %} + "BUFFER_POOL": { + "egress_lossless_pool": { + "mode": "dynamic", + "size": "60817392", + "type": "egress" + }, + "egress_lossy_pool": { + "mode": "dynamic", + "size": "49946624", + "type": "egress" + }, + "ingress_lossless_pool": { + "mode": "dynamic", + "size": "49946624", + "type": "ingress", + "xoff": "4063232" + }, + "ingress_zero_pool": { + "mode": "static", + "size": "0", + "type": "ingress" + } + }, + "BUFFER_PROFILE": { + "egress_lossless_profile": { + "dynamic_th": "7", + "pool": "egress_lossless_pool", + "size": "0" + }, + "egress_lossless_zero_profile": { + "dynamic_th": "-8", + "pool": "egress_lossless_pool", + "size": "0" + }, + "egress_lossy_profile": { + "dynamic_th": "7", + "pool": "egress_lossy_pool", + "size": "9216" + }, + "egress_lossy_zero_profile": { + "dynamic_th": "-8", + "pool": "egress_lossy_pool", + "size": "0" + }, + "ingress_lossless_profile": { + "dynamic_th": "7", + "pool": "ingress_lossless_pool", + "size": "0" + }, + "ingress_lossless_zero_profile": { + "dynamic_th": "-8", + "pool": "ingress_lossless_pool", + "size": "0" + }, + "ingress_lossy_pg_zero_profile": { + "pool": "ingress_zero_pool", + "size": "0", + "static_th": "0" + }, + "ingress_lossy_profile": { + "dynamic_th": "3", + "pool": "ingress_lossless_pool", + "size": "0" + }, + "pg_lossless_200000_5m_profile": { + "dynamic_th": "0", + "pool": "ingress_lossless_pool", + "size": "19456", + "xoff": "66560", + "xon": "19456" + }, + "pg_lossless_400000_5m_profile": { + "dynamic_th": "0", + "pool": "ingress_lossless_pool", + "size": "38912", + "xoff": "115712", + "xon": "38912" + }, + "q_lossy_profile": { + "dynamic_th": "3", + "pool": "egress_lossy_pool", + "size": "0" + } + }, + "BUFFER_PORT_EGRESS_PROFILE_LIST": { + {% for port_name in fanout_port_config %} + "{{ port_name }}": { + "profile_list": "egress_lossy_profile" + }{% if not loop.last %},{% endif %} + {% endfor %} + }, + "BUFFER_PORT_INGRESS_PROFILE_LIST": { + {% for port_name in fanout_port_config %} + "{{ port_name }}": { + "profile_list": "ingress_lossy_profile" + }{% if not loop.last %},{% endif %} + {% endfor %} + }, +{% endif %} +{% if fanout_hwsku not in ("Nokia-7215", "Nokia-7215-A1", "Arista-720DT-G48S4") %} + "BUFFER_QUEUE": { + {% for port_name in fanout_port_config %} + "{{ port_name }}|0-7": { + "profile": "q_lossy_profile" + }{% if not loop.last %},{% endif %} + {% endfor %} + }, + "BUFFER_PG": { + {% for port_name in fanout_port_config %} + "{{ port_name }}|0-7": { + "profile": "ingress_lossy_profile" + }{% if not loop.last %},{% endif %} + {% endfor %} + }, +{% endif %} +{% if fanout_hwsku in ("Nokia-7215", "Nokia-7215-A1", "Arista-720DT-G48S4") %} + + "FLEX_COUNTER_TABLE": { + "ACL": { + "FLEX_COUNTER_STATUS": "disable" + }, + "BUFFER_POOL_WATERMARK": { + "FLEX_COUNTER_STATUS": "disable" + }, + "PFCWD": { + "FLEX_COUNTER_STATUS": "disable" + }, + "PG_DROP": { + "FLEX_COUNTER_STATUS": "disable" + }, + "PG_WATERMARK": { + "FLEX_COUNTER_STATUS": "disable" + }, + "PORT": { + "FLEX_COUNTER_STATUS": "enable" + }, + "PORT_BUFFER_DROP": { + "FLEX_COUNTER_STATUS": "disable" + }, + "QUEUE": { + "FLEX_COUNTER_STATUS": "disable" + }, + "QUEUE_WATERMARK": { + "FLEX_COUNTER_STATUS": "disable" + }, + "RIF": { + "FLEX_COUNTER_STATUS": "disable" + } + }, + +{% else %} + + "FLEX_COUNTER_TABLE": { + "PFCWD": { + "FLEX_COUNTER_STATUS": "enable" + }, + "PORT": { + "FLEX_COUNTER_STATUS": "enable" + }, + "QUEUE": { + "FLEX_COUNTER_STATUS": "enable" + } + }, + + "QUEUE": { + {% for port_name in fanout_port_config %} + "{{ port_name }}|0-7": { + "scheduler": "[SCHEDULER|scheduler.0]" + }{% if not loop.last %},{% endif %} + {% endfor %} + }, + + "CABLE_LENGTH": { + "AZURE": { + {% for port_name in fanout_port_config %} + "{{ port_name }}": "300m"{% if not loop.last %},{% endif %} + {% endfor %} + } + }, + + "PFC_WD": { + "GLOBAL": { + "POLL_INTERVAL": "200" + } + }, + + "SCHEDULER": { + "scheduler.0": { + "type": "DWRR", + "weight": "14" + }, + "scheduler.1": { + "type": "DWRR", + "weight": "15" + } + }, + + "WRED_PROFILE": { + "AZURE_LOSSLESS": { + "red_max_threshold": "2097152", + "red_drop_probability": "5", + "wred_green_enable": "true", + "ecn": "ecn_all", + "green_min_threshold": "250000", + "red_min_threshold": "1048576", + "wred_yellow_enable": "true", + "yellow_min_threshold": "1048576", + "green_max_threshold": "2097152", + "green_drop_probability": "5", + "yellow_max_threshold": "2097152", + "yellow_drop_probability": "5", + "wred_red_enable": "true" + } + }, + + "SCHEDULER": { + "scheduler.0": { + "type": "DWRR", + "weight": "14" + }, + "scheduler.1": { + "type": "DWRR", + "weight": "15" + } + }, + +{% endif %} + + "FEATURE": { + "acms": { + "auto_restart": "disabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "disabled" + }, + "mux": { + "auto_restart": "disabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "disabled" + }, + "bgp": { + "state": "disabled", + "has_timer": false, + "has_global_scope": false, + "has_per_asic_scope": true, + "auto_restart": "disabled", + "high_mem_alert": "disabled" + }, + "database": { + "state": "always_enabled", + "has_timer": false, + "has_global_scope": true, + "has_per_asic_scope": true, + "auto_restart": "always_enabled", + "high_mem_alert": "disabled" + }, + "dhcp_relay": { + "state": "disabled", + "has_timer": false, + "has_global_scope": true, + "has_per_asic_scope": false, + "auto_restart": "disabled", + "high_mem_alert": "disabled" + }, + "lldp": { + "state": "disabled", + "has_timer": false, + "has_global_scope": true, + "has_per_asic_scope": true, + "auto_restart": "disabled", + "high_mem_alert": "disabled" + }, + "pmon": { + "state": "enabled", + "has_timer": false, + "has_global_scope": true, + "has_per_asic_scope": false, + "auto_restart": "enabled", + "high_mem_alert": "disabled" + }, + "radv": { + "state": "disabled", + "has_timer": false, + "has_global_scope": true, + "has_per_asic_scope": false, + "auto_restart": "disabled", + "high_mem_alert": "disabled" + }, + "snmp": { + "state": "enabled", + "has_timer": true, + "has_global_scope": true, + "has_per_asic_scope": false, + "auto_restart": "enabled", + "high_mem_alert": "disabled" + }, + "swss": { + "state": "enabled", + "has_timer": false, + "has_global_scope": false, + "has_per_asic_scope": true, + "auto_restart": "enabled", + "high_mem_alert": "disabled" + }, + "syncd": { + "state": "enabled", + "has_timer": false, + "has_global_scope": false, + "has_per_asic_scope": true, + "auto_restart": "enabled", + "high_mem_alert": "disabled" + }, + "teamd": { + "state": "disabled", + "has_timer": false, + "has_global_scope": false, + "has_per_asic_scope": true, + "auto_restart": "disabled", + "high_mem_alert": "disabled" + }, + "mgmt-framework": { + "state": "disabled", + "has_timer": true, + "has_global_scope": true, + "has_per_asic_scope": false, + "auto_restart": "disabled", + "high_mem_alert": "disabled" + }, + "nat": { + "state": "disabled", + "has_timer": false, + "has_global_scope": true, + "has_per_asic_scope": false, + "auto_restart": "disabled", + "high_mem_alert": "disabled" + }, + "sflow": { + "state": "disabled", + "has_timer": false, + "has_global_scope": true, + "has_per_asic_scope": false, + "auto_restart": "disabled", + "high_mem_alert": "disabled" + }, + "telemetry": { + "state": "disabled", + "has_timer": true, + "has_global_scope": true, + "has_per_asic_scope": false, + "auto_restart": "disabled", + "high_mem_alert": "disabled" + }, + "restapi": { + "auto_restart": "disabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "disabled" + } + } +} diff --git a/ansible/roles/fanout/templates/sonic_deploy_cisco_8101_202205.j2 b/ansible/roles/fanout/templates/sonic_deploy_cisco_8101_202205.j2 new file mode 100644 index 0000000000..6b998b4fec --- /dev/null +++ b/ansible/roles/fanout/templates/sonic_deploy_cisco_8101_202205.j2 @@ -0,0 +1,687 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "default_pfcwd_status": "disable", + "hwsku": "{{ device_info["HwSku"] }}", + "hostname": "{{ inventory_hostname }}" + } + }, + "VLAN": { +{% for vlanid in device_vlan_list | unique %} + "Vlan{{ vlanid }}": { + "vlanid": "{{ vlanid }}" + }{% if not loop.last %},{% endif %} + +{% endfor %} + }, +{% set ns = {"notFirstPrinted": False} %} + "VLAN_MEMBER": { +{% for port_name in device_port_vlans %} +{% if device_port_vlans[port_name]["mode"].lower() == "access" %} +{% if ns.notFirstPrinted %}, +{% endif %} + "Vlan{{ device_port_vlans[port_name]["vlanids"] }}|{{ port_name }}": { + "tagging_mode": "untagged" + }{% if ns.update({"notFirstPrinted": True}) %} {% endif %} +{% elif device_port_vlans[port_name]["mode"].lower() == "trunk" %} +{% for vlanid in device_port_vlans[port_name]["vlanlist"] %} +{% if ns.notFirstPrinted %}, +{% endif %} + "Vlan{{ vlanid }}|{{ port_name }}": { + "tagging_mode": "tagged" + }{% if ns.update({"notFirstPrinted": True}) %} {% endif %} +{% endfor %} +{% endif %} +{% endfor %} + + }, + "MGMT_INTERFACE": { + "eth0|{{ device_info["ManagementIp"] }}": { + "gwaddr": "{{ device_info["ManagementGw"] }}" + } + }, + "PORT": { +{% for port_name in device_port_vlans %} +{% set ns = {"port_name": port_name} %} +{% if ns.update({"peerport": device_conn[port_name]["peerport"], "peerdevice": device_conn[port_name]["peerdevice"], "speed": device_conn[port_name]["speed"]})%}{% endif %} +{% if "root" in ns.peerdevice %} +{% if ns.update({"fullspeed": "400000"}) %}{% endif %} +{% else %} +{% if ns.update({"fullspeed": ns.speed})%}{% endif %} +{% endif %} + "{{ lookup("cisco_8101_port_convert", port_name, speed=ns.fullspeed, output="sonic") }}": { + "admin_status": "up", + "alias": "{{ lookup("cisco_8101_port_convert", port_name, speed=ns.fullspeed, output="alias") }}", + "description": "{{ ns.peerdevice }}-{{ ns.peerport }}", +{% if ns.speed == "100000" %} + "fec": "rs", +{% endif %} + "index": "{{ lookup("cisco_8101_port_convert", port_name, speed=ns.speed, output="index") }}", + "lanes": "{{ lookup("cisco_8101_port_convert", port_name, speed=ns.speed, output="lanes") }}", + "pfc_asym": "off", + "speed": "{{ ns.speed }}", + "subport": "{{ lookup("cisco_8101_port_convert", port_name, speed=ns.fullspeed, output="subport") }}", +{% if device_port_vlans[port_name]["mode"].lower() == "access" %} + "tpid": "0x9100", +{% endif %} + "mtu": "9100" + }{% if not loop.last %}, +{% endif %} +{% endfor %} + + }, + "AAA": { + "authentication": { + "failthrough": "True", + "fallback": "True", + "login": "tacacs+" + } + }, + "ACL_TABLE": { + "DATAACL": { + "policy_desc": "DATAACL", + "services": [ + "DataPlane" + ], + "stage": "ingress", + "type": "CTRLPLANE" + }, + "NTP_ACL": { + "policy_desc": "NTP_ACL", + "services": [ + "NTP" + ], + "stage": "ingress", + "type": "CTRLPLANE" + }, + "SNMP_ACL": { + "policy_desc": "SNMP_ACL", + "services": [ + "SNMP" + ], + "stage": "ingress", + "type": "CTRLPLANE" + }, + "SSH_ONLY": { + "policy_desc": "SSH_ONLY", + "services": [ + "SSH" + ], + "stage": "ingress", + "type": "CTRLPLANE" + } + }, + "AUTO_TECHSUPPORT": { + "GLOBAL": { + "available_mem_threshold": "10.0", + "max_core_limit": "5.0", + "max_techsupport_limit": "10.0", + "min_available_mem": "200", + "rate_limit_interval": "180", + "since": "2 days ago", + "state": "enabled" + } + }, + "AUTO_TECHSUPPORT_FEATURE": { + "acms": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "bgp": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "disabled" + }, + "database": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "dhcp_relay": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "lldp": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "macsec": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "mux": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "pmon": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "radv": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "restapi": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "snmp": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "swss": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "syncd": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "teamd": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "disabled" + }, + "telemetry": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + }, + "vnet-monitor": { + "available_mem_threshold": "10.0", + "rate_limit_interval": "600", + "state": "enabled" + } + }, + "BGP_DEVICE_GLOBAL": { + "STATE": { + "tsa_enabled": "false" + } + }, + "CONSOLE_SWITCH": { + "console_mgmt": { + "enabled": "no" + } + }, + "CRM": { + "Config": { + "acl_counter_high_threshold": "85", + "acl_counter_low_threshold": "70", + "acl_counter_threshold_type": "percentage", + "acl_entry_high_threshold": "85", + "acl_entry_low_threshold": "70", + "acl_entry_threshold_type": "percentage", + "acl_group_high_threshold": "85", + "acl_group_low_threshold": "70", + "acl_group_threshold_type": "percentage", + "acl_table_high_threshold": "85", + "acl_table_low_threshold": "70", + "acl_table_threshold_type": "percentage", + "dnat_entry_high_threshold": "85", + "dnat_entry_low_threshold": "70", + "dnat_entry_threshold_type": "percentage", + "fdb_entry_high_threshold": "85", + "fdb_entry_low_threshold": "70", + "fdb_entry_threshold_type": "percentage", + "ipmc_entry_high_threshold": "85", + "ipmc_entry_low_threshold": "70", + "ipmc_entry_threshold_type": "percentage", + "ipv4_neighbor_high_threshold": "85", + "ipv4_neighbor_low_threshold": "70", + "ipv4_neighbor_threshold_type": "percentage", + "ipv4_nexthop_high_threshold": "85", + "ipv4_nexthop_low_threshold": "70", + "ipv4_nexthop_threshold_type": "percentage", + "ipv4_route_high_threshold": "85", + "ipv4_route_low_threshold": "70", + "ipv4_route_threshold_type": "percentage", + "ipv6_neighbor_high_threshold": "85", + "ipv6_neighbor_low_threshold": "70", + "ipv6_neighbor_threshold_type": "percentage", + "ipv6_nexthop_high_threshold": "85", + "ipv6_nexthop_low_threshold": "70", + "ipv6_nexthop_threshold_type": "percentage", + "ipv6_route_high_threshold": "85", + "ipv6_route_low_threshold": "70", + "ipv6_route_threshold_type": "percentage", + "mpls_inseg_high_threshold": "85", + "mpls_inseg_low_threshold": "70", + "mpls_inseg_threshold_type": "percentage", + "mpls_nexthop_high_threshold": "85", + "mpls_nexthop_low_threshold": "70", + "mpls_nexthop_threshold_type": "percentage", + "nexthop_group_high_threshold": "85", + "nexthop_group_low_threshold": "70", + "nexthop_group_member_high_threshold": "85", + "nexthop_group_member_low_threshold": "70", + "nexthop_group_member_threshold_type": "percentage", + "nexthop_group_threshold_type": "percentage", + "polling_interval": "300", + "snat_entry_high_threshold": "85", + "snat_entry_low_threshold": "70", + "snat_entry_threshold_type": "percentage" + } + }, + "DHCP_SERVER": { + "192.0.0.1": {}, + "192.0.0.2": {}, + "192.0.0.3": {}, + "192.0.0.4": {}, + "192.0.0.5": {}, + "192.0.0.6": {}, + "192.0.0.7": {}, + "192.0.0.8": {}, + "192.0.0.9": {}, + "192.0.0.10": {}, + "192.0.0.11": {}, + "192.0.0.12": {}, + "192.0.0.13": {}, + "192.0.0.14": {}, + "192.0.0.15": {}, + "192.0.0.16": {}, + "192.0.0.17": {}, + "192.0.0.18": {}, + "192.0.0.19": {}, + "192.0.0.20": {}, + "192.0.0.21": {}, + "192.0.0.22": {}, + "192.0.0.23": {}, + "192.0.0.24": {}, + "192.0.0.25": {}, + "192.0.0.26": {}, + "192.0.0.27": {}, + "192.0.0.28": {}, + "192.0.0.29": {}, + "192.0.0.30": {}, + "192.0.0.31": {}, + "192.0.0.32": {}, + "192.0.0.33": {}, + "192.0.0.34": {}, + "192.0.0.35": {}, + "192.0.0.36": {}, + "192.0.0.37": {}, + "192.0.0.38": {}, + "192.0.0.39": {}, + "192.0.0.40": {}, + "192.0.0.41": {}, + "192.0.0.42": {}, + "192.0.0.43": {}, + "192.0.0.44": {}, + "192.0.0.45": {}, + "192.0.0.46": {}, + "192.0.0.47": {}, + "192.0.0.48": {} + }, + "DSCP_TO_TC_MAP": { + "AZURE": { + "0": "1", + "1": "1", + "10": "1", + "11": "1", + "12": "1", + "13": "1", + "14": "1", + "15": "1", + "16": "1", + "17": "1", + "18": "1", + "19": "1", + "2": "1", + "20": "1", + "21": "1", + "22": "1", + "23": "1", + "24": "1", + "25": "1", + "26": "1", + "27": "1", + "28": "1", + "29": "1", + "3": "3", + "30": "1", + "31": "1", + "32": "1", + "33": "1", + "34": "1", + "35": "1", + "36": "1", + "37": "1", + "38": "1", + "39": "1", + "4": "4", + "40": "1", + "41": "1", + "42": "1", + "43": "1", + "44": "1", + "45": "1", + "46": "5", + "47": "1", + "48": "6", + "49": "1", + "5": "2", + "50": "1", + "51": "1", + "52": "1", + "53": "1", + "54": "1", + "55": "1", + "56": "1", + "57": "1", + "58": "1", + "59": "1", + "6": "1", + "60": "1", + "61": "1", + "62": "1", + "63": "1", + "7": "1", + "8": "0", + "9": "1" + } + }, + "FEATURE": { + "acms": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "disabled" + }, + "bgp": { + "auto_restart": "enabled", + "check_up_status": "false", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "disabled" + }, + "database": { + "auto_restart": "always_enabled", + "has_global_scope": "True", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "always_enabled" + }, + "dhcp_relay": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "disabled" + }, + "lldp": { + "auto_restart": "disabled", + "has_global_scope": "True", + "has_per_asic_scope": "True", + "has_timer": "True", + "high_mem_alert": "disabled", + "set_owner": "kube", + "state": "disabled" + }, + "macsec": { + "auto_restart": "enabled", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "disabled" + }, + "mux": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "always_disabled" + }, + "pmon": { + "auto_restart": "enabled", + "check_up_status": "false", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "kube", + "state": "enabled" + }, + "radv": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "kube", + "state": "enabled" + }, + "restapi": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "disabled" + }, + "snmp": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "True", + "high_mem_alert": "disabled", + "set_owner": "kube", + "state": "enabled" + }, + "swss": { + "auto_restart": "enabled", + "check_up_status": "false", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "enabled" + }, + "syncd": { + "auto_restart": "enabled", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "enabled" + }, + "teamd": { + "auto_restart": "enabled", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "disabled" + }, + "telemetry": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "True", + "high_mem_alert": "disabled", + "set_owner": "kube", + "state": "disabled" + }, + "vnet-monitor": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "set_owner": "local", + "state": "disabled" + } + }, + "FLEX_COUNTER_TABLE": { + "ACL": { + "FLEX_COUNTER_DELAY_STATUS": "false", + "FLEX_COUNTER_STATUS": "enable", + "POLL_INTERVAL": "10000" + }, + "BUFFER_POOL_WATERMARK": { + "FLEX_COUNTER_DELAY_STATUS": "false", + "FLEX_COUNTER_STATUS": "enable" + }, + "PFCWD": { + "FLEX_COUNTER_DELAY_STATUS": "false", + "FLEX_COUNTER_STATUS": "enable" + }, + "PG_DROP": { + "FLEX_COUNTER_DELAY_STATUS": "false", + "FLEX_COUNTER_STATUS": "enable" + }, + "PG_WATERMARK": { + "FLEX_COUNTER_DELAY_STATUS": "false", + "FLEX_COUNTER_STATUS": "enable" + }, + "PORT": { + "FLEX_COUNTER_DELAY_STATUS": "false", + "FLEX_COUNTER_STATUS": "enable" + }, + "PORT_BUFFER_DROP": { + "FLEX_COUNTER_DELAY_STATUS": "false", + "FLEX_COUNTER_STATUS": "enable" + }, + "QUEUE": { + "FLEX_COUNTER_DELAY_STATUS": "false", + "FLEX_COUNTER_STATUS": "enable" + }, + "QUEUE_WATERMARK": { + "FLEX_COUNTER_DELAY_STATUS": "false", + "FLEX_COUNTER_STATUS": "enable" + }, + "RIF": { + "FLEX_COUNTER_DELAY_STATUS": "false", + "FLEX_COUNTER_STATUS": "enable" + } + }, + "KDUMP": { + "config": { + "enabled": "false", + "memory": "0M-2G:256M,2G-4G:320M,4G-8G:384M,8G-:448M", + "num_dumps": "3" + } + }, + "MAP_PFC_PRIORITY_TO_QUEUE": { + "AZURE": { + "0": "0", + "1": "1", + "2": "2", + "3": "3", + "4": "4", + "5": "5", + "6": "6", + "7": "7" + } + }, + "NTP_SERVER": { +{% for ntp_server in ntp_servers %} + "{{ ntp_server }}": {}{% if not loop.last %},{% endif %} +{% endfor %} + + }, + "PASSW_HARDENING": { + "POLICIES": { + "digits_class": "true", + "expiration": "180", + "expiration_warning": "15", + "history_cnt": "10", + "len_min": "8", + "lower_class": "true", + "reject_user_passw_match": "true", + "special_class": "true", + "state": "disabled", + "upper_class": "true" + } + }, + "SCHEDULER": { + "scheduler.0": { + "type": "DWRR", + "weight": "14" + }, + "scheduler.1": { + "type": "DWRR", + "weight": "15" + } + }, + "SNMP": { + "LOCATION": { + "Location": "public" + } + }, + "SNMP_COMMUNITY": { + "public": { + "TYPE": "RO" + } + }, + "TC_TO_PRIORITY_GROUP_MAP": { + "AZURE": { + "0": "0", + "1": "0", + "2": "0", + "3": "3", + "4": "4", + "5": "0", + "6": "0", + "7": "7" + } + }, + "TC_TO_QUEUE_MAP": { + "AZURE": { + "0": "0", + "1": "1", + "2": "2", + "3": "3", + "4": "4", + "5": "5", + "6": "6", + "7": "7" + } + }, + "VERSIONS": { + "DATABASE": { + "VERSION": "version_3_0_7" + } + }, + "WRED_PROFILE": { + "AZURE_LOSSLESS": { + "ecn": "ecn_green_yellow", + "green_drop_probability": "5", + "green_max_threshold": "4194304", + "green_min_threshold": "1048576", + "wred_green_enable": "true", + "wred_yellow_enable": "true", + "yellow_drop_probability": "0", + "yellow_max_threshold": "6144000", + "yellow_min_threshold": "0" + } + } +} diff --git a/ansible/roles/test/files/acstests/acltb_test.py b/ansible/roles/test/files/acstests/acltb_test.py index a364518b93..20742af33d 100644 --- a/ansible/roles/test/files/acstests/acltb_test.py +++ b/ansible/roles/test/files/acstests/acltb_test.py @@ -14,7 +14,7 @@ dst_ip_tor='172.16.1.0';dst_ip_tor_forwarded='172.16.2.0';dst_ip_tor_blocked='172.16.3.0'; dst_ip_spine='192.168.0.0';dst_ip_spine_forwarded='192.168.0.16';dst_ip_spine_blocked='192.168.0.17'" ''' -from __future__ import print_function + import logging @@ -409,7 +409,7 @@ def runTest(self): self.tor_ports, "spine->tor") - failed_cases = filter(lambda r: not r['result'], self.test_results) + failed_cases = [r for r in self.test_results if not r['result']] if len(failed_cases) == 0: print('!!!! All test cases passed! !!!!') assert (len(failed_cases) == 0), "TEST FAILED. Failed test cases: " + str(failed_cases) diff --git a/ansible/roles/test/files/acstests/everflow_policer_test.py b/ansible/roles/test/files/acstests/everflow_policer_test.py index de4e0fd06d..09c5cc96b9 100644 --- a/ansible/roles/test/files/acstests/everflow_policer_test.py +++ b/ansible/roles/test/files/acstests/everflow_policer_test.py @@ -216,7 +216,7 @@ def checkMirroredFlow(self): if self.asic_type in ["mellanox"]: import binascii payload = binascii.unhexlify("0"*44) + str(payload) # Add the padding - elif self.asic_type in ["innovium"]: + elif self.asic_type in ["innovium"] or self.hwsku in ["rd98DX35xx_cn9131", "rd98DX35xx", "Nokia-7215-A1"]: import binascii payload = binascii.unhexlify("0"*24) + str(payload) # Add the padding @@ -252,6 +252,7 @@ def checkMirroredFlow(self): masked_exp_pkt.set_do_not_care_scapy(scapy.GRE, "seqnum_present") if self.asic_type in ["marvell"]: masked_exp_pkt.set_do_not_care_scapy(scapy.IP, "id") + masked_exp_pkt.set_do_not_care_scapy(scapy.GRE, "seqnum_present") if exp_pkt.haslayer(scapy.ERSPAN_III): masked_exp_pkt.set_do_not_care_scapy(scapy.ERSPAN_III, "span_id") @@ -269,7 +270,7 @@ def match_payload(pkt): pkt = scapy.Ether(pkt).load pkt = pkt[22:] # Mask the Mellanox specific inner header pkt = scapy.Ether(pkt) - elif self.asic_type in ["innovium"]: + elif self.asic_type in ["innovium"] or self.hwsku in ["rd98DX35xx_cn9131", "rd98DX35xx", "Nokia-7215-A1"]: pkt = scapy.Ether(pkt)[scapy.GRE].payload pkt_str = str(pkt) pkt = scapy.Ether(pkt_str[8:]) diff --git a/ansible/roles/test/files/acstests/lag_test.py b/ansible/roles/test/files/acstests/py3/lag_test.py similarity index 99% rename from ansible/roles/test/files/acstests/lag_test.py rename to ansible/roles/test/files/acstests/py3/lag_test.py index 61cc1e4366..3786bf30e7 100644 --- a/ansible/roles/test/files/acstests/lag_test.py +++ b/ansible/roles/test/files/acstests/py3/lag_test.py @@ -117,7 +117,7 @@ def getMedianInterval(self, masked_exp_pkt): # Get the median intervals.sort() - current_pkt_timing = intervals[self.interval_count / 2] + current_pkt_timing = intervals[int(self.interval_count / 2)] return current_pkt_timing def runTest(self): diff --git a/ansible/roles/test/files/acstests/py3/router_utils.py b/ansible/roles/test/files/acstests/py3/router_utils.py new file mode 120000 index 0000000000..080c70debc --- /dev/null +++ b/ansible/roles/test/files/acstests/py3/router_utils.py @@ -0,0 +1 @@ +../router_utils.py \ No newline at end of file diff --git a/ansible/roles/test/files/helpers/bfd_responder.py b/ansible/roles/test/files/helpers/bfd_responder.py index 0447a3284b..fce30d0b0d 100644 --- a/ansible/roles/test/files/helpers/bfd_responder.py +++ b/ansible/roles/test/files/helpers/bfd_responder.py @@ -12,6 +12,9 @@ logging.getLogger("scapy.runtime").setLevel(logging.ERROR) scapy2.conf.use_pcap = True +IPv4 = '4' +IPv6 = '6' + def get_if(iff, cmd): s = socket.socket() @@ -78,7 +81,7 @@ def poll(self): class BFDResponder(object): def __init__(self, sessions): self.sessions = sessions - self.bfd_default_ip_tos = 192 + self.bfd_default_ip_priority = 192 return def action(self, interface): @@ -111,13 +114,15 @@ def extract_bfd_info(self, data): mac_dst = ether.dst ip_src = ether.payload.src ip_dst = ether.payload.dst - ip_tos = ether.payload.tos + ip_version = str(ether.payload.version) + ip_priority_field = 'tos' if ip_version == IPv4 else 'tc' + ip_priority = getattr(ether.payload, ip_priority_field) bfdpkt = BFD(ether.payload.payload.payload.load) bfd_remote_disc = bfdpkt.my_discriminator bfd_state = bfdpkt.sta - if ip_tos != self.bfd_default_ip_tos: - raise RuntimeError("Received BFD packet with incorrect tos: {}".format(ip_tos)) - logging.debug('BFD packet info: sip {}, dip {}, tos {}'.format(ip_src, ip_dst, ip_tos)) + if ip_priority != self.bfd_default_ip_priority: + raise RuntimeError("Received BFD packet with incorrect priority value: {}".format(ip_priority)) + logging.debug('BFD packet info: sip {}, dip {}, priority {}'.format(ip_src, ip_dst, ip_priority)) return mac_src, mac_dst, ip_src, ip_dst, bfd_remote_disc, bfd_state def craft_bfd_packet(self, session, data, mac_src, mac_dst, ip_src, ip_dst, bfd_remote_disc, bfd_state): diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/Dockerfile b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/Dockerfile new file mode 100644 index 0000000000..be79593c23 --- /dev/null +++ b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.9-bullseye + +RUN mkdir -m 0755 -p /root/pkgs + +COPY ./pkgs /root/pkgs +COPY ["install.sh", "start.sh", "pfc_gen.py", "pfc_gen_cpu.py", "/root/"] + +RUN /root/install.sh +RUN echo "export PYTHONPATH=/usr/lib/python3/dist-packages/" >> /root/.bashrc + +CMD ["/root/start.sh"] diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/Makefile b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/Makefile new file mode 100644 index 0000000000..b8fc747f21 --- /dev/null +++ b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/Makefile @@ -0,0 +1,9 @@ +all: save + +build: Dockerfile + cp ../../helpers/pfc_gen.py ./pfc_gen_cpu.py + docker build -t pfc_storm . + rm ./pfc_gen_cpu.py + +save: build + docker save pfc_storm:latest | gzip >pfc_storm.tgz diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/install.sh b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/install.sh new file mode 100755 index 0000000000..f921bc5e8d --- /dev/null +++ b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/install.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +dpkg -i /root/pkgs/*.deb diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pfc_gen.py b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pfc_gen.py new file mode 100755 index 0000000000..000d7397c4 --- /dev/null +++ b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pfc_gen.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python +''' +This file contains Python command to enable PFC storm using SDK APIs on a fanout switch. +The idea is + 1. Set QoS mappings + 2. Set PG to lossless and enable the shared headroom pool + 3. Set shared headroom pool size to 0 + 4. The PFC will be triggered + +''' +import re +import sys +import time +from python_sdk_api.sx_api import * # noqa F401 +import argparse +import logging +import logging.handlers + + +class SdkError(Exception): + pass + + +def on_sdk_error(info): + print(info) + raise SdkError + + +PG_SIZE = 200 +PG_XON = 100 +PG_XOFF = 100 +LOSSY = 1 +LOSSLESS = 0 + + +def pfc_port_set(handle, port, priority, fc_mode): + rc = sx_api_port_pfc_enable_set(handle, port, priority, fc_mode) # noqa F405 + if rc != SX_STATUS_SUCCESS: # noqa F405 + on_sdk_error(("sx_api_port_pfc_enable_set: 0x%x enabled, rc %d" % (port, rc))) + + +switch_prio_p = new_sx_cos_priority_t_arr(1) # noqa F405 +ieee_prio_p = new_sx_cos_ieee_prio_t_arr(1) # noqa F405 +prio_to_buff_p = new_sx_cos_port_prio_buff_t_p() # noqa F405 + + +def set_ieee_priority_sp_mapping(handle, port, priority): + sx_cos_priority_t_arr_setitem(switch_prio_p, 0, priority) # noqa F405 + sx_cos_ieee_prio_t_arr_setitem(ieee_prio_p, 0, priority) # noqa F405 + + rc = sx_api_cos_prio_to_ieeeprio_set(handle, switch_prio_p, ieee_prio_p, 1) # noqa F405 + if rc != SX_STATUS_SUCCESS: # noqa F405 + on_sdk_error("Failed to set priority to IEEE priority mapping, rc = %d\n" % (rc)) + + +def set_port_prio_buffer_mapping(handle, port, sp, pg): + rc = sx_api_cos_port_prio_buff_map_get(handle, port, prio_to_buff_p) # noqa F405 + if rc != SX_STATUS_SUCCESS: # noqa F405 + on_sdk_error("sx_api_cos_port_prio_buff_map_get failed, rc = %d\n" % (rc)) # noqa F405 + + sx_cos_port_buff_t_arr_setitem(prio_to_buff_p.prio_to_buff, sp, pg) # noqa F405 + + rc = sx_api_cos_port_prio_buff_map_set(handle, SX_ACCESS_CMD_SET, port, prio_to_buff_p) # noqa F405 + if rc != SX_STATUS_SUCCESS: # noqa F405 + on_sdk_error("sx_api_cos_port_prio_buff_map_set failed, rc = %d\n" % (rc)) + + +def set_port_pg_lossless(handle, log_port, pg, enable): + port_buffer_attr_list_p = new_sx_cos_port_buffer_attr_t_arr(1) # noqa F405 + + attr_item_min = sx_cos_port_buffer_attr_t_arr_getitem(port_buffer_attr_list_p, 0) # noqa F405 + + attr_item_min.type = SX_COS_INGRESS_PORT_PRIORITY_GROUP_ATTR_E # noqa F405 + attr_item_min.attr.ingress_port_pg_buff_attr.pg = pg + attr_item_min.attr.ingress_port_pg_buff_attr.pool_id = 0 + if enable: + attr_item_min.attr.ingress_port_pg_buff_attr.is_lossy = LOSSLESS + attr_item_min.attr.ingress_port_pg_buff_attr.xon = PG_XON + attr_item_min.attr.ingress_port_pg_buff_attr.xoff = PG_XOFF + attr_item_min.attr.ingress_port_pg_buff_attr.use_shared_headroom = 1 + attr_item_min.attr.ingress_port_pg_buff_attr.size = PG_SIZE + else: + attr_item_min.attr.ingress_port_pg_buff_attr.is_lossy = LOSSY + attr_item_min.attr.ingress_port_pg_buff_attr.xon = 0 + attr_item_min.attr.ingress_port_pg_buff_attr.xoff = 0 + attr_item_min.attr.ingress_port_pg_buff_attr.use_shared_headroom = 0 + attr_item_min.attr.ingress_port_pg_buff_attr.size = 0 + + print("\nSetting Port PG:") + print("type = SX_COS_INGRESS_PORT_PRIORITY_GROUP_ATTR_E") + print(("size = %d" % (attr_item_min.attr.ingress_port_pg_buff_attr.size))) + print(("PG = %d" % (attr_item_min.attr.ingress_port_pg_buff_attr.pg))) + print(("is_lossy (0=Lossless) = %d" % (attr_item_min.attr.ingress_port_pg_buff_attr.is_lossy))) + print(("Xon = %d" % (attr_item_min.attr.ingress_port_pg_buff_attr.xon))) + print(("Xoff = %d" % (attr_item_min.attr.ingress_port_pg_buff_attr.xoff))) + print(("use_shared_headroom = %d" % (attr_item_min.attr.ingress_port_pg_buff_attr.use_shared_headroom))) + + attr_item_min = sx_cos_port_buffer_attr_t_arr_setitem(port_buffer_attr_list_p, 0, attr_item_min) # noqa F405 + + rc = sx_api_cos_port_buff_type_set(handle, SX_ACCESS_CMD_SET, log_port, port_buffer_attr_list_p, 1) # noqa F405 + if rc != SX_STATUS_SUCCESS: # noqa F405 + sys.exit(rc) + on_sdk_error(("sx_api_cos_port_buff_type_set [cmd=%d, log_port=0x%x , cnt=%d, rc=%d] " % + (SX_ACCESS_CMD_SET, log_port, 1, rc))) # noqa F405 + + +def build_port_name_dict(): + result = {} + for port in range(64): + result['ethsl1p{}'.format(port+1)] = port, 0 + for split_port in range(8): + result['ethsl1p{}sp{}'.format(port+1, split_port+1)] = port, split_port + + return result + + +def get_port_attibutes(handle): + # Get ports count + port_cnt_p = new_uint32_t_p() # noqa F405 + uint32_t_p_assign(port_cnt_p, 0) # noqa F405 + port_attributes_list = new_sx_port_attributes_t_arr(0) # noqa F405 + rc = sx_api_port_device_get(handle, 1, 0, port_attributes_list, port_cnt_p) # noqa F405 + if rc != SX_STATUS_SUCCESS: # noqa F405 + print("sx_api_port_device_get failed, rc = %d" % (rc)) + sys.exit(rc) + port_cnt = uint32_t_p_value(port_cnt_p) # noqa F405 + + # Get ports + port_attributes_list = new_sx_port_attributes_t_arr(port_cnt) # noqa F405 + rc = sx_api_port_device_get(handle, 1, 0, port_attributes_list, port_cnt_p) # noqa F405 + if (rc != SX_STATUS_SUCCESS): # noqa F405 + print("sx_api_port_device_get failed, rc = %d") + sys.exit(rc) + return port_attributes_list, port_cnt + + +def get_label_log_port_map(handle): + label_log_port_map = {} + port_attributes_list, port_cnt = get_port_attibutes(handle) + for i in range(0, port_cnt): + port_attributes = sx_port_attributes_t_arr_getitem(port_attributes_list, i) # noqa F405 + log_port = int(port_attributes.log_port) + label_port = port_attributes.port_mapping.module_port + 1 + if label_port not in label_log_port_map: + label_log_port_map[label_port] = [log_port] + else: + label_log_port_map[label_port].append(log_port) + + return label_log_port_map + + +def get_port_map(handle): + portmap = {} + port_attributes_list, port_cnt = get_port_attibutes(handle) + for i in range(0, port_cnt): + port_attributes = sx_port_attributes_t_arr_getitem(port_attributes_list, i) # noqa F405 + log_port = int(port_attributes.log_port) + subports = portmap.get(port_attributes.port_mapping.module_port) + if subports: + subports.append(log_port) + subports.sort() + else: + subports = [log_port] + + portmap[port_attributes.port_mapping.module_port] = subports + + return portmap + + +def parse_priority(x): + priorities = [] + for i in range(8): + if (1 << i) & x: + priorities.append(i) + return priorities + + +def get_log_ports(handle, args): + log_ports = [] + if args.label_port_list: + label_log_port_map = get_label_log_port_map(handle) + for port_index in args.label_port_list.split(','): + label_port = re.findall(r'\d+', port_index)[0] + matched_str_list = re.findall(r'[a-z]?', port_index) + index_str_list = list(filter(None, matched_str_list)) + if index_str_list == []: + log_port_index = 0 + else: + log_port_index = ord(index_str_list[0]) - ord('a') + log_ports.append(label_log_port_map[int(label_port)][log_port_index]) + else: + portmap = get_port_map(handle) + print(portmap) + port_name_dict = build_port_name_dict() + for port in args.interface_list.split(','): + p, sp = port_name_dict[port] + log_port = portmap[p][sp] + print(log_port) + log_ports.append(log_port) + return log_ports + + +def start_pfc(handle, log_ports, priority): + for log_port in log_ports: + set_ieee_priority_sp_mapping(handle, log_port, priority) + set_port_prio_buffer_mapping(handle, log_port, priority, priority) + + pfc_port_set(handle, log_port, priority, SX_PORT_FLOW_CTRL_MODE_TX_EN_RX_EN) # noqa F405 + + # Set port PG to use the port shared headroom + set_port_pg_lossless(handle, log_port, priority, True) + + +def stop_pfc(handle, log_ports, priority): + for log_port in log_ports: + set_port_pg_lossless(handle, log_port, priority, False) + pfc_port_set(handle, log_port, priority, SX_PORT_FLOW_CTRL_MODE_TX_DIS_RX_DIS) # noqa F405 + set_port_prio_buffer_mapping(handle, log_port, priority, 0) + +###################################################### +# main +###################################################### + + +def main(): + + parser = argparse.ArgumentParser(description='pfc_gen.py -i -p ', + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("-i", "--interface_list", type=str, help="Interface list to send packets, seperated by ','", + required=True) + parser.add_argument("-l", "--label_port_list", + type=str, help="SDK label port list to send packets, separated by ','", default="") + parser.add_argument('-p', "--priority", type=int, help="PFC class enable bitmap.", default=-1) + parser.add_argument('-d', "--disable", action='store_true', help="PFC class enable bitmap.") + parser.add_argument("-r", "--rsyslog-server", type=str, default="127.0.0.1", help="Rsyslog server IPv4 address") + parser.add_argument("-n", "--num", type=int, help="Number of packets to be sent", default=1) + parser.add_argument("-s", "--send-pfc-frame-interval", type=float, help="Interval between two PFC frames", + default=0) + args = parser.parse_args() + + print("[+] opening sdk") + rc, handle = sx_api_open(None) # noqa F405 + print(("sx_api_open handle:0x%x , rc %d " % (handle, rc))) + if (rc != SX_STATUS_SUCCESS): # noqa F405 + print("Failed to open api handle.\nPlease check that SDK is running.") + sys.exit(rc) + + log_ports = get_log_ports(handle, args) + priority = parse_priority(args.priority)[0] + + logger = logging.getLogger('MyLogger') + logger.setLevel(logging.DEBUG) + # Configure logging + handler = logging.handlers.SysLogHandler(address=(args.rsyslog_server, 514)) + logger.addHandler(handler) + + if args.disable: + print("disable") + stop_pfc(handle, log_ports, priority) + logger.debug('PFC_STORM_END') + sys.exit(0) + + try: + start_pfc(handle, log_ports, priority) + logger.debug('PFC_STORM_START') + if args.num != 0 and args.send_pfc_frame_interval != 0: + total_time = args.num / args.send_pfc_frame_interval + time.sleep(total_time) + stop_pfc(handle, log_ports, priority) + logger.debug('PFC_STORM_END') + except SdkError: + stop_pfc(handle, log_ports, priority) + + +if __name__ == "__main__": + main() diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/applibs_1.mlnx.4.5.4414_amd64.deb b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/applibs_1.mlnx.4.5.4414_amd64.deb new file mode 100644 index 0000000000..be71a0c9cc Binary files /dev/null and b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/applibs_1.mlnx.4.5.4414_amd64.deb differ diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/libnl-3-200_3.5.0-1_amd64.deb b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/libnl-3-200_3.5.0-1_amd64.deb new file mode 100644 index 0000000000..80ee4d4e98 Binary files /dev/null and b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/libnl-3-200_3.5.0-1_amd64.deb differ diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/python-sdk-api_1.mlnx.4.5.4414_amd64.deb b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/python-sdk-api_1.mlnx.4.5.4414_amd64.deb new file mode 100644 index 0000000000..6aeec19ff8 Binary files /dev/null and b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/python-sdk-api_1.mlnx.4.5.4414_amd64.deb differ diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/sx-complib_1.mlnx.4.5.4414_amd64.deb b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/sx-complib_1.mlnx.4.5.4414_amd64.deb new file mode 100644 index 0000000000..958d5c5c62 Binary files /dev/null and b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/sx-complib_1.mlnx.4.5.4414_amd64.deb differ diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/sx-gen-utils_1.mlnx.4.5.4414_amd64.deb b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/sx-gen-utils_1.mlnx.4.5.4414_amd64.deb new file mode 100644 index 0000000000..e5c608be97 Binary files /dev/null and b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/sx-gen-utils_1.mlnx.4.5.4414_amd64.deb differ diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/sxd-libs_1.mlnx.4.5.4414_amd64.deb b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/sxd-libs_1.mlnx.4.5.4414_amd64.deb new file mode 100644 index 0000000000..06f55e4554 Binary files /dev/null and b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/pkgs/sxd-libs_1.mlnx.4.5.4414_amd64.deb differ diff --git a/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/start.sh b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/start.sh new file mode 100755 index 0000000000..66aab6a250 --- /dev/null +++ b/ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/start.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +sleep inf diff --git a/ansible/roles/test/files/ptftests/arista.py b/ansible/roles/test/files/ptftests/arista.py index 896d949fc2..0c0926659c 100644 --- a/ansible/roles/test/files/ptftests/arista.py +++ b/ansible/roles/test/files/ptftests/arista.py @@ -290,8 +290,9 @@ def extract_from_logs(self, regexp, data): m = re_compiled.match(line) if not m: continue + # add year to avoid ValueError exception for FEB29 during leap year raw_data.append((datetime.datetime.strptime( - m.group(1), "%b %d %X"), m.group(2), m.group(3))) + str(datetime.datetime.now().year) + " " + m.group(1), "%Y %b %d %X"), m.group(2), m.group(3))) if len(raw_data) > 0: initial_time = raw_data[0][0] diff --git a/ansible/roles/test/files/ptftests/py3/advanced-reboot.py b/ansible/roles/test/files/ptftests/py3/advanced-reboot.py index 09c4972cd9..a535909f45 100644 --- a/ansible/roles/test/files/ptftests/py3/advanced-reboot.py +++ b/ansible/roles/test/files/ptftests/py3/advanced-reboot.py @@ -628,6 +628,7 @@ def sad_revert(self): def setUp(self): self.fails['dut'] = set() + self.fails['infrastructure'] = set() self.dut_mac = self.test_params['dut_mac'] self.vlan_mac = self.test_params['vlan_mac'] self.lo_prefix = self.test_params['lo_prefix'] @@ -1943,14 +1944,17 @@ def examine_flow(self, filename=None): self.lost_packets = dict() self.max_disrupt, self.total_disruption = 0, 0 sent_packets = dict() + # Track packet id's that were neither sent or received + missing_sent_and_received_packet_id_sequences = [] self.fails['dut'].add("Sniffer failed to capture any traffic") self.assertTrue(packets, "Sniffer failed to capture any traffic") self.fails['dut'].clear() prev_payload = None if packets: - prev_payload, prev_time = 0, 0 + prev_payload, prev_time = -1, 0 sent_payload = 0 received_counter = 0 # Counts packets from dut. + received_but_not_sent_packets = set() sent_counter = 0 received_t1_to_vlan = 0 received_vlan_to_t1 = 0 @@ -1985,31 +1989,60 @@ def examine_flow(self, filename=None): prev_time = received_time continue if received_payload - prev_payload > 1: - # Packets in a row are missing, a disruption. + if received_payload not in sent_packets: + self.log("Ignoring received packet with payload {}, as it was not sent".format( + received_payload)) + received_but_not_sent_packets.add(received_payload) + continue + # Packets in a row are missing, a potential disruption. self.log("received_payload: {}, prev_payload: {}, sent_counter: {}, received_counter: {}".format( received_payload, prev_payload, sent_counter, received_counter)) # How many packets lost in a row. lost_id = (received_payload - 1) - prev_payload - # How long disrupt lasted. - disrupt = ( - sent_packets[received_payload] - sent_packets[prev_payload + 1]) - # Add disrupt to the dict: - self.lost_packets[prev_payload] = ( - lost_id, disrupt, received_time - disrupt, received_time) - self.log("Disruption between packet ID %d and %d. For %.4f " % ( - prev_payload, received_payload, disrupt)) - for lost_index in range(prev_payload + 1, received_payload): - # lost received for packet sent from vlan to T1. - if (lost_index % 5) == 0: - missed_vlan_to_t1 += 1 + + # Find previous sequential sent packet that was captured + missing_sent_and_received_pkt_count = 0 + prev_pkt_pt = prev_payload + 1 + prev_sent_packet_time = None + while prev_pkt_pt < received_payload: + if prev_pkt_pt in sent_packets: + prev_sent_packet_time = sent_packets[prev_pkt_pt] + break # Found it else: - missed_t1_to_vlan += 1 - self.log("") - if not self.disruption_start: - self.disruption_start = datetime.datetime.fromtimestamp( - prev_time) - self.disruption_stop = datetime.datetime.fromtimestamp( - received_time) + if prev_pkt_pt not in received_but_not_sent_packets: + missing_sent_and_received_pkt_count += 1 + prev_pkt_pt += 1 + if missing_sent_and_received_pkt_count > 0: + missing_sent_and_received_packet_id_sequences_fmtd = \ + str(prev_payload + 1) if missing_sent_and_received_pkt_count == 1\ + else "{}-{}".format(prev_payload + 1, received_payload - 1) + missing_sent_and_received_packet_id_sequences.append( + missing_sent_and_received_packet_id_sequences_fmtd) + if prev_sent_packet_time is not None: + # Disruption occurred - some sent packets were not received + + # How long disrupt lasted. + this_sent_packet_time = sent_packets[received_payload] + disrupt = this_sent_packet_time - prev_sent_packet_time + + # Add disrupt to the dict: + self.lost_packets[prev_payload] = ( + lost_id, disrupt, received_time - disrupt, received_time) + self.log("Disruption between packet ID %d and %d. For %.4f " % ( + prev_payload, received_payload, disrupt)) + for lost_index in range(prev_payload + 1, received_payload): + # lost received for packet sent from vlan to T1. + if lost_index in sent_packets: + if (lost_index % 5) == 0: + missed_vlan_to_t1 += 1 + else: + missed_t1_to_vlan += 1 + self.log("") + if not self.disruption_start: + self.disruption_start = datetime.datetime.fromtimestamp( + prev_time) + self.disruption_stop = datetime.datetime.fromtimestamp( + received_time) prev_payload = received_payload prev_time = received_time self.log( @@ -2043,6 +2076,11 @@ def examine_flow(self, filename=None): self.total_disrupt_time = 0 self.log("Gaps in forwarding not found.") + if missing_sent_and_received_packet_id_sequences: + self.fails["infrastructure"].add( + "Missing sent and received packets: {}" + .format(missing_sent_and_received_packet_id_sequences)) + self.dataplane_loss_checked_successfully = True if self.reboot_type == "fast-reboot" and not self.lost_packets: @@ -2142,8 +2180,8 @@ def wait_dut_to_warm_up(self): up_time = None if elapsed > warm_up_timeout_secs: - raise Exception("IO didn't come up within warm up timeout. Control plane: {}, Data plane: {}".format( - ctrlplane, dataplane)) + raise Exception("IO didn't come up within warm up timeout. Control plane: {}, Data plane: {}." + "Actual warm up time {}".format(ctrlplane, dataplane, elapsed)) time.sleep(1) # check until flooding is over. Flooding happens when FDB entry of diff --git a/ansible/roles/test/files/ptftests/py3/copp_tests.py b/ansible/roles/test/files/ptftests/py3/copp_tests.py index 898f64bec6..6ad9e1a487 100644 --- a/ansible/roles/test/files/ptftests/py3/copp_tests.py +++ b/ansible/roles/test/files/ptftests/py3/copp_tests.py @@ -45,6 +45,9 @@ class ControlPlaneBaseTest(BaseTest): PPS_LIMIT = 600 PPS_LIMIT_MIN = PPS_LIMIT * 0.9 PPS_LIMIT_MAX = PPS_LIMIT * 1.3 + DEFAULT_PPS_LIMIT = 300 + DEFAULT_PPS_LIMIT_MIN = DEFAULT_PPS_LIMIT * 0.9 + DEFAULT_PPS_LIMIT_MAX = DEFAULT_PPS_LIMIT * 1.3 NO_POLICER_LIMIT = PPS_LIMIT * 1.4 TARGET_PORT = "3" # Historically we have port 3 as a target port TASK_TIMEOUT = 600 # Wait up to 10 minutes for tasks to complete @@ -77,6 +80,8 @@ def __init__(self): self.hw_sku == "Cisco-8111-C32" or self.hw_sku == "Cisco-8111-O62C2"): self.PPS_LIMIT_MAX = self.PPS_LIMIT * 1.4 + self.asic_type = test_params.get('asic_type', None) + self.topo_type = test_params.get('topo_type', None) def log(self, message, debug=False): current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -260,7 +265,8 @@ def check_constraints(self, send_count, recv_count, time_delta_ms, rx_pps): int(self.PPS_LIMIT_MAX), str(self.PPS_LIMIT_MIN <= rx_pps <= self.PPS_LIMIT_MAX)) ) - assert self.PPS_LIMIT_MIN <= rx_pps <= self.PPS_LIMIT_MAX, "rx_pps {}".format(rx_pps) + assert self.PPS_LIMIT_MIN <= rx_pps <= self.PPS_LIMIT_MAX, "Copp policer constraint check failed, " \ + "Actual PPS: {} Expected PPS range: {} - {}".format(rx_pps, self.PPS_LIMIT_MIN, self.PPS_LIMIT_MAX) else: self.log("Checking constraints (NoPolicyApplied):") self.log( @@ -269,7 +275,8 @@ def check_constraints(self, send_count, recv_count, time_delta_ms, rx_pps): int(self.PPS_LIMIT_MIN), str(rx_pps <= self.PPS_LIMIT_MIN)) ) - assert rx_pps <= self.PPS_LIMIT_MIN, "rx_pps {}".format(rx_pps) + assert rx_pps <= self.PPS_LIMIT_MIN, "Copp policer constraint check failed, Actual PPS: {} " \ + "Expected PPS range: 0 - {}".format(rx_pps, self.PPS_LIMIT_MIN) # SONIC config contains policer CIR=600 for ARP @@ -336,10 +343,20 @@ def contruct_packet(self, port_number): return packet -# SONIC config contains policer CIR=300 for DHCP +# SONIC config contains policer CIR=100 for DHCP class DHCPTest(PolicyTest): def __init__(self): PolicyTest.__init__(self) + # Marvell based platforms have cir/cbs in steps of 125 + if self.hw_sku in {"Nokia-M0-7215", "Nokia-7215", "Nokia-7215-A1"}: + self.PPS_LIMIT = 250 + # M0 devices have CIR of 300 for DHCP + elif self.topo_type in {"m0", "mx"}: + self.PPS_LIMIT = 300 + else: + self.PPS_LIMIT = 100 + self.PPS_LIMIT_MIN = self.PPS_LIMIT * 0.9 + self.PPS_LIMIT_MAX = self.PPS_LIMIT * 1.3 def runTest(self): self.log("DHCPTest") @@ -370,10 +387,20 @@ def contruct_packet(self, port_number): return packet -# SONIC config contains policer CIR=300 for DHCPv6 +# SONIC config contains policer CIR=100 for DHCPv6 class DHCP6Test(PolicyTest): def __init__(self): PolicyTest.__init__(self) + # Marvell based platforms have cir/cbs in steps of 125 + if self.hw_sku in {"Nokia-M0-7215", "Nokia-7215", "Nokia-7215-A1"}: + self.PPS_LIMIT = 250 + # M0 devices have CIR of 300 for DHCP + elif self.topo_type in {"m0", "mx"}: + self.PPS_LIMIT = 300 + else: + self.PPS_LIMIT = 100 + self.PPS_LIMIT_MIN = self.PPS_LIMIT * 0.9 + self.PPS_LIMIT_MAX = self.PPS_LIMIT * 1.3 def runTest(self): self.log("DHCP6Test") @@ -423,10 +450,20 @@ def contruct_packet(self, port_number): return packet -# SONIC config contains policer CIR=300 for LLDP +# SONIC config contains policer CIR=100 for LLDP class LLDPTest(PolicyTest): def __init__(self): PolicyTest.__init__(self) + # Marvell based platforms have cir/cbs in steps of 125 + if self.hw_sku in {"Nokia-M0-7215", "Nokia-7215", "Nokia-7215-A1"}: + self.PPS_LIMIT = 250 + # M0 devices have CIR of 300 for DHCP + elif self.topo_type in {"m0", "mx"}: + self.PPS_LIMIT = 300 + else: + self.PPS_LIMIT = 100 + self.PPS_LIMIT_MIN = self.PPS_LIMIT * 0.9 + self.PPS_LIMIT_MAX = self.PPS_LIMIT * 1.3 def runTest(self): self.log("LLDPTest") @@ -444,10 +481,20 @@ def contruct_packet(self, port_number): return packet -# SONIC config contains policer CIR=300 for UDLD +# SONIC config contains policer CIR=100 for UDLD class UDLDTest(PolicyTest): def __init__(self): PolicyTest.__init__(self) + # Marvell based platforms have cir/cbs in steps of 125 + if self.hw_sku in {"Nokia-M0-7215", "Nokia-7215", "Nokia-7215-A1"}: + self.PPS_LIMIT = 250 + # M0 devices have CIR of 300 for DHCP + elif self.topo_type in {"m0", "mx"}: + self.PPS_LIMIT = 300 + else: + self.PPS_LIMIT = 100 + self.PPS_LIMIT_MIN = self.PPS_LIMIT * 0.9 + self.PPS_LIMIT_MAX = self.PPS_LIMIT * 1.3 def runTest(self): self.log("UDLDTest") @@ -492,6 +539,42 @@ def contruct_packet(self, port_number): return packet + def check_constraints(self, send_count, recv_count, time_delta_ms, rx_pps): + self.log("") + if self.has_trap: + self.log("Checking constraints (PolicyApplied):") + self.log( + "PPS_LIMIT_MIN (%d) <= rx_pps (%d) <= PPS_LIMIT_MAX (%d): %s" % + (int(self.PPS_LIMIT_MIN), + int(rx_pps), + int(self.PPS_LIMIT_MAX), + str(self.PPS_LIMIT_MIN <= rx_pps <= self.PPS_LIMIT_MAX)) + ) + assert self.PPS_LIMIT_MIN <= rx_pps <= self.PPS_LIMIT_MAX, "Copp policer constraint check failed, " \ + "Actual PPS: {} Expected PPS range: {} - {}".format(rx_pps, self.PPS_LIMIT_MIN, self.PPS_LIMIT_MAX) + elif self.asic_type not in ['broadcom']: + self.log("Checking constraints (NoPolicyApplied):") + self.log( + "rx_pps (%d) <= PPS_LIMIT_MIN (%d): %s" % + (int(rx_pps), + int(self.PPS_LIMIT_MIN), + str(rx_pps <= self.PPS_LIMIT_MIN)) + ) + assert rx_pps <= self.PPS_LIMIT_MIN, "Copp policer constraint check failed, Actual PPS: {} " \ + "Expected PPS range: 0 - {}".format(rx_pps, self.PPS_LIMIT_MIN) + else: + self.log("Checking constraints (DefaultPolicyApplied):") + self.log( + "DEFAULT_PPS_LIMIT_MIN (%d) <= rx_pps (%d) <= DEFAULT_PPS_LIMIT_MAX (%d): %s" % + (int(self.DEFAULT_PPS_LIMIT_MIN), + int(rx_pps), + int(self.DEFAULT_PPS_LIMIT_MAX), + str(self.DEFAULT_PPS_LIMIT_MIN <= rx_pps <= self.DEFAULT_PPS_LIMIT_MAX)) + ) + assert self.DEFAULT_PPS_LIMIT_MIN <= rx_pps <= self.DEFAULT_PPS_LIMIT_MAX, "Copp policer constraint " \ + "check failed, Actual PPS: {} Expected PPS range: {} - {}".format( + rx_pps, self.DEFAULT_PPS_LIMIT_MIN, self.DEFAULT_PPS_LIMIT_MAX) + # SONIC config contains policer CIR=6000 for LACP class LACPTest(PolicyTest): diff --git a/ansible/roles/test/files/ptftests/py3/dhcp_relay_stress_test.py b/ansible/roles/test/files/ptftests/py3/dhcp_relay_stress_test.py new file mode 100644 index 0000000000..38eebb9852 --- /dev/null +++ b/ansible/roles/test/files/ptftests/py3/dhcp_relay_stress_test.py @@ -0,0 +1,39 @@ +import time +import ptf.testutils as testutils +from dhcp_relay_test import DHCPTest + + +class DHCPContinuousStressTest(DHCPTest): + """ + Keep sending packets, but don't verify form ptf side. + """ + def __init__(self): + DHCPTest.__init__(self) + + def setUp(self): + DHCPTest.setUp(self) + self.send_interval = 1 / self.test_params["pps"] + self.duration = self.test_params["duration"] + self.client_ports = self.other_client_port + self.client_ports.append(self.client_port_index) + + def send_packet_with_interval(self, pkt, index): + testutils.send_packet(self, index, pkt) + time.sleep(self.send_interval) + + def runTest(self): + dhcp_discover = self.create_dhcp_discover_packet(self.dest_mac_address, self.client_udp_src_port) + dhcp_offer = self.create_dhcp_offer_packet() + dhcp_request = self.create_dhcp_request_packet(self.dest_mac_address, self.client_udp_src_port) + dhcp_ack = self.create_dhcp_ack_packet() + + start_time = time.time() + while time.time() - start_time <= self.duration: + for client_port in self.client_ports: + self.send_packet_with_interval(dhcp_discover, client_port) + for server_port in self.server_port_indices: + self.send_packet_with_interval(dhcp_offer, server_port) + for client_port in self.client_ports: + self.send_packet_with_interval(dhcp_request, client_port) + for server_port in self.server_port_indices: + self.send_packet_with_interval(dhcp_ack, server_port) diff --git a/ansible/roles/test/files/ptftests/py3/dhcp_relay_test.py b/ansible/roles/test/files/ptftests/py3/dhcp_relay_test.py index e5f6d66476..9c4f8b804e 100644 --- a/ansible/roles/test/files/ptftests/py3/dhcp_relay_test.py +++ b/ansible/roles/test/files/ptftests/py3/dhcp_relay_test.py @@ -134,9 +134,9 @@ def setUp(self): 0, self.server_port_indices[0]) self.relay_iface_ip = self.test_params['relay_iface_ip'] - self.relay_iface_mac = self.test_params['relay_iface_mac'] + self.relay_iface_mac = self.test_params.get('relay_iface_mac', '') - self.client_iface_alias = self.test_params['client_iface_alias'] + self.client_iface_alias = self.test_params.get('client_iface_alias', '') self.client_port_index = int(self.test_params['client_port_index']) self.client_mac = self.dataplane.get_mac(0, self.client_port_index) @@ -148,8 +148,6 @@ def setUp(self): # 'single' for regular single tor testing self.dual_tor = (self.test_params['testing_mode'] == 'dual') - self.testbed_mode = self.test_params['testbed_mode'] - # option82 is a byte string created by the relay agent. It contains the circuit_id and remote_id fields. # circuit_id is stored as suboption 1 of option 82. # It consists of the following: @@ -188,6 +186,7 @@ def setUp(self): self.dest_mac_address = self.test_params['dest_mac_address'] self.client_udp_src_port = self.test_params['client_udp_src_port'] + self.enable_source_port_ip_in_relay = self.test_params.get('enable_source_port_ip_in_relay', False) def tearDown(self): DataplaneBaseTest.tearDown(self) @@ -231,7 +230,12 @@ def create_dhcp_discover_relayed_packet(self): # be loopback. We could pull from minigraph and check here. ether = scapy.Ether(dst=self.BROADCAST_MAC, src=self.uplink_mac, type=0x0800) - ip = scapy.IP(src=self.DEFAULT_ROUTE_IP, + + source_ip = self.switch_loopback_ip + if self.enable_source_port_ip_in_relay: + source_ip = self.relay_iface_ip + + ip = scapy.IP(src=source_ip, dst=self.BROADCAST_IP, len=328, ttl=64) udp = scapy.UDP(sport=self.DHCP_SERVER_PORT, dport=self.DHCP_SERVER_PORT, len=308) @@ -422,7 +426,11 @@ def create_dhcp_request_relayed_packet(self): # be loopback. We could pull from minigraph and check here. ether = scapy.Ether(dst=self.BROADCAST_MAC, src=self.uplink_mac, type=0x0800) - ip = scapy.IP(src=self.DEFAULT_ROUTE_IP, + + source_ip = self.switch_loopback_ip + if self.enable_source_port_ip_in_relay: + source_ip = self.relay_iface_ip + ip = scapy.IP(src=source_ip, dst=self.BROADCAST_IP, len=336, ttl=64) udp = scapy.UDP(sport=self.DHCP_SERVER_PORT, dport=self.DHCP_SERVER_PORT, len=316) @@ -564,7 +572,6 @@ def verify_relayed_discover(self): masked_discover.set_do_not_care_scapy(scapy.IP, "ttl") masked_discover.set_do_not_care_scapy(scapy.IP, "proto") masked_discover.set_do_not_care_scapy(scapy.IP, "chksum") - masked_discover.set_do_not_care_scapy(scapy.IP, "src") masked_discover.set_do_not_care_scapy(scapy.IP, "dst") masked_discover.set_do_not_care_scapy(scapy.IP, "options") @@ -641,7 +648,6 @@ def verify_relayed_request(self): masked_request.set_do_not_care_scapy(scapy.IP, "ttl") masked_request.set_do_not_care_scapy(scapy.IP, "proto") masked_request.set_do_not_care_scapy(scapy.IP, "chksum") - masked_request.set_do_not_care_scapy(scapy.IP, "src") masked_request.set_do_not_care_scapy(scapy.IP, "dst") masked_request.set_do_not_care_scapy(scapy.IP, "options") diff --git a/ansible/roles/test/files/ptftests/py3/fdb_mac_learning_test.py b/ansible/roles/test/files/ptftests/py3/fdb_mac_learning_test.py new file mode 100644 index 0000000000..9a0d598d71 --- /dev/null +++ b/ansible/roles/test/files/ptftests/py3/fdb_mac_learning_test.py @@ -0,0 +1,33 @@ +import ptf +from ptf.base_tests import BaseTest +from ptf.testutils import send, simple_eth_packet, test_params_get + + +class FdbMacLearningTest(BaseTest): + def __init__(self): + BaseTest.__init__(self) + self.test_params = test_params_get() # noqa: F405 + + # -------------------------------------------------------------------------- + def setUp(self): + self.dataplane = ptf.dataplane_instance + self.router_mac = self.test_params['router_mac'] + self.dummy_mac_prefix = self.test_params['dummy_mac_prefix'] + self.dut_ptf_ports = self.test_params['dut_ptf_ports'] + self.mac_table = [] + + # -------------------------------------------------------------------------- + def populateFdbForInterface(self): + for dut_port, ptf_port in self.dut_ptf_ports: + mac = self.dummy_mac_prefix + ":" + "{:02X}".format(ptf_port) + pkt = simple_eth_packet(eth_dst=self.router_mac, # noqa: F405 + eth_src=mac, + eth_type=0x1234) + send(self, ptf_port, pkt) + self.mac_table.append((ptf_port, mac)) + + # -------------------------------------------------------------------------- + def runTest(self): + self.populateFdbForInterface() + + # -------------------------------------------------------------------------- diff --git a/ansible/roles/test/files/ptftests/py3/generic_hash_test.py b/ansible/roles/test/files/ptftests/py3/generic_hash_test.py new file mode 100644 index 0000000000..e73b2349b3 --- /dev/null +++ b/ansible/roles/test/files/ptftests/py3/generic_hash_test.py @@ -0,0 +1,530 @@ +""" +Description: This file contains the generic hash test for SONiC +""" + +# --------------------------------------------------------------------- +# Global imports +# --------------------------------------------------------------------- +import logging +import random +import time +import ptf +import ptf.packet as scapy +import re +import ptf.testutils as testutils +import lpm +from ipaddress import ip_address +from ptf.base_tests import BaseTest +from ptf.mask import Mask + + +class GenericHashTest(BaseTest): + # --------------------------------------------------------------------- + # Class variables + # --------------------------------------------------------------------- + DEFAULT_BALANCING_RANGE = 0.25 + BALANCING_TEST_TIMES = 625 + VXLAN_PORT = 4789 + VXLAN_VNI = 20001 + NVGRE_TNI = 20001 + L4_SRC_PORT = 1234 + L4_DST_PORT = 80 + + _required_params = [ + 'sending_ports', + 'expected_port_groups', + 'hash_field', + 'ipver', + 'src_ip_range', + 'dst_ip_range', + 'ecmp_hash', + 'lag_hash' + ] + + def __init__(self): + BaseTest.__init__(self) + self.test_params = testutils.test_params_get() + self.check_required_params() + + def check_required_params(self): + for param in self._required_params: + if param not in self.test_params: + raise Exception(f"Missing required parameter {param}") + + def setUp(self): + self.dataplane = ptf.dataplane_instance + self.router_mac = self.test_params['router_mac'] + self.ipver = self.test_params['ipver'] + self.inner_ipver = self.test_params.get('inner_ipver') + if self.inner_ipver == 'None': + self.inner_ipver = None + src_ip_range = [str(x) for x in self.test_params['src_ip_range'].split(',')] + dst_ip_range = [str(x) for x in self.test_params['dst_ip_range'].split(',')] + self.src_ip_interval = lpm.LpmDict.IpInterval(ip_address(src_ip_range[0]), ip_address(src_ip_range[1])) + self.dst_ip_interval = lpm.LpmDict.IpInterval(ip_address(dst_ip_range[0]), ip_address(dst_ip_range[1])) + if self.inner_ipver: + inner_src_ip_range = [str(x) for x in self.test_params['inner_src_ip_range'].split(',')] + inner_dst_ip_range = [str(x) for x in self.test_params['inner_dst_ip_range'].split(',')] + self.inner_src_ip_interval = lpm.LpmDict.IpInterval(ip_address(inner_src_ip_range[0]), + ip_address(inner_src_ip_range[1])) + self.inner_dst_ip_interval = lpm.LpmDict.IpInterval(ip_address(inner_dst_ip_range[0]), + ip_address(inner_dst_ip_range[1])) + self.hash_field = self.test_params['hash_field'] + self.sending_ports = self.test_params['sending_ports'] + self.expected_port_groups = self.test_params['expected_port_groups'] + self.expected_port_list = sum(self.expected_port_groups, []) + self.balancing_range = self.test_params.get('balancing_range', self.DEFAULT_BALANCING_RANGE) + self.balancing_test_times = self.test_params.get('balancing_test_times', self.BALANCING_TEST_TIMES) + self.ecmp_hash = self.test_params['ecmp_hash'] + self.lag_hash = self.test_params['lag_hash'] + self.vlan_range = self.test_params.get('vlan_range', [1032, 1060]) + self.ethertype_range = self.test_params.get('ethertype_range', [0x0800, 0x0900]) + self.is_l2_test = self.test_params.get('is_l2_test', False) + self.encap_type = self.test_params.get('encap_type') + self.vxlan_port = self.test_params.get('vxlan_port', self.VXLAN_PORT) + self.vxlan_vni = self.test_params.get('vxlan_vni', self.VXLAN_VNI) + self.nvgre_tni = self.test_params.get('nvgre_tni', self.NVGRE_TNI) + logging.info("=============Test Setup==============") + logging.info(f"balancing_range: {self.balancing_range}") + logging.info(f"balancing_test_times: {self.balancing_test_times}") + logging.info(f"hash_field: {self.hash_field}") + logging.info(f"ipver: {self.ipver}") + if self.inner_ipver: + logging.info(f"inner_ipver: {self.inner_ipver}") + logging.info(f"encap_type: {self.encap_type}") + if self.encap_type == 'vxlan': + logging.info(f"vxlan_port: {self.vxlan_port}") + logging.info(f"sending_ports: {self.sending_ports}") + logging.info(f"expected_port_groups: {self.expected_port_groups}") + logging.info(f"ecmp_hash: {self.ecmp_hash}") + logging.info(f"lag_hash: {self.lag_hash}") + logging.info(f"is_l2_test: {self.is_l2_test}") + + def get_ip_proto(self): + # ip_proto 2 is IGMP, should not be forwarded by router + # ip_proto 253, 254 is experimental + # Nvidia ASIC can't forward ip_proto 254, BRCM is OK, skip for all for simplicity + # For Nvidia platforms, when the ip_proto are 4, 6, 17, 41, the parser behavior is different with other + # protocols, skip them for simplicity + skip_protos = [2, 4, 6, 17, 41, 253, 254] + if self.ipver == 'ipv6': + # Skip ip_proto 0 for IPv6 + skip_protos.append(0) + return random.choice(list(set(range(255)) - set(skip_protos))) + + def randomize_mac(self, base_mac): + return base_mac[:-5] + '{0:02x}:{1:02x}'.format(random.randint(0, 255), random.randint(0, 255)) + + def generate_pkt(self, src_ip, dst_ip, src_port, dst_port, ip_proto, inner_src_ip, inner_dst_ip): + + def _get_pkt_ip_protocol(pkt): + if 'IPv6' in pkt.summary(): + return pkt['IPv6'].nh + elif pkt.getlayer('IP'): + return pkt['IP'].proto + else: + return None + + def _get_src_mac(): + src_base_mac = self.dataplane.get_mac(0, self.sending_ports[0]) + if self.hash_field == 'SRC_MAC': + src_mac = self.randomize_mac(src_base_mac) + else: + src_mac = src_base_mac + return src_mac + + def _get_dst_mac(): + dst_base_mac = '11:22:33:44:55:66' + if self.is_l2_test: + if self.hash_field == 'DST_MAC': + dst_mac = self.randomize_mac(dst_base_mac) + else: + dst_mac = '11:22:33:44:55:66' + else: + dst_mac = self.router_mac + return dst_mac + + def _get_vlan_id(): + if self.hash_field == 'VLAN_ID': + vlan_id = random.choice(range(self.vlan_range[0], self.vlan_range[1])) + else: + vlan_id = 0 + return vlan_id + + def _get_single_layer_packet(): + # Generate a tcp packet to cover the outer IP header fields + if self.ipver == 'ipv4': # IP version is ipv4 + pkt = testutils.simple_tcp_packet( + pktlen=100 if vlan_id == 0 else 104, + eth_dst=dst_mac, + eth_src=src_mac, + dl_vlan_enable=False if vlan_id == 0 else True, + vlan_vid=vlan_id, + vlan_pcp=0, + ip_src=src_ip, + ip_dst=dst_ip, + tcp_sport=src_port, + tcp_dport=dst_port, + ip_ttl=64) + if self.hash_field == 'IP_PROTOCOL': + pkt['IP'].proto = ip_proto + masked_expected_pkt = Mask(pkt) + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "dst") + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "src") + masked_expected_pkt.set_do_not_care_packet(scapy.IP, "chksum") + masked_expected_pkt.set_do_not_care_packet(scapy.IP, "ttl") + masked_expected_pkt.set_do_not_care_packet(scapy.TCP, "chksum") + else: # IP version is ipv6 + pkt = testutils.simple_tcpv6_packet( + pktlen=100 if vlan_id == 0 else 104, + eth_dst=dst_mac, + eth_src=src_mac, + dl_vlan_enable=False if vlan_id == 0 else True, + vlan_vid=vlan_id, + vlan_pcp=0, + ipv6_dst=dst_ip, + ipv6_src=src_ip, + tcp_sport=src_port, + tcp_dport=dst_port, + ipv6_hlim=64) + if self.hash_field == 'IP_PROTOCOL': + pkt['IPv6'].nh = ip_proto + masked_expected_pkt = Mask(pkt) + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "src") + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "dst") + masked_expected_pkt.set_do_not_care_packet(scapy.IPv6, "hlim") + masked_expected_pkt.set_do_not_care_packet(scapy.TCP, "chksum") + if self.hash_field == 'ETHERTYPE': + pkt['Ether'].type = random.choice(range(self.ethertype_range[0], self.ethertype_range[1])) + if not self.is_l2_test: + pkt_summary = f"{self.ipver} packet with src_mac:{src_mac}, dst_mac:{dst_mac}, src_ip:{src_ip}, " \ + f"dst_ip:{dst_ip}, src_port:{src_port}, dst_port: {dst_port}, " \ + f"ip_protocol:{_get_pkt_ip_protocol(pkt)}" + else: + pkt_summary = f"Ethernet packet with src_mac:{src_mac}, dst_mac:{dst_mac}, " \ + f"ether_type:{hex(pkt['Ether'].type)}, vlan_id:{vlan_id if vlan_id != 0 else 'N/A'}" + return pkt, masked_expected_pkt, pkt_summary + + def _get_ipinip_packet(): + # Generate an ipinip packet + if self.ipver == 'ipv4': # Outer IP version is ipv4 + pkt = testutils.simple_ipv4ip_packet( + eth_dst=self.router_mac, + eth_src=src_mac, + ip_src=src_ip, + ip_dst=dst_ip, + ip_ttl=64, + inner_frame=inner_pkt['IP'] if self.inner_ipver == 'ipv4' else inner_pkt['IPv6']) + masked_expected_pkt = Mask(pkt) + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "dst") + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "src") + masked_expected_pkt.set_do_not_care_packet(scapy.IP, "chksum") + masked_expected_pkt.set_do_not_care_packet(scapy.IP, "ttl") + else: # Outer IP version is ipv6 + pkt = testutils.simple_ipv6ip_packet( + eth_dst=self.router_mac, + eth_src=src_mac, + ipv6_src=src_ip, + ipv6_dst=dst_ip, + ipv6_hlim=64, + inner_frame=inner_pkt['IP'] if self.inner_ipver == 'ipv4' else inner_pkt['IPv6']) + masked_expected_pkt = Mask(pkt) + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "dst") + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "src") + masked_expected_pkt.set_do_not_care_packet(scapy.IPv6, "hlim") + pkt_summary = f"{self.ipver} ipinip packet with src_ip:{src_ip}, dst_ip:{dst_ip}, " \ + f"ip_protocol:{_get_pkt_ip_protocol(pkt)}, inner_ipver:{self.inner_ipver}, " \ + f"inner_src_ip:{inner_src_ip}, inner_dst_ip:{inner_dst_ip}, inner_src_port:{src_port}," \ + f" inner_dst_port:{dst_port}, inner_ip_protocol:{_get_pkt_ip_protocol(inner_pkt)}" + return pkt, masked_expected_pkt, pkt_summary + + def _get_vxlan_packet(): + # Generate an vxlan packet to cover the inner IP header fields + if self.ipver == 'ipv4': # Outer IP version is ipv4 + pkt = testutils.simple_vxlan_packet( + eth_dst=self.router_mac, + eth_src=src_mac, + ip_id=0, + ip_src=src_ip, + ip_dst=dst_ip, + ip_ttl=64, + udp_sport=self.L4_SRC_PORT, + udp_dport=self.vxlan_port, + vxlan_vni=self.vxlan_vni, + with_udp_chksum=False, + inner_frame=inner_pkt) + masked_expected_pkt = Mask(pkt) + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "dst") + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "src") + masked_expected_pkt.set_do_not_care_packet(scapy.IP, "chksum") + masked_expected_pkt.set_do_not_care_packet(scapy.IP, "ttl") + else: # Outer IP version is ipv6 + pkt = testutils.simple_vxlanv6_packet( + eth_dst=self.router_mac, + eth_src=src_mac, + ipv6_src=src_ip, + ipv6_dst=dst_ip, + ipv6_hlim=64, + udp_sport=self.L4_SRC_PORT, + udp_dport=self.vxlan_port, + vxlan_vni=self.vxlan_vni, + with_udp_chksum=False, + inner_frame=inner_pkt) + masked_expected_pkt = Mask(pkt) + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "dst") + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "src") + masked_expected_pkt.set_do_not_care_packet(scapy.IPv6, "hlim") + pkt_summary = f"{self.ipver} vxlan packet with src_ip:{src_ip}, dst_ip:{dst_ip}, " \ + f"src_port:{self.L4_SRC_PORT}, dst_port: {self.vxlan_port}, ip_protocol:{_get_pkt_ip_protocol(pkt)}, " \ + f"inner_ipver:{self.inner_ipver}, inner_src_ip:{inner_src_ip}, inner_dst_ip:{inner_dst_ip}, " \ + f"inner_src_port:{src_port}, inner_dst_port:{dst_port}, " \ + f"inner_ip_protocol:{_get_pkt_ip_protocol(inner_pkt)}" + return pkt, masked_expected_pkt, pkt_summary + + def _get_nvgre_packet(): + # Generate an nvgre packet to cover the inner IP header fields + if self.ipver == 'ipv4': # Outer IP version is ipv4 + pkt = testutils.simple_nvgre_packet( + eth_dst=self.router_mac, + eth_src=src_mac, + ip_id=0, + ip_src=src_ip, + ip_dst=dst_ip, + ip_ttl=64, + nvgre_tni=self.nvgre_tni, + nvgre_flowid=0, + inner_frame=inner_pkt) + masked_expected_pkt = Mask(pkt) + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "dst") + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "src") + masked_expected_pkt.set_do_not_care_packet(scapy.IP, "chksum") + masked_expected_pkt.set_do_not_care_packet(scapy.IP, "ttl") + else: # Outer IP version is ipv6 + pkt = GenericHashTest.simple_nvgrev6_packet( + eth_dst=self.router_mac, + eth_src=src_mac, + ipv6_src=src_ip, + ipv6_dst=dst_ip, + ipv6_hlim=64, + nvgre_tni=self.nvgre_tni, + nvgre_flowid=0, + inner_frame=inner_pkt) + masked_expected_pkt = Mask(pkt) + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "dst") + masked_expected_pkt.set_do_not_care_packet(scapy.Ether, "src") + masked_expected_pkt.set_do_not_care_packet(scapy.IPv6, "hlim") + pkt_summary = f"{self.ipver} nvgre packet with src_ip:{src_ip}, dst_ip:{dst_ip}, " \ + f"ip_protocol:{_get_pkt_ip_protocol(pkt)}, inner_ipver:{self.inner_ipver}, " \ + f"inner_src_ip:{inner_src_ip}, inner_dst_ip:{inner_dst_ip}, inner_src_port:{src_port}, " \ + f"inner_dst_port:{dst_port}, inner_ip_protocol:{_get_pkt_ip_protocol(inner_pkt)}" + return pkt, masked_expected_pkt, pkt_summary + + src_mac = _get_src_mac() + dst_mac = _get_dst_mac() + vlan_id = _get_vlan_id() + + if 'INNER' not in self.hash_field: + packet, masked_expected_packet, packet_summary = _get_single_layer_packet() + else: + # For the inner fields, need an encapsulated packet + inner_pkt = self.generate_inner_pkt(inner_src_ip, inner_dst_ip, src_port, dst_port, ip_proto) + if self.encap_type == 'ipinip': + packet, masked_expected_packet, packet_summary = _get_ipinip_packet() + elif self.encap_type == 'vxlan': + packet, masked_expected_packet, packet_summary = _get_vxlan_packet() + elif self.encap_type == 'nvgre': + packet, masked_expected_packet, packet_summary = _get_nvgre_packet() + return packet, masked_expected_packet, packet_summary + + def generate_inner_pkt(self, src_ip, dst_ip, src_port, dst_port, ip_proto): + src_mac = '00:12:ab:34:cd:01' + dst_mac = '01:12:ab:34:cd:00' + if self.hash_field == 'INNER_SRC_MAC': + src_mac = self.randomize_mac(src_mac) + if self.hash_field == 'INNER_DST_MAC': + dst_mac = self.randomize_mac(dst_mac) + if self.hash_field == 'INNER_ETHERTYPE': + eth_type = random.choice(range(self.ethertype_range[0], self.ethertype_range[1])) + inner_pkt = testutils.simple_eth_packet( + eth_dst=dst_mac, + eth_src=src_mac, + eth_type=eth_type + ) + elif self.inner_ipver == 'ipv4': # Inner IP version is ipv4 + inner_pkt = testutils.simple_tcp_packet( + eth_dst=dst_mac, + eth_src=src_mac, + ip_dst=dst_ip, + ip_src=src_ip, + tcp_sport=src_port, + tcp_dport=dst_port, + ip_ttl=64 + ) + inner_pkt["IP"].proto = ip_proto + else: # Inner IP version is ipv6 + inner_pkt = testutils.simple_tcpv6_packet( + eth_dst=dst_mac, + eth_src=src_mac, + ipv6_dst=dst_ip, + ipv6_src=src_ip, + tcp_sport=src_port, + tcp_dport=dst_port, + ipv6_hlim=64 + ) + inner_pkt["IPv6"].nh = ip_proto + return inner_pkt + + def check_ip_route(self, pkt, masked_expected_pkt, sending_port): + """ + send the packet and check it is received by one of the expected ports + """ + testutils.send_packet(self, sending_port, pkt) + port_index, received = testutils.verify_packet_any_port( + self, masked_expected_pkt, self.expected_port_list, timeout=0.1) + # The port_index is the index of expected_port_list, need to convert it to the ptf port index + return self.expected_port_list[port_index], received + + def check_within_expected_range(self, actual, expected): + """ + Check if the actual number is within the accepted range of the expected number + """ + percentage = (actual - expected) / float(expected) + return percentage, abs(percentage) <= self.balancing_range + + @staticmethod + def simple_nvgrev6_packet( + pktlen=300, + eth_dst='00:01:02:03:04:05', + eth_src='00:06:07:08:09:0a', + dl_vlan_enable=False, + vlan_vid=0, + vlan_pcp=0, + dl_vlan_cfi=0, + ipv6_src='1::2', + ipv6_dst='3::4', + ipv6_fl=0, + ipv6_tc=0, + ipv6_hlim=64, + nvgre_tni=None, + nvgre_flowid=0, + inner_frame=None + ): + """ + Helper function to construct an IPv6 NVGRE packet + """ + if testutils.MINSIZE > pktlen: + pktlen = testutils.MINSIZE + + nvgre_hdr = scapy.NVGRE(vsid=nvgre_tni, flowid=nvgre_flowid) + + if dl_vlan_enable: + pkt = scapy.Ether(dst=eth_dst, src=eth_src) / \ + scapy.Dot1Q(prio=vlan_pcp, id=dl_vlan_cfi, vlan=vlan_vid) / \ + scapy.IPv6(src=ipv6_src, dst=ipv6_dst, fl=ipv6_fl, tc=ipv6_tc, hlim=ipv6_hlim, nh=47) / \ + nvgre_hdr + else: + pkt = scapy.Ether(dst=eth_dst, src=eth_src) / \ + scapy.IPv6(src=ipv6_src, dst=ipv6_dst, fl=ipv6_fl, tc=ipv6_tc, hlim=ipv6_hlim, nh=47) / \ + nvgre_hdr + + if inner_frame: + pkt = pkt / inner_frame + else: + pkt = pkt / scapy.IP() + pkt = pkt / ("D" * (pktlen - len(pkt))) + + return pkt + + def check_balancing(self, hit_count_map): + """ + Check if the traffic is balanced across the ECMP groups and the LAG members + """ + + def _calculate_balance(hit_cnt_per_port): + result = True + for port_index in hit_count_map.keys(): + (p, r) = self.check_within_expected_range(hit_count_map[port_index], hit_cnt_per_port) + result &= r + return result + + def _check_ecmp_and_lag_hash_balancing(): + logging.info('Checking there is ecmp and lag hash') + expected_hit_cnt_per_port = self.balancing_test_times + assert _calculate_balance(expected_hit_cnt_per_port), "The balancing result is beyond the range." + + def _check_only_ecmp_hash_balancing(): + logging.info('Checking there is only ecmp hash') + if len(self.expected_port_groups[0]) > 1: + logging.info('There are multi-member portchannels, check no hash over the members') + for port_group in self.expected_port_groups: + hit_port_number = len(set(port_group).intersection(set(hit_count_map.keys()))) + if hit_port_number > 1: + logging.info('Check only one port in a portchannel received traffic') + assert False, 'The traffic is balanced over portchannel members.' + if hit_port_number == 0: + logging.info('Check the traffic is balanced over all the portchannels') + assert False, 'Traffic is not balanced over all nexthops.' + # Check the balance + expected_hit_cnt_per_port = expected_total_hit_cnt / len(self.expected_port_groups) + assert _calculate_balance(expected_hit_cnt_per_port), "The balancing result is beyond the range." + + def _check_only_lag_hash_balancing(): + logging.info('Checking there is only lag hash') + hit_ports = sorted(hit_count_map.keys()) + assert hit_ports in self.expected_port_groups, "Traffic is not received by all lag members in 1 nexthop." + # Check the traffic is balanced over the members + expected_hit_cnt_per_port = expected_total_hit_cnt / len(self.expected_port_groups[0]) + assert _calculate_balance(expected_hit_cnt_per_port), "The balancing result is beyond the range." + + expected_total_hit_cnt = self.balancing_test_times * len(self.expected_port_list) + # If check ecmp hash and lag hash, traffic should be balanced through all expected ports + if self.ecmp_hash and self.lag_hash: + _check_ecmp_and_lag_hash_balancing() + # If check ecmp hash but not lag hash, traffic should be balanced through + # all portchannels but not the members in a same portchannel + elif self.ecmp_hash and not self.lag_hash: + _check_only_ecmp_hash_balancing() + # If check lag hash but not ecmp hash, traffic should be received + # by only one portchannel and balanced over the members + elif not self.ecmp_hash and self.lag_hash: + _check_only_lag_hash_balancing() + + def runTest(self): + logging.info("=============Test Start==============") + hit_count_map = {} + for _ in range(0, self.balancing_test_times * len(self.expected_port_list)): + src_ip = self.src_ip_interval.get_random_ip() if self.hash_field == 'SRC_IP' \ + else self.src_ip_interval.get_first_ip() + dst_ip = self.dst_ip_interval.get_random_ip() if self.hash_field == 'DST_IP' \ + else self.dst_ip_interval.get_first_ip() + inner_src_ip = '' + inner_dst_ip = '' + if self.inner_ipver: + inner_src_ip = self.inner_src_ip_interval.get_random_ip() if self.hash_field == 'INNER_SRC_IP' \ + else self.inner_src_ip_interval.get_first_ip() + inner_dst_ip = self.inner_dst_ip_interval.get_random_ip() if self.hash_field == 'INNER_DST_IP' \ + else self.inner_dst_ip_interval.get_first_ip() + src_port = random.randint(0, 65535) if self.hash_field in ['L4_SRC_PORT', 'INNER_L4_SRC_PORT'] else \ + self.L4_SRC_PORT + dst_port = random.randint(0, 65535) if self.hash_field in ['L4_DST_PORT', 'INNER_L4_DST_PORT'] else \ + self.L4_DST_PORT + ip_proto = self.get_ip_proto() if self.hash_field in ['IP_PROTOCOL', 'INNER_IP_PROTOCOL'] else 6 + + pkt, masked_expected_pkt, pkt_summary = self.generate_pkt( + src_ip, dst_ip, src_port, dst_port, ip_proto, inner_src_ip, inner_dst_ip) + sending_port = self.sending_ports[0] if self.hash_field != 'IN_PORT' \ + else random.choice(self.sending_ports) + logging.info('Sending ' + pkt_summary + ' from ptf port ' + str(sending_port)) + (matched_port, received) = self.check_ip_route(pkt, masked_expected_pkt, sending_port) + + # Check there is no packet loss + assert received is not None, 'Packet is not received at any expected port.' + + logging.info("Received packet at index {}: {}".format( + str(matched_port), re.sub(r"(?<=\w)(?=(?:\w\w)+$)", ' ', received.hex()))) + time.sleep(0.02) + + hit_count_map[matched_port] = hit_count_map.get(matched_port, 0) + 1 + logging.info(f"hash_field={self.hash_field}, hit count map: {hit_count_map}") + # Check if the traffic is properly balanced + self.check_balancing(hit_count_map) diff --git a/ansible/roles/test/files/ptftests/py3/pfc_wd_background_traffic.py b/ansible/roles/test/files/ptftests/py3/pfc_wd_background_traffic.py index b8487f5cc6..7f5e3afd47 100644 --- a/ansible/roles/test/files/ptftests/py3/pfc_wd_background_traffic.py +++ b/ansible/roles/test/files/ptftests/py3/pfc_wd_background_traffic.py @@ -1,8 +1,9 @@ import ptf import logging +import random from ptf.base_tests import BaseTest import time -from ptf.testutils import test_params_get, simple_ip_packet, send_packet +from ptf.testutils import test_params_get, simple_udp_packet, send_packet class PfcWdBackgroundTrafficTest(BaseTest): @@ -39,7 +40,7 @@ def runTest(self): for queue in self.queues: print(f"traffic from {src_port} to {dst_port}: {queue} ") logging.info(f"traffic from {src_port} to {dst_port}: {queue} ") - pkt = simple_ip_packet( + pkt = simple_udp_packet( eth_src=src_mac, eth_dst=self.router_mac, ip_src=self.src_ips[i], @@ -52,7 +53,7 @@ def runTest(self): if self.bidirection: print(f"traffic from {dst_port} to {src_port}: {queue} ") logging.info(f"traffic from {dst_port} to {src_port}: {queue} ") - pkt = simple_ip_packet( + pkt = simple_udp_packet( eth_src=dst_mac, eth_dst=self.router_mac, ip_src=self.dst_ips[i], @@ -67,10 +68,21 @@ def runTest(self): logging.info("Start to send the background traffic") print("Start to send the background traffic") timeout = 500 + pkt_count_in_batch = 100 while True: for port, pkts in pkts_dict.items(): for pkt in pkts: - send_packet(self, port, pkt, self.pkt_count) + sent_count = 0 + """ + Randomize the sport/dport to add entropy to the packets so that + the traffic can be hashed to different egress ports. + This is to ensure all the LAG members in the LAG take traffic. + """ + while sent_count < self.pkt_count: + pkt['UDP'].sport = random.randint(1, 65535) + pkt['UDP'].dport = random.randint(1, 65535) + send_packet(self, port, pkt, pkt_count_in_batch) + sent_count += pkt_count_in_batch now = time.time() if now - start > timeout: diff --git a/ansible/roles/test/files/ptftests/py3/vxlan_traffic.py b/ansible/roles/test/files/ptftests/py3/vxlan_traffic.py index dba0f5e186..496e0bb871 100644 --- a/ansible/roles/test/files/ptftests/py3/vxlan_traffic.py +++ b/ansible/roles/test/files/ptftests/py3/vxlan_traffic.py @@ -71,7 +71,7 @@ Logger = logging.getLogger(__name__) # Some constants used in this code -MIN_PACKET_COUNT = 4 +MIN_PACKET_COUNT = 10 MINIMUM_PACKETS_FOR_ECMP_VALIDATION = 300 TEST_ECN = True diff --git a/ansible/roles/test/files/ptftests/sonic.py b/ansible/roles/test/files/ptftests/sonic.py index ee38c56985..f8400be332 100644 --- a/ansible/roles/test/files/ptftests/sonic.py +++ b/ansible/roles/test/files/ptftests/sonic.py @@ -166,14 +166,23 @@ def run(self): log_data = {} self.log('Collecting logs') - log_lines = self.do_cmd("sudo cat " - "/var/log/syslog{,.1} " - "/var/log/teamd.log{,.1} " - "/var/log/frr/bgpd.log " - "/var/log/frr/zebra.log").split('\n') - syslog_regex_r = r'^(\S+\s+\d+\s+\d+:\d+:\d+)\.\d+ \S+ [A-Z]+ ([a-z\-]+#[/a-zA-Z0-9_]+)' \ + log_files = [ + "/var/log/syslog{.1,}", + "/var/log/teamd.log{.1,}", + "/var/log/frr/bgpd.log", + "/var/log/frr/zebra.log" + ] + + logs_to_output = {log_file: self.do_cmd("sudo cat {}".format(log_file)).split('\n') + for log_file in log_files} + log_lines = [line for output in logs_to_output.values() for line in output] + syslog_regex_r = r'^(?:\d{4}\s+)?(\S+\s+\d+\s+\d+:\d+:\d+)\.\d+ \S+ [A-Z]+ ([a-z\-]+#[/a-zA-Z0-9_]+)' \ r'(?:\s+\d+-\d+-\d+\s+\d+:\d+:\d+,\d+\s+[A-Z]+\s+\w+)?(?:\[\d+\])?: (.+)$' - parsed_logs = self.extract_from_logs(syslog_regex_r, log_lines, min_timestamp=start_time) + parsed_logs = {} + total_lag_flaps = 0 + for _, output in logs_to_output.items(): + parsed_logs.update(self.extract_from_logs(syslog_regex_r, output, min_timestamp=start_time)) + total_lag_flaps += self.check_lag_flaps("PortChannel1", output, start_time)[1] self.log('Log output "{}"'.format('\n'.join(["{} {} {}".format(k[0], j, k[1]) for j in parsed_logs for k in parsed_logs[j]]))) log_data = self.parse_logs(parsed_logs) @@ -204,7 +213,7 @@ def run(self): cli_data['lacp'] = (0, 0) cli_data['bgp_v4'] = self.check_series_status(data, "bgp_route_v4", "BGP v4 routes") cli_data['bgp_v6'] = self.check_series_status(data, "bgp_route_v6", "BGP v6 routes") - cli_data['po'] = self.check_lag_flaps("PortChannel1", log_lines, start_time) + cli_data['po'] = (0, total_lag_flaps) if 'route_timeout' in log_data: route_timeout = log_data['route_timeout'] @@ -216,31 +225,33 @@ def run(self): msg = 'BGP route GR timeout: neighbor %s (ASN %s' % (nei, asn) self.fails.add(msg) - if cli_data['po'][1] > 0: + if cli_data['po'][1] > 0 and self.reboot_type == 'warm-reboot': self.fails.add('Port channel flap occurred!') + if cli_data['po'][1] > 1 and self.reboot_type == 'fast-reboot': + self.fails.add('More than one port channel flap occurred!') + self.log('Finishing run()') return self.fails, self.info, cli_data, log_data, { "lacp_all": list(set(self.lacp_pdu_timings)) } def extract_from_logs(self, regexp, data, min_timestamp=None): - raw_data = [] result = defaultdict(list) re_compiled = re.compile(regexp) - for line in data: + current_year = datetime.datetime.now().year + for line in reversed(data): m = re_compiled.match(line) if not m: continue - log_time = datetime.datetime.strptime(str(datetime.datetime.now().year) + " " + m.group(1), "%Y %b %d %X") + log_time = datetime.datetime.strptime(str(current_year) + " " + m.group(1), "%Y %b %d %X") # Python 3 version (Python 2 doesn't have timestamp(): - # raw_data.append((log_time.timestamp(), m.group(2), m.group(3))) - raw_data.append((time.mktime(log_time.timetuple()), m.group(2), m.group(3))) - - if len(raw_data) > 0: - for when, what, status in raw_data: - if min_timestamp and when >= min_timestamp: - result[what].append((when, status)) + # when, what, status = log_time.timestamp(), m.group(2), m.group(3) + when, what, status = time.mktime(log_time.timetuple()), m.group(2), m.group(3) + if min_timestamp and when >= min_timestamp: + result[what].insert(0, (when, status)) + else: + break return result diff --git a/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py b/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py index 7eab6ef987..8d8aa1caa7 100644 --- a/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py +++ b/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py @@ -404,8 +404,15 @@ def line_is_expected(self, str, expect_messages_regex): ''' ret_code = False - if (expect_messages_regex is not None) and (expect_messages_regex.findall(str)): - ret_code = True + if self.run_id.startswith("test_advanced_reboot_test_"): + # Use the stricter (and better-performing) match instead of findall, but only when analyzing + # logs for advanced reboot test cases. This is so that other test cases are not affected in + # case their regexes don't start with .* + if (expect_messages_regex is not None) and (expect_messages_regex.match(str)): + ret_code = True + else: + if (expect_messages_regex is not None) and (expect_messages_regex.findall(str)): + ret_code = True return ret_code diff --git a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt index f4687f494c..d6fd47b1f9 100644 --- a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt +++ b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_common_ignore.txt @@ -32,6 +32,13 @@ r, ".* sonic systemd.* kdump-tools.service - Kernel crash dump capture service.* r, ".* ERR swss#orchagent: :- getPort: Failed to get cached bridge port ID.*" r, ".* ERR syncd#syncd: .* SAI_API_PORT:brcm_sai_get_port_attribute:\d+ Error -2 processing port attribute ID: 17.*" +# Errors for config reload on broadcom platform on 202311 +r, ".* ERR swss#orchagent: :- queryHashNativeHashFieldListEnumCapabilities: Failed to get attribute.*" +r, ".* ERR swss#orchagent: :- queryHashNativeHashFieldListAttrCapabilities: Failed to get attribute.*" +r, ".* ERR swss#orchagent: :- querySwitchEcmpHashAlgorithmEnumCapabilities: Failed to get attribute.*" +r, ".* ERR swss#orchagent: :- querySwitchLagHashAlgorithmEnumCapabilities: Failed to get attribute.*" +r, ".* ERR syncd\d*#syncd.*SAI_API_PORT:_brcm_sai_read_fec_stat_err_counters.* failed with error Feature unavailable.*" + # White list below messages found on KVM for now. Need to address them later. r, ".* ERR macsec#wpa_supplicant.*l2_packet_send.*Network is down.*" r, ".* ERR macsec#wpa_supplicant.*could not process SIGINT or SIGTERM in two seconds.*" @@ -61,7 +68,7 @@ r, ".* ERR bgp#bgpcfgd: .*BGPVac.*attribute is supported.*" # https://msazure.visualstudio.com/One/_workitems/edit/14233938 r, ".* ERR swss\d*#fdbsyncd: :- readData: netlink reports an error=-25 on reading a netlink socket.*" -r, ".* ERR swss\d*#fdbsyncd: :- readData: netlink reports an error=-33 on reading a netlink socket.*" +r, ".* ERR swss\d*#.*syncd: :- readData: netlink reports an error=-33 on reading a netlink socket.*" # https://dev.azure.com/msazure/One/_workitems/edit/14213168 r, ".* ERR /hostcfgd: sonic-kdump-config --disable - failed.*" @@ -262,6 +269,7 @@ r, ".* ERR syncd\d*#syncd:.*SAI_API_BUFFER: get_buffer_pool_stats unknown counte r, ".* ERR syncd\d*#syncd:.*SAI_API_BUFFER: get_ingress_priority_group_stats unknown counter 5.*" r, ".* ERR syncd\d*#syncd:.*SAI_API_BUFFER: get_ingress_priority_group_stats unknown counter 7.*" r, ".* ERR syncd.*#syncd:.*SAI_API_HOSTIF: src/sai_trap.cpp:.*: Invalid trap event code .*" +r, ".* ERR syncd.*#syncd:.*SDK_LOG|-E-HLD-0.*" r, ".* ERR syncd\d*#syncd:.*SAI_API_LAG: resolve_feat_over_member_ports: found port index .*" r, ".* ERR syncd\d*#syncd:.*SAI_API_LAG: resolve_feat_over_member_ports: port index .* now selected.*" r, ".* ERR syncd\d*#syncd:.*SAI_API_PORT: Invalid port counter .*index.*" @@ -274,3 +282,7 @@ r, ".* INFO .*[duty_cycle_map]: illegal pwm value .*" r, ".* INFO .*command '/usr/sbin/smartctl' failed: [116] Stale file handle.*" r, ".* INFO healthd.*Key 'TEMPERATURE_INFO|ASIC' field 'high_threshold' unavailable in database 'STATE_DB'.*" r, ".* INFO healthd.*Key 'TEMPERATURE_INFO|ASIC' field 'temperature' unavailable in database 'STATE_DB'.*" + +# Ignore rsyslog librelp error if rsyslogd on host or container is down or going down +r, ".* ERR .*#rsyslogd: librelp error 10008 forwarding to server .* - suspending.*" +r, ".* ERR rsyslogd: imrelp.*error 'error when receiving data, session broken', object .* - input may not work as intended.*" diff --git a/ansible/roles/test/templates/decap_conf.j2 b/ansible/roles/test/templates/decap_conf.j2 index 133a4c2cf4..c5536d70b4 100644 --- a/ansible/roles/test/templates/decap_conf.j2 +++ b/ansible/roles/test/templates/decap_conf.j2 @@ -1,9 +1,14 @@ [ {% if outer_ipv4 %} + { + "TUNNEL_DECAP_TERM_TABLE:TEST_IPINIP_V4_TUNNEL:{{ lo_ip }}" : { + "term_type":"P2MP" + }, + "OP": "{{ op }}" + }, { "TUNNEL_DECAP_TABLE:TEST_IPINIP_V4_TUNNEL" : { "tunnel_type":"IPINIP", - "dst_ip":"{{ lo_ip }}", "dscp_mode":"{{ dscp_mode }}", "ecn_mode":"{{ ecn_mode }}", "ttl_mode":"{{ ttl_mode }}" @@ -14,10 +19,15 @@ {% if outer_ipv6 and outer_ipv4 %} , {% endif %} {% if outer_ipv6 %} + { + "TUNNEL_DECAP_TERM_TABLE:TEST_IPINIP_V6_TUNNEL:{{ lo_ipv6 }}" : { + "term_type":"P2MP" + }, + "OP": "{{ op }}" + }, { "TUNNEL_DECAP_TABLE:TEST_IPINIP_V6_TUNNEL" : { "tunnel_type":"IPINIP", - "dst_ip":"{{ lo_ipv6 }}", "dscp_mode":"{{ dscp_mode }}", "ecn_mode":"{{ ecn_mode }}", "ttl_mode":"{{ ttl_mode }}" diff --git a/ansible/roles/test/vars/testcases.yml b/ansible/roles/test/vars/testcases.yml index 9eee5974dd..675ff042a4 100644 --- a/ansible/roles/test/vars/testcases.yml +++ b/ansible/roles/test/vars/testcases.yml @@ -49,7 +49,7 @@ testcases: continuous_reboot: filename: continuous_reboot.yml vtestbed_compatible: no - topologies: [t0, t0-52, t0-56, t0-56-po2vlan, t0-56-o8v48, t0-64, t0-64-32, t0-116, t0-120, t1, t1-lag, t1-64-lag, t1-64-lag-clet, t1-56-lag, t1-28-lag] + topologies: [t0, t0-28, t0-52, t0-56, t0-56-po2vlan, t0-56-o8v48, t0-64, t0-64-32, t0-116, t0-120, t1, t1-lag, t1-64-lag, t1-64-lag-clet, t1-56-lag, t1-28-lag] copp: filename: copp.yml @@ -259,7 +259,7 @@ testcases: reboot: filename: reboot.yml - topologies: [dualtor, t0, t0-52, t0-56, t0-56-po2vlan, t0-64, t0-64-32, t0-116, t0-120, t1, t1-lag, t1-28-lag, t1-64-lag, t1-64-lag-clet, t1-56-lag, ptf32, ptf64] + topologies: [dualtor, dualtor-64-breakout, dualtor-aa-64-breakout, t0, t0-28, t0-52, t0-56, t0-56-po2vlan, t0-64, t0-64-32, t0-116, t0-120, t1, t1-lag, t1-28-lag, t1-64-lag, t1-64-lag-clet, t1-56-lag, ptf32, ptf64] repeat_harness: filename: repeat_harness.yml diff --git a/ansible/roles/vm_set/tasks/add_ceos_list.yml b/ansible/roles/vm_set/tasks/add_ceos_list.yml index 836c1b7ee3..c44c163eb8 100644 --- a/ansible/roles/vm_set/tasks/add_ceos_list.yml +++ b/ansible/roles/vm_set/tasks/add_ceos_list.yml @@ -1,58 +1,109 @@ -- name: Create VMs network - become: yes - vm_topology: - cmd: 'create' - vm_names: "{{ VM_targets }}" - fp_mtu: "{{ fp_mtu_size }}" - max_fp_num: "{{ max_fp_num }}" - topo: "{{ topology }}" - -- name: Check if cEOS image exists or not +- name: Check if cEOS docker image exists or not docker_image_info: name: - - "{{ ceos_image_orig }}" - "{{ ceos_image }}" become: yes - register: ceos_stat + register: ceos_docker_image_stat -- name: Fail if cEOS image does not exist and downloading is not enabled - fail: msg="Please ensure that cEOS docker image {{ ceos_image_orig }} is imported locally in test server" - when: (ceos_stat.images | length == 0) and skip_ceos_image_downloading +- name: Prepare ceos_image if it does not exist + block: -- include_vars: vars/azure_storage.yml + - name: Check if ceos_image_orig exists or not + docker_image_info: + name: + - "{{ ceos_image_orig }}" + become: yes + register: ceos_image_orig_stat -- name: Download cEOS image - become: yes - get_url: - url: "{{ vm_images_url }}/{{ ceos_image_filename }}?{{ ceosimage_saskey }}" - dest: "{{ root_path }}/images/{{ ceos_image_filename }}" - timeout: 1800 - environment: "{{ proxy_env | default({}) }}" - when: (ceos_stat.images | length == 0) and not skip_ceos_image_downloading - -- name: Import cEOS image - become: yes - shell: "docker import {{ root_path }}/images/{{ ceos_image_filename }} {{ ceos_image_orig }}" - when: (ceos_stat.images | length == 0) and not skip_ceos_image_downloading + - name: Prepare ceos_image_orig if it does not exist + block: + - name: Check if local ceos image file exists or not + stat: + path: "{{ root_path }}/images/{{ ceos_image_filename }}" + register: ceos_image_file_stat -- name: create directory for build ceos image - become: yes - file: - path: "/tmp/ceosimage" - state: directory + - name: Download cEOS image file if no local ceos image file exists + block: + - name: Fail if skip_ceos_image_downloading is true + fail: + msg: [ + "Failed, no ceos docker image, no ceos image file and skip_ceos_image_downloading is true", + "Please manually put cEOS image to {{ root_path }}/images/{{ ceos_image_filename }}" + ] + when: skip_ceos_image_downloading == true -- name: copy the ceos image template - become: yes - template: src=ceos_dockerfile.j2 dest=/tmp/ceosimage/Dockerfile mode=0644 + - name: Init ceos_image_urls when ceos_image_url value type is string + set_fact: + ceos_image_urls: + - "{{ ceos_image_url }}" + when: ceos_image_url | type_debug == 'string' + + - name: Init ceos_image_urls when ceos_image_url value type is list + set_fact: + ceos_image_urls: "{{ ceos_image_url }}" + when: ceos_image_url | type_debug == 'list' + + - name: Init working_image_urls list + set_fact: + working_image_urls: [] + + - name: Loop ceos_image_urls to find out working URLs + include_tasks: probe_image_url.yml + loop: "{{ ceos_image_urls }}" -- name: Build the ceos image with increase inotify limit + - name: Fail if no working ceos image download url is found + fail: + msg: [ + "Failed, no working ceos image download URL is found. There are 2 options to fix it:", + " 1. Fix ceos_image_url defined in ansible/group_vars/vm_host/ceos.yml", + " 2. Manually put cEOS image to {{ root_path }}/images/{{ ceos_image_filename }}", + ] + when: working_image_urls | length == 0 + + - name: Download cEOS image file from working ceos_image_urls using the first working URL + get_url: + url: "{{ working_image_urls[0] }}" + dest: "{{ root_path }}/images/{{ ceos_image_filename }}" + environment: "{{ proxy_env | default({}) }}" + register: ceos_image_download_result + + when: ceos_image_file_stat.stat.exists == false + + - name: Import ceos_image_orig docker image + become: yes + shell: "docker import {{ root_path }}/images/{{ ceos_image_filename }} {{ ceos_image_orig }}" + + when: ceos_image_orig_stat.images | length == 0 + + - name: Create directory for building ceos docker image + become: yes + file: + path: "/tmp/ceosimage" + state: directory + + - name: Copy the ceos image template + become: yes + template: src=ceos_dockerfile.j2 dest=/tmp/ceosimage/Dockerfile mode=0644 + + - name: Build the ceos image with increasing inotify limit + become: yes + docker_image: + name: "{{ ceos_image }}" + build: + path: "/tmp/ceosimage" + pull: no + source: build + + when: ceos_docker_image_stat.images | length == 0 + +- name: Create VMs network become: yes - docker_image: - name: "{{ ceos_image }}" - build: - path: "/tmp/ceosimage" - pull: no - source: build + vm_topology: + cmd: 'create' + vm_names: "{{ VM_targets }}" + fp_mtu: "{{ fp_mtu_size }}" + max_fp_num: "{{ max_fp_num }}" + topo: "{{ topology }}" - name: Create net base containers become: yes diff --git a/ansible/roles/vm_set/tasks/add_topo.yml b/ansible/roles/vm_set/tasks/add_topo.yml index 96c97a9796..81563910f1 100644 --- a/ansible/roles/vm_set/tasks/add_topo.yml +++ b/ansible/roles/vm_set/tasks/add_topo.yml @@ -265,6 +265,9 @@ nic_simulator_action: start when: topology.host_interfaces_active_active is defined and topology.host_interfaces_active_active|length > 0 + - name: Start tacacs+ daily daemon + include_tasks: start_tacacs_daily_daemon.yml + when: container_type == "PTF" - name: Save PTF image diff --git a/ansible/roles/vm_set/tasks/announce_routes.yml b/ansible/roles/vm_set/tasks/announce_routes.yml index 39333b0f5d..fe309231cb 100644 --- a/ansible/roles/vm_set/tasks/announce_routes.yml +++ b/ansible/roles/vm_set/tasks/announce_routes.yml @@ -82,12 +82,14 @@ delegate_to: "{{ ptf_host }}" - name: Add exabgpv4 supervisor config and start related processes + when: "{{ topology['VMs'] | length > 0 }}" supervisorctl: name: "exabgpv4:" state: present # present contains `supervisorctl reread` and `supervisorctl add` delegate_to: "{{ ptf_host }}" - name: Add exabgpv6 supervisor config and start related processes + when: "{{ topology['VMs'] | length > 0 }}" supervisorctl: name: "exabgpv6:" state: present # present contains `supervisorctl reread` and `supervisorctl add` diff --git a/ansible/roles/vm_set/tasks/kickstart_vm.yml b/ansible/roles/vm_set/tasks/kickstart_vm.yml index ca74b1329a..1d76c6b611 100644 --- a/ansible/roles/vm_set/tasks/kickstart_vm.yml +++ b/ansible/roles/vm_set/tasks/kickstart_vm.yml @@ -13,14 +13,14 @@ - block: - name: Wait until vm {{ vm_name }} is loaded kickstart: telnet_port={{ serial_port }} - login={{ eos_default_login }} - password={{ eos_default_password }} - hostname={{ hostname }} + login="{{ eos_default_login }}" + password="{{ eos_default_password }}" + hostname="{{ hostname }}" mgmt_ip="{{ mgmt_ip_address }}/{{ mgmt_prefixlen }}" - mgmt_gw={{ vm_mgmt_gw | default(mgmt_gw) }} - new_login={{ eos_login }} - new_password={{ eos_password }} - new_root_password={{ eos_root_password }} + mgmt_gw="{{ vm_mgmt_gw | default(mgmt_gw) }}" + new_login="{{ eos_login }}" + new_password="{{ eos_password }}" + new_root_password="{{ eos_root_password }}" register: kickstart_output until: '"kickstart_code" in kickstart_output and kickstart_output.kickstart_code == 0' retries: 5 @@ -129,7 +129,7 @@ - name: Respin vm {{ vm_name }} include_tasks: respin_cisco_vm.yml vars: - src_disk_image: "{{ root_path }}/images/{{ cisco_image_filename }}" + src_disk_image: "{{ root_path }}/images/{{ vcisco_image_filename }}" disk_image: "{{ root_path }}/disks/{{ vm_name }}.img" when: vm_name in respin_vms @@ -161,7 +161,7 @@ - name: Respin failed vm {{ vm_name }} include_tasks: respin_cisco_vm.yml vars: - src_disk_image: "{{ root_path }}/images/{{ cisco_image_filename }}" + src_disk_image: "{{ root_path }}/images/{{ vcisco_image_filename }}" disk_image: "{{ root_path }}/disks/{{ vm_name }}.img" - name: Check failed cisco {{ vm_name }} reachablity diff --git a/ansible/roles/vm_set/tasks/main.yml b/ansible/roles/vm_set/tasks/main.yml index d0d7dddc47..69bbae918e 100644 --- a/ansible/roles/vm_set/tasks/main.yml +++ b/ansible/roles/vm_set/tasks/main.yml @@ -1,17 +1,19 @@ # This role creates a set of VM with veos or SONiC or cisco or Ubuntu for Kubernetes master # Input parameters for the role: -# - action: 'start', 'stop' or 'renumber' for creating, removeing, or renumbering vm set respectively +# - action: 'start', 'stop' or 'renumber' for creating, removing, or renumbering vm set respectively # - id: sequence number for vm set on the host. # - external_port: interface which will be used as parent for vlan interface creation # - vlan_base: first vlan id for the VMs # - VMs: a dictionary which contains hostnames of VMs as a key and a dictionary with parameters (num, memory, mgmt_ip) for every VM. # - topology: a dictionary which contains hostnames of VMs as a key and vlans value which define a topology (numbers of connected ports for every VM) # - mgmt_bridge: linux bridge which is used for management interface connections -# - root_path: path where disk images for VMs are created -# - hdd_image_filename: base hdd image for VMs -# - cd_image_filename: base cd image for VMs -# - vm_images_url: url where base images are located -# - vmimages_saskey: a key for Azure download service. Could be set to '' + +# Variables used by the role are mostly defined in files under ansible/group_vars/vm_host directory. +# Supported neighbor types are: veos, sonic, cisco, ubuntu, k8s +# For each of the supported neighbor types, there is a file in ansible/group_vars/vm_host directory which defines the +# the variables for the neighbor type. The neighbor VM image files usually can be manually prepared or automatically +# downloaded from the URLs defined in the variables. Please update the URLs to the actual URLs of the image files in +# your environment. # Need latest ubuntu 4.10 kernel to fix a openvswitch bug # https://bugs.launchpad.net/ubuntu/+source/kernel-package/+bug/1685742 diff --git a/ansible/roles/vm_set/tasks/probe_image_url.yml b/ansible/roles/vm_set/tasks/probe_image_url.yml new file mode 100644 index 0000000000..dadb97a3dd --- /dev/null +++ b/ansible/roles/vm_set/tasks/probe_image_url.yml @@ -0,0 +1,15 @@ +- name: Probe if the URL works + uri: + url: "{{ item }}" + method: HEAD + status_code: 200 + return_content: no + timeout: 3 + environment: "{{ proxy_env | default({}) }}" + register: image_url_probe_result + failed_when: false + +- name: Append working URL to working_image_urls list + set_fact: + working_image_urls: "{{ working_image_urls + [ item ] }}" + when: image_url_probe_result.status == 200 diff --git a/ansible/roles/vm_set/tasks/renumber_topo.yml b/ansible/roles/vm_set/tasks/renumber_topo.yml index aa57c26681..8d47140726 100644 --- a/ansible/roles/vm_set/tasks/renumber_topo.yml +++ b/ansible/roles/vm_set/tasks/renumber_topo.yml @@ -207,4 +207,7 @@ nic_simulator_action: start when: topology.host_interfaces_active_active is defined and topology.host_interfaces_active_active|length > 0 + - name: Start tacacs+ daily daemon + include_tasks: start_tacacs_daily_daemon.yml + when: container_type == "PTF" diff --git a/ansible/roles/vm_set/tasks/start.yml b/ansible/roles/vm_set/tasks/start.yml index 3b77efb703..846c7e601f 100644 --- a/ansible/roles/vm_set/tasks/start.yml +++ b/ansible/roles/vm_set/tasks/start.yml @@ -20,91 +20,231 @@ - block: - name: Check hdd image - stat: path={{ root_path }}/images/{{ hdd_image_filename }} - register: hdd_stat - - - name: Fail if there are no hdd image and skip image downloading is active - fail: msg="Please put {{ hdd_image_filename }} to {{ root_path }}/images" - when: not hdd_stat.stat.exists and skip_image_downloading - - - name: Download hdd image - get_url: url="{{ vm_images_url }}/{{ hdd_image_filename }}?{{ vmimage_saskey }}" dest="{{ root_path }}/images/{{ hdd_image_filename }}" - environment: "{{ proxy_env | default({}) }}" - when: not hdd_stat.stat.exists and not skip_image_downloading + stat: path={{ root_path }}/images/{{ veos_hdd_image_filename }} + register: veos_hdd_image_file_stat + + - name: Download veos hdd image if no local file exists + block: + + - name: Fail if skip_veos_image_downloading is true + fail: + msg: "Failed, no local veos hdd image and skip_veos_image_downloading is true" + when: skip_veos_image_downloading + + - name: Init veos_hdd_image_urls when veos_hdd_image_url value type is string + set_fact: + veos_hdd_image_urls: + - "{{ veos_hdd_image_url }}" + when: veos_hdd_image_url | type_debug == 'string' + + - name: Init veos_hdd_image_urls when veos_hdd_image_url value type is list + set_fact: + veos_hdd_image_urls: "{{ veos_hdd_image_url }}" + when: veos_hdd_image_url | type_debug == 'list' + + - name: Init working_image_urls list + set_fact: + working_image_urls: [] + + - name: Loop veos_hdd_image_urls to find out working URLs + include_tasks: probe_image_url.yml + loop: "{{ veos_hdd_image_urls }}" + + - name: Fail if no working veos hdd image download url is found + fail: + msg: [ + "Failed, no working veos hdd image download URL is found. There are 2 options to fix it:", + " 1. Fix veos_hdd_image_url defined in ansible/group_vars/vm_host/veos.yml", + " 2. Manually put veos hdd image to {{ root_path }}/images/{{ veos_hdd_image_filename }}", + ] + when: working_image_urls | length == 0 + + - name: Download veos hdd image from the first URL in working_image_urls + get_url: + url: "{{ working_image_urls[0] }}" + dest: "{{ root_path }}/images/{{ veos_hdd_image_filename }}" + environment: "{{ proxy_env | default({}) }}" + + when: not veos_hdd_image_file_stat.stat.exists - name: Check cd image - stat: path={{ root_path }}/images/{{ cd_image_filename }} - register: cd_stat - - - name: Fail if there are no cd image and skip image downloading is active - fail: msg="Please put {{ cd_image_filename }} to {{ root_path }}/images" - when: not cd_stat.stat.exists and skip_image_downloading - - - name: Download cd image - get_url: url="{{ vm_images_url }}/{{ cd_image_filename }}?{{ cdimage_saskey }}" dest="{{ root_path }}/images/{{ cd_image_filename }}" - environment: "{{ proxy_env | default({}) }}" - when: not cd_stat.stat.exists and not skip_image_downloading + stat: path={{ root_path }}/images/{{ veos_cd_image_filename }} + register: veos_cd_image_file_stat + + - name: Download veos cd image if no local file exists + block: + + - name: Fail if skip_veos_image_downloading is true + fail: + msg: "Failed, no local veos cd image and skip_veos_image_downloading is true" + when: skip_veos_image_downloading + + - name: Init veos_cd_image_urls when veos_cd_image_url value type is string + set_fact: + veos_cd_image_urls: + - "{{ veos_cd_image_url }}" + when: veos_cd_image_url | type_debug == 'string' + + - name: Init veos_cd_image_urls when veos_cd_image_url value type is list + set_fact: + veos_cd_image_urls: "{{ veos_cd_image_url }}" + when: veos_cd_image_url | type_debug == 'list' + + - name: Init working_image_urls list + set_fact: + working_image_urls: [] + + - name: Loop veos_cd_image_urls to find out working URLs + include_tasks: probe_image_url.yml + loop: "{{ veos_cd_image_urls }}" + + - name: Fail if no working veos cd image download url is found + fail: + msg: [ + "Failed, no working veos cd image download URL is found. There are 2 options to fix it:", + " 1. Fix veos_cd_image_url defined in ansible/group_vars/vm_host/veos.yml", + " 2. Manually put veos cd image to {{ root_path }}/images/{{ veos_cd_image_filename }}", + ] + when: working_image_urls | length == 0 + + - name: Download veos cd image from the first URL in working_image_urls + get_url: + url: "{{ working_image_urls[0] }}" + dest: "{{ root_path }}/images/{{ veos_cd_image_filename }}" + environment: "{{ proxy_env | default({}) }}" + + when: not veos_cd_image_file_stat.stat.exists - set_fact: - src_image_name: "{{ hdd_image_filename }}" + src_image_name: "{{ veos_hdd_image_filename }}" when: (vm_type | lower) == "veos" - block: - name: Check SONiC image - stat: path={{ root_path }}/images/{{ sonic_image_filename }} - register: img_stat - - - name: Fail if there are no SONiC image and skip image downloading is active - fail: msg="Please put {{ sonic_image_filename }} to {{ root_path }}/images" - when: not img_stat.stat.exists and skip_image_downloading - - - name: Fail if there is no SONiC image download URL specified - fail: msg="Please set sonic_image_url to the URL where the SONiC image can be downloaded from" - when: not img_stat.stat.exists and not skip_image_downloading and sonic_image_url is not defined - - - name: Download SONiC image - get_url: url="{{ sonic_image_url }}" dest="{{ root_path }}/images/{{ sonic_image_filename }}" - environment: "{{ proxy_env | default({}) }}" - when: not img_stat.stat.exists and not skip_image_downloading and sonic_image_url is defined + stat: path={{ root_path }}/images/{{ vsonic_image_filename }} + register: sonic_img_stat + + - name: Download SONiC image if no local file exists + block: + + - name: Fail if skip_vsonic_image_downloading is true + fail: + msg: "Failed, no local SONiC image and skip_vsonic_image_downloading is true" + when: skip_vsonic_image_downloading + + - name: Init vsonic_image_urls when vsonic_image_url value type is string + set_fact: + vsonic_image_urls: + - "{{ vsonic_image_url }}" + when: vsonic_image_url | type_debug == 'string' + + - name: Init vsonic_image_urls when vsonic_image_url value type is list + set_fact: + vsonic_image_urls: "{{ vsonic_image_url }}" + when: vsonic_image_url | type_debug == 'list' + + - name: Init working_image_urls list + set_fact: + working_image_urls: [] + + - name: Loop vsonic_image_urls to find out working URLs + include_tasks: probe_image_url.yml + loop: "{{ vsonic_image_urls }}" + + - name: Fail if no working SONiC image download url is found + fail: + msg: [ + "Failed, no working SONiC image download URL is found. There are 2 options to fix it:", + " 1. Fix vsonic_image_url defined in ansible/group_vars/vm_host/sonic.yml", + " 2. Manually put SONiC image to {{ root_path }}/images/{{ vsonic_image_filename }}", + ] + when: working_image_urls | length == 0 + + - name: Download SONiC image from the first URL in working_image_urls + get_url: + url: "{{ working_image_urls[0] }}" + dest: "{{ root_path }}/images/{{ vsonic_image_filename }}" + environment: "{{ proxy_env | default({}) }}" + + - name: Get downloaded SONiC image info + stat: path={{ root_path }}/images/{{ vsonic_image_filename }} + register: img_stat + + when: not sonic_img_stat.stat.exists - name: Get downloaded SONiC image info - stat: path={{ root_path }}/images/{{ sonic_image_filename }} - register: downloaded_img_stat + stat: path={{ root_path }}/images/{{ vsonic_image_filename }} + register: downloaded_sonic_img_stat - block: - name: Rename file to have a .gz suffix - command: mv {{ root_path }}/images/{{ sonic_image_filename }} {{ root_path }}/images/{{ sonic_image_filename }}.gz + command: mv {{ root_path }}/images/{{ vsonic_image_filename }} {{ root_path }}/images/{{ vsonic_image_filename }}.gz - name: Decompress file - command: gunzip {{ root_path }}/images/{{ sonic_image_filename }}.gz + command: gunzip {{ root_path }}/images/{{ vsonic_image_filename }}.gz - when: '"application/gzip" in downloaded_img_stat.stat.mimetype' + when: '"application/gzip" in downloaded_sonic_img_stat.stat.mimetype' - set_fact: - src_image_name: "{{ sonic_image_filename }}" + src_image_name: "{{ vsonic_image_filename }}" when: (vm_type | lower) == "vsonic" - block: - name: Check cisco image - stat: path={{ root_path }}/images/{{ cisco_image_filename }} - register: img_stat - - - name: Fail if there are no cisco image and skip image downloading is active - fail: msg="Please put {{ cisco_image_filename }} to {{ root_path }}/images" - when: not img_stat.stat.exists and skip_image_downloading - - - name: Download cisco image - get_url: url="{{ vm_images_url }}/{{ cisco_image_filename }}?{{ vciscoimage_saskey }}" dest="{{ root_path }}/images/{{ cisco_image_filename }}" - environment: "{{ proxy_env | default({}) }}" - when: not img_stat.stat.exists and not skip_image_downloading + stat: path={{ root_path }}/images/{{ vcisco_image_filename }} + register: cisco_img_stat + + - name: Download cisco image if no local file exists + block: + + - name: Fail if skip_vcisco_image_downloading is true + fail: + msg: "Failed, no local cisco image and skip_vcisco_image_downloading is true" + when: skip_vcisco_image_downloading + + - name: Init vcisco_image_urls when vcisco_image_url value type is string + set_fact: + vcisco_image_urls: + - "{{ vcisco_image_url }}" + when: vcisco_image_url | type_debug == 'string' + + - name: Init vcisco_image_urls when vcisco_image_url value type is list + set_fact: + vcisco_image_urls: "{{ vcisco_image_url }}" + when: vcisco_image_url | type_debug == 'list' + + - name: Init working_image_urls list + set_fact: + working_image_urls: [] + + - name: Loop vcisco_image_urls to find out working URLs + include_tasks: probe_image_url.yml + loop: "{{ vcisco_image_urls }}" + + - name: Fail if no working cisco image download url is found + fail: + msg: [ + "Failed, no working cisco image download URL is found. There are 2 options to fix it:", + " 1. Fix vcisco_image_url defined in ansible/group_vars/vm_host/cisco.yml", + " 2. Manually put cisco image to {{ root_path }}/images/{{ vcisco_image_filename }}", + ] + when: working_image_urls | length == 0 + + - name: Download cisco image from the first URL in working_image_urls + get_url: + url: "{{ working_image_urls[0] }}" + dest: "{{ root_path }}/images/{{ vcisco_image_filename }}" + environment: "{{ proxy_env | default({}) }}" + + when: not cisco_img_stat.stat.exists - set_fact: - src_image_name: "{{ cisco_image_filename }}" + src_image_name: "{{ vcisco_image_filename }}" when: (vm_type | lower) == "vcisco" @@ -140,7 +280,7 @@ serial_port: "{{ vm_console_base|int + vm_name[4:]|int }}" src_disk_image: "{{ root_path }}/images/{{ src_image_name }}" disk_image_dir: "{{ root_path }}/disks" - cdrom_image: "{{ root_path }}/images/{{ cd_image_filename }}" + cdrom_image: "{{ root_path }}/images/{{ veos_cd_image_filename }}" mgmt_tap: "{{ vm_name }}-m" backplane_tap: "{{ vm_name }}-back" with_items: "{{ VM_hosts }}" @@ -157,7 +297,7 @@ serial_port: "{{ vm_console_base|int + vm_name[4:]|int }}" src_disk_image: "{{ root_path }}/images/{{ src_image_name }}" disk_image_dir: "{{ root_path }}/disks" - cdrom_image: "{{ root_path }}/images/{{ cd_image_filename }}" + cdrom_image: "{{ root_path }}/images/{{ veos_cd_image_filename }}" mgmt_tap: "{{ vm_name }}-m" backplane_tap: "{{ vm_name }}-back" with_items: "{{ VM_hosts }}" diff --git a/ansible/roles/vm_set/tasks/start_tacacs_daily_daemon.yml b/ansible/roles/vm_set/tasks/start_tacacs_daily_daemon.yml new file mode 100644 index 0000000000..1687c3590f --- /dev/null +++ b/ansible/roles/vm_set/tasks/start_tacacs_daily_daemon.yml @@ -0,0 +1,132 @@ +--- +- name: Include tacacs_passkey by testbed_facts['inv_name'] + block: + - name: set default testbed file + set_fact: + testbed_file: testbed.yaml + when: testbed_file is not defined + + - name: Gathering testbed information + test_facts: testbed_name="{{ testbed_name }}" testbed_file="{{ testbed_file }}" + delegate_to: localhost + ignore_errors: yes + + - name: Include tacacs_passkey by inv_name + include_vars: "{{ playbook_dir }}/group_vars/{{testbed_facts['inv_name']}}/{{testbed_facts['inv_name']}}.yml" + when: testbed_facts is defined + + when: tacacs_passkey is not defined + +- name: Include tacacs_passkey by inventory_file + block: + - name: Get inventory folder name + set_fact: inv_file="{{ playbook_dir }}/group_vars/{{ inventory_file.split("/")[-1] }}/{{ inventory_file.split("/")[-1] }}.yml" + + - debug: msg="inv_file {{ inv_file }}" + + - name: Check inventory file exist + stat: + path: "{{ inv_file }}" + register: inventory_file + + - name: Include tacacs_passkey from inventory file + include_vars: "{{ inv_file }}" + when: inventory_file.stat.exists + + when: tacacs_passkey is not defined + +- name: Include default tacacs_passkey + include_vars: "{{ playbook_dir }}/group_vars/lab/lab.yml" + when: tacacs_passkey is not defined + +- debug: msg="tacacs_passkey {{ tacacs_passkey }}" + +- name: Set ptf host + set_fact: + ptf_host: "ptf_{{ vm_set_name }}" + +- debug: msg="ptf_host {{ ptf_host }}" + +- name: Include duthost user name + block: + - name: Set duthost user name + set_fact: + dut_host_user: "{{ secret_group_vars['str']['ansible_ssh_user'] }}" + + when: + - secret_group_vars is defined + - secret_group_vars['str'] is defined + - secret_group_vars['str']['ansible_ssh_user'] is defined + + - name: Set default duthost user name from group_vars/lab/secrets.yml + block: + - name: Include sonicadmin_user from group_vars/lab/secrets.yml + include_vars: "{{ playbook_dir }}/group_vars/lab/secrets.yml" + + - name: Set dut user name + set_fact: + dut_host_user: "{{ sonicadmin_user }}" + + when: + - dut_host_user is not defined + +- debug: msg="dut_host_user {{ dut_host_user }}" + +- name: Include duthost password + block: + - name: Encrypt encrypted_sonic_password from secret_group_vars + block: + + - name: Encrypt TACACS password from secret_group_vars + shell: python3 -c "import crypt; print(crypt.crypt('{{secret_group_vars['str']['altpasswords'][0]}}', 'abc'))" + register: encrypted_sonic_password + delegate_to: "{{ ptf_host }}" + + when: + - secret_group_vars is defined + - secret_group_vars['str'] is defined + - secret_group_vars['str']['altpasswords'] is defined + + - name: Encrypt default encrypted_sonic_password from group_vars/lab/secrets.yml + block: + - name: Include sonicadmin_password from group_vars/lab/secrets.yml + include_vars: "{{ playbook_dir }}/group_vars/lab/secrets.yml" + + - name: Encrypt TACACS password from sonicadmin_password + shell: python3 -c "import crypt; print(crypt.crypt('{{sonicadmin_password}}', 'abc'))" + register: encrypted_sonic_password + delegate_to: "{{ ptf_host }}" + + when: + - tacacs_user_passwd is not defined + + - name: Set TACACS password from encrypted_sonic_password + set_fact: + tacacs_user_passwd: '{{ encrypted_sonic_password.stdout }}' + +- debug: msg="tacacs_user_passwd {{ tacacs_user_passwd }}" + +- block: + - name: Generate tacacs daily daemon config file + template: + src: tac_plus_daily.conf.j2 + dest: /etc/tac_plus_daily.conf + delegate_to: "{{ ptf_host }}" + + - name: Upload tacacs daily daemon file + copy: + src: 'scripts/tacacs_daily_daemon' + dest: /root/tacacs_daily_daemon + mode: "0755" + delegate_to: "{{ ptf_host }}" + + - name: Upload tacacs daily start script + copy: + src: 'scripts/start_tacacs.sh' + dest: /root/start_tacacs.sh + mode: "0755" + delegate_to: "{{ ptf_host }}" + + - name: Start tacacs daily daemon + shell: /root/start_tacacs.sh + delegate_to: "{{ptf_host}}" diff --git a/ansible/roles/vm_set/tasks/start_vm.yml b/ansible/roles/vm_set/tasks/start_vm.yml index d746fcd38b..c7f40ef5c1 100644 --- a/ansible/roles/vm_set/tasks/start_vm.yml +++ b/ansible/roles/vm_set/tasks/start_vm.yml @@ -79,5 +79,5 @@ - name: "Pause after started every {{ batch_size }} VMs" pause: seconds="{{ interval }}" when: - - "{{ vm_index }} % {{ batch_size }} == 0" - - "{{ interval }} > 0" + - (vm_index|int % batch_size|int) == 0 + - interval|int > 0 diff --git a/ansible/roles/vm_set/templates/sonic.xml.j2 b/ansible/roles/vm_set/templates/sonic.xml.j2 index 496241a5e0..db7a3e4266 100644 --- a/ansible/roles/vm_set/templates/sonic.xml.j2 +++ b/ansible/roles/vm_set/templates/sonic.xml.j2 @@ -3,13 +3,25 @@ {% if hwsku == 'msft_four_asic_vs' %} 8 10 + + + + {% elif hwsku == 'msft_multi_asic_vs' %} 8 16 + + + + {% else %} 4 4 4 + + + + {% endif %} /machine diff --git a/ansible/roles/vm_set/templates/tac_plus_daily.conf.j2 b/ansible/roles/vm_set/templates/tac_plus_daily.conf.j2 new file mode 100644 index 0000000000..f8fe7000ff --- /dev/null +++ b/ansible/roles/vm_set/templates/tac_plus_daily.conf.j2 @@ -0,0 +1,22 @@ +# Created by Henry-Nicolas Tourneur(henry.nicolas@tourneur.be) +# See man(5) tac_plus.conf for more details + +# Define where to log accounting data, this is the default. + +accounting file = /var/log/tac_plus_daily.acct + +# This is the key that clients have to use to access Tacacs+ + +key = {{ tacacs_passkey }} + +group = netadmin { + default service = permit + service = exec { + priv-lvl = 15 + } +} + +user = {{ dut_host_user }} { + login = des {{ tacacs_user_passwd }} + member = netadmin +} diff --git a/ansible/scripts/start_tacacs.sh b/ansible/scripts/start_tacacs.sh new file mode 100644 index 0000000000..dae3daef9d --- /dev/null +++ b/ansible/scripts/start_tacacs.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +/root/tacacs_daily_daemon & diff --git a/ansible/scripts/tacacs_daily_daemon b/ansible/scripts/tacacs_daily_daemon new file mode 100644 index 0000000000..01b5c7f03a --- /dev/null +++ b/ansible/scripts/tacacs_daily_daemon @@ -0,0 +1,36 @@ +#!/bin/sh + +# This TACACS server need bind to port 49, because SONiC load_minigraph only support port 49 +TACPLUS_PORT=49 + +# Always kill running tac_plus process on port $TACPLUS_PORT to reload config +# It will be start later by tacacs_daily_daemon +TACPLUS_PID=$(ps -ef | grep "tac_plus .* -p $TACPLUS_PORT" | grep -v "grep" | awk '{print $2}') +if [ $TACPLUS_PID ]; then + echo "tac_plus already running on port $TACPLUS_PORT, stop it to reload config" + kill -9 $TACPLUS_PID +fi + +# Exit if tacacs_daily_daemon already running +PROCESS_NUM=$(pgrep -l tacacs_daily_daemon | wc -l) +if [ $PROCESS_NUM -ge 1 ]; then + echo "tacacs_daily_daemon already running" + exit 0 +fi + +# Exit if tacacs_daily_daemon already running in background +# need check -ge3, because every grep command will create a process with same name +PROCESS_NUM=$(ps -ef | grep "/bin/sh .*tacacs_daily_daemon" | grep -v "grep" | wc -l) +if [ $PROCESS_NUM -ge 3 ]; then + echo "tacacs_daily_daemon already running in background" $PROCESS_NUM + exit 0 +fi + +echo "starting tac_plus for daily work" +while true; +do + # start tac_plus will kill existed tac_plus instance bind to same port + # Enable Authentication/Authorization/Accounting debug by: -d 88 + /usr/sbin/tac_plus -d 88 -l /var/log/tac_plus_daily.log -C /etc/tac_plus_daily.conf -p $TACPLUS_PORT -G + echo "tac_plus existed, restarting" +done diff --git a/ansible/snappi-sonic b/ansible/snappi-sonic new file mode 100644 index 0000000000..1d2f045f3c --- /dev/null +++ b/ansible/snappi-sonic @@ -0,0 +1,23 @@ +[sonic_dell64_40] +sonic-s6100-dut1 ansible_host=10.251.0.233 +sonic-s6100-dut2 ansible_host=10.251.0.234 + +[sonic_dell64_40:vars] +hwsku="Force10-S6100" +iface_speed='40000' + +[Server_6] +snappi-sonic ansible_host=10.251.0.234 os=snappi + +[sonic:children] +sonic_dell64_40 + +[sonic:vars] +mgmt_subnet_mask_length='23' + +[snappi-sonic:children] +sonic +snappi_chassis + +[ptf] +snappi-sonic-ptf ansible_host='10.251.0.232' diff --git a/ansible/ssh_session_gen.py b/ansible/ssh_session_gen.py index f45fc16060..88e0a846d9 100644 --- a/ansible/ssh_session_gen.py +++ b/ansible/ssh_session_gen.py @@ -58,10 +58,11 @@ def __init__( "FanoutLeafSonic": {"user": leaf_fanout_user, "pass": leaf_fanout_pass}, "FanoutRoot": {"user": root_fanout_user, "pass": root_fanout_pass}, "ConsoleServer": {"user": console_server_user, "pass": console_server_pass}, + "MgmtTsToRRouter": {"user": console_server_user, "pass": console_server_pass}, "PTF": {"user": ptf_user, "pass": ptf_pass}, } - def get_ssh_cred(self, device: DeviceInfo) -> Tuple[str, str, str]: + def get_ssh_cred(self, device: DeviceInfo) -> Tuple[str, str, str, str]: """ Get SSH info for a testbed node. @@ -72,6 +73,7 @@ def get_ssh_cred(self, device: DeviceInfo) -> Tuple[str, str, str]: tuple: SSH IP, user and password. """ ssh_ip = device.management_ip + ssh_ipv6 = None ssh_user = ( self.ssh_overrides[device.device_type]["user"] if device.device_type in self.ssh_overrides @@ -83,11 +85,12 @@ def get_ssh_cred(self, device: DeviceInfo) -> Tuple[str, str, str]: else "" ) - if not ssh_ip or not ssh_user or not ssh_pass: + if not ssh_ip or not ssh_user or not ssh_pass or not ssh_ipv6: try: host_vars = self.ansible_hosts.get_host_vars(device.hostname) ssh_ip = host_vars["ansible_host"] if not ssh_ip else ssh_ip + ssh_ipv6 = host_vars["ansible_hostv6"] if not ssh_ipv6 and "ansible_hostv6" in host_vars else ssh_ipv6 ssh_user = host_vars["creds"]["username"] if not ssh_user else ssh_user ssh_pass = ( host_vars["creds"]["password"][-1] if not ssh_pass else ssh_pass @@ -98,10 +101,11 @@ def get_ssh_cred(self, device: DeviceInfo) -> Tuple[str, str, str]: ) ssh_ip = "" if ssh_ip is None else ssh_ip + ssh_ipv6 = "" if ssh_ipv6 is None else ssh_ipv6 ssh_user = "" if ssh_user is None else ssh_user ssh_pass = "" if ssh_pass is None else ssh_pass - return ssh_ip, ssh_user, ssh_pass + return ssh_ip, ssh_ipv6, ssh_user, ssh_pass class DeviceSshSessionRepoGenerator(object): @@ -121,13 +125,20 @@ def generate_ssh_session_for_device(self, device: DeviceInfo, session_path: str) if not device.is_ssh_supported(): return - ssh_ip, ssh_user, ssh_pass = self.ssh_info_solver.get_ssh_cred(device) - if ssh_ip is None: + ssh_ip, ssh_ipv6, ssh_user, ssh_pass = self.ssh_info_solver.get_ssh_cred(device) + if not ssh_ip and not ssh_ipv6: print( f"WARNING: Management IP is not specified for testbed node, skipped: {device.hostname}" ) return + if device.console_device: + console_ssh_ip, _, console_ssh_user, console_ssh_pass = self.ssh_info_solver.get_ssh_cred( + device.console_device) + console_ssh_port = device.console_port + else: + console_ssh_ip, console_ssh_user, console_ssh_pass, console_ssh_port = None, None, None, 0 + if not ssh_user: print( "WARNING: SSH credential is missing for device: {}".format( @@ -138,8 +149,13 @@ def generate_ssh_session_for_device(self, device: DeviceInfo, session_path: str) self.repo_generator.generate( session_path, ssh_ip, + ssh_ipv6, ssh_user, ssh_pass, + console_ssh_ip, + console_ssh_port, + console_ssh_user, + console_ssh_pass ) @@ -252,7 +268,8 @@ def _generate_ssh_sessions_for_device_inventory( ) for device in device_inventory.devices.values(): - device_type = device_type_pattern.sub("-", device.device_type).lower() + device_type = device.device_type.lower() + print(device_type) session_path = os.path.join( "devices", device_inventory.inv_name, device_type, device.hostname ) @@ -270,13 +287,9 @@ def main(args): args.target, args.template_file_path ) elif args.format == "ssh": - ssh_config_params = {} - for param in args.ssh_config_params: - key, value = param.split("=") - ssh_config_params[key] = value repo_generator = SshConfigSshSessionRepoGenerator( - args.target, ssh_config_params + args.target, args.ssh_config_params, args.console_ssh_config_params ) else: print("Unsupported output format: {}".format(args.format)) @@ -328,6 +341,19 @@ def main(args): device_repo_generator.generate() +class SSHConfigParamsParser(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + current_params = getattr(namespace, self.dest) + if current_params is None: + current_params = {} + for configEntry in values: + if '=' not in configEntry: + raise argparse.ArgumentError(self, f"Invalid syntax '{configEntry}' for SSH parameter") + key, value = configEntry.split('=', 2) + current_params[key] = value + setattr(namespace, self.dest, current_params) + + if __name__ == "__main__": # Parse arguments example_text = """Examples: @@ -407,16 +433,27 @@ def main(args): help="Output target format, currently supports securecrt or ssh.", ) - parser.add_argument( + group = parser.add_argument_group("templates") + + group.add_argument( "--ssh-config-params", type=str, dest="ssh_config_params", nargs="+", - default="", - help="Extra SSH config parameters, only used when --format=ssh. E.g. ProxyJump=jumpbox", + action=SSHConfigParamsParser, + help="Extra SSH config parameters used for all devices, only used when --format=ssh. E.g. ProxyJump=jumpbox", ) - parser.add_argument( + group.add_argument( + "--console-ssh-config-params", + type=str, + dest="console_ssh_config_params", + nargs="+", + action=SSHConfigParamsParser, + help="Extra SSH config parameters used for consoles, only used when --format=ssh. E.g. ProxyJump=jumpbox", + ) + + group.add_argument( "-p", "--template", type=str, @@ -425,84 +462,86 @@ def main(args): "Only used when --format=securecrt.", ) - parser.add_argument( + group = parser.add_argument_group("credentials") + + group.add_argument( "--dut-user", type=str, dest="dut_user", help="SSH user name of DUTs. If not specified, we will use ansible to get the SSH configuration.", ) - parser.add_argument( + group.add_argument( "--dut-pass", type=str, dest="dut_pass", help="SSH password of DUTs. If not specified, we will use ansible to get the SSH configuration.", ) - parser.add_argument( + group.add_argument( "--ptf-user", type=str, dest="ptf_user", help="SSH user name of PTF containers. If not specified, we will use ansible to get the SSH configuration.", ) - parser.add_argument( + group.add_argument( "--ptf-pass", type=str, dest="ptf_pass", help="SSH password of PTF containers. If not specified, we will use ansible to get the SSH configuration.", ) - parser.add_argument( + group.add_argument( "--server-user", type=str, dest="server_user", help="SSH user name of servers. If not specified, we will use ansible to get the SSH configuration.", ) - parser.add_argument( + group.add_argument( "--server-pass", type=str, dest="server_pass", help="SSH password of servers. If not specified, we will use ansible to get the SSH configuration.", ) - parser.add_argument( + group.add_argument( "--leaf-fanout-user", type=str, dest="leaf_fanout_user", help="SSH user name of leaf fanouts. If not specified, we will use ansible to get the SSH configuration.", ) - parser.add_argument( + group.add_argument( "--leaf-fanout-pass", type=str, dest="leaf_fanout_pass", help="SSH password of leaf fanouts. If not specified, we will use ansible to get the SSH configuration.", ) - parser.add_argument( + group.add_argument( "--root-fanout-user", type=str, dest="root_fanout_user", help="SSH user name of root fanouts. If not specified, we will use ansible to get the SSH configuration.", ) - parser.add_argument( + group.add_argument( "--root-fanout-pass", type=str, dest="root_fanout_pass", help="SSH password of root fanouts. If not specified, we will use ansible to get the SSH configuration.", ) - parser.add_argument( + group.add_argument( "--console-server-user", type=str, dest="console_server_user", help="SSH user name of console server. If not specified, we will use ansible to get the SSH configuration.", ) - parser.add_argument( + group.add_argument( "--console-server-pass", type=str, dest="console_server_pass", diff --git a/ansible/templates/minigraph_png.j2 b/ansible/templates/minigraph_png.j2 index 8eda4b3f58..8680e0dd91 100644 --- a/ansible/templates/minigraph_png.j2 +++ b/ansible/templates/minigraph_png.j2 @@ -41,8 +41,18 @@ DeviceInterfaceLink {{ inventory_hostname }} {{ vlan_intfs[host_index] }} +{% if 'ptp' in topo %} +{% if device_conn[inventory_hostname][port_alias_map[vlan_intfs[host_index]]] is defined %} + {{ device_conn[inventory_hostname][port_alias_map[vlan_intfs[host_index]]]['peerdevice'] }} + {{ device_conn[inventory_hostname][port_alias_map[vlan_intfs[host_index]]]['peerport'] }} +{% else %} + Unknown Device + Unknown Port +{% endif %} +{% else %} Servers{{ host_index }} eth0 +{% endif %} {% endfor %} {% if 'dualtor' in topo %} @@ -103,7 +113,17 @@ {% endif %} +{% if enable_compute_ai_deployment|default('false')|bool %} + {% if vm_topo_config['dut_type'] | lower == 'torrouter' %} + + {% elif vm_topo_config['dut_type'] | lower == 'leafrouter' %} + + {% else %} + {% endif %} +{% else %} + +{% endif %} {{ inventory_hostname }} {{ hwsku }} {% if 'loopback' in dual_tor_facts %} @@ -179,6 +199,8 @@ {% if vm_topo_config['vm'][dev]['intfs'][dut_index|int]|length %} {% if 'properties' in vm_topo_config['vm'][dev] and 'device_type' in vm_topo_config['vm'][dev]['properties'] %} {% set dev_type = vm_topo_config['vm'][dev]['properties']['device_type'] %} +{% elif ('T1' in dev) and (enable_compute_ai_deployment|default('false')|bool) %} +{% set dev_type = 'BackEndLeafRouter' %} {% elif 'T1' in dev %} {% set dev_type = 'LeafRouter' %} {% elif 'T2' in dev %} @@ -189,6 +211,8 @@ {% else %} {% set dev_type = 'AZNGHub' %} {% endif %} +{% elif ('T0' in dev) and (enable_compute_ai_deployment|default('false')|bool) %} +{% set dev_type = 'BackEndToRRouter' %} {% elif 'T0' in dev %} {% set dev_type = 'ToRRouter' %} {% elif 'M1' in dev %} diff --git a/ansible/testbed.csv b/ansible/testbed.csv index 2a9dcf27e5..3892122bbb 100644 --- a/ansible/testbed.csv +++ b/ansible/testbed.csv @@ -1,14 +1,16 @@ # conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,inv_name,auto_recover,comment -ptf1-m,ptf1,ptf32,docker-ptf,ptf-unknown,10.255.0.188/24,,server_1,,str-msn2700-01,lab,False,Test ptf Mellanox -ptf2-b,ptf2,ptf64,docker-ptf,ptf-unknown,10.255.0.189/24,,server_1,,lab-s6100-01,lab,False,Test ptf Broadcom -vms-sn2700-t1,vms1-1,t1,docker-ptf,ptf-unknown,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests Mellanox SN2700 vms -vms-sn2700-t1-lag,vms1-1,t1-lag,docker-ptf,ptf-unknown,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests Mellanox SN2700 vms -vms-sn2700-t0,vms1-1,t0,docker-ptf,ptf-unknown,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests Mellanox SN2700 vms -vms-s6000-t0,vms2-1,t0,docker-ptf,ptf-unknown,10.255.0.179/24,,server_1,VM0100,lab-s6000-01,lab,True,Tests Dell S6000 vms -vms-a7260-t0,vms3-1,t0-116,docker-ptf,ptf-unknown,10.255.0.180/24,,server_1,VM0100,lab-a7260-01,lab,True,Tests Arista A7260 vms -vms-s6100-t0,vms4-1,t0-64,docker-ptf,ptf-unknown,10.255.0.181/24,,server_1,VM0100,lab-s6100-01,lab,True,Tests Dell S6100 vms -vms-s6100-t1,vms4-1,t1-64,docker-ptf,ptf-unknown,10.255.0.182/24,,server_1,VM0100,lab-s6100-01,lab,True,Tests Dell S6100 vms -vms-s6100-t1-lag,vms5-1,t1-64-lag,docker-ptf,ptf-unknown,10.255.0.183/24,,server_1,VM0100,lab-s6100-01,lab,True,ests Dell S6100 vms -vms-multi-dut,vms1-duts,ptf64,docker-ptf,ptf-unknown,10.255.0.184/24,,server_1,VM0100,[dut-host1;dut-host2],lab,True,Example Multi DUTs testbed +ptf1-m,ptf1,ptf32,docker-ptf,ptf_ptf1,10.255.0.188/24,,server_1,,str-msn2700-01,lab,False,Test ptf Mellanox +ptf2-b,ptf2,ptf64,docker-ptf,ptf_ptf2,10.255.0.189/24,,server_1,,lab-s6100-01,lab,False,Test ptf Broadcom +vms-sn2700-t1,vms1-1,t1,docker-ptf,ptf_vms1-1,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests Mellanox SN2700 vms +vms-sn2700-t1-lag,vms1-2,t1-lag,docker-ptf,ptf_vms1-2,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests Mellanox SN2700 vms +vms-sn2700-t0,vms1-3,t0,docker-ptf,ptf_vms1-3,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests Mellanox SN2700 vms +vms-s6000-t0,vms2-1,t0,docker-ptf,ptf_vms2-1,10.255.0.179/24,,server_1,VM0100,lab-s6000-01,lab,True,Tests Dell S6000 vms +vms-a7260-t0,vms3-1,t0-116,docker-ptf,ptf_vms3-1,10.255.0.180/24,,server_1,VM0100,lab-a7260-01,lab,True,Tests Arista A7260 vms +vms-s6100-t0,vms4-1,t0-64,docker-ptf,ptf_vms4-1,10.255.0.181/24,,server_1,VM0100,lab-s6100-01,lab,True,Tests Dell S6100 vms +vms-s6100-t1,vms4-2,t1-64,docker-ptf,ptf_vms4-2,10.255.0.182/24,,server_1,VM0100,lab-s6100-01,lab,True,Tests Dell S6100 vms +vms-s6100-t1-lag,vms5-1,t1-64-lag,docker-ptf,ptf_vms5-1,10.255.0.183/24,,server_1,VM0100,lab-s6100-01,lab,True,ests Dell S6100 vms +vms-multi-dut,vms1-duts,ptf64,docker-ptf,ptf_vms1-duts,10.255.0.184/24,,server_1,VM0100,[dut-host1;dut-host2],lab,True,Example Multi DUTs testbed vms-example-ixia-1,vms6-1,t0-64,docker-ptf-ixia,example-ixia-ptf-1,10.0.0.30/32,,server_6,VM0600,example-s6100-dut-1,lab,True,superman -ixanvl-vs-conf,anvl,ptf32,docker-ptf-anvl,ptf-unknown,10.250.0.100/24,,server_1,,vlab-01,lab,True,Test ptf ANVL SONIC VM +ixanvl-vs-conf,anvl,ptf32,docker-ptf-anvl,ptf_anvl,10.250.0.100/24,,server_1,,vlab-01,lab,True,Test ptf ANVL SONIC VM +vms-snappi-sonic,vms6-1,ptf64,docker-ptf-snappi,snappi-sonic-ptf,10.251.0.232,,Server_6,,sonic-s6100-dut1,snappi-sonic,True,Batman +vms-snappi-sonic-multidut,vms6-1,ptf64,docker-ptf-snappi,snappi-sonic-ptf,10.251.0.232,,Server_6,,[sonic-s6100-dut1;sonic-s6100-dut2],snappi-sonic,True,Batman diff --git a/ansible/testbed.yaml b/ansible/testbed.yaml index c3dad3f36b..d318e0f6f0 100644 --- a/ansible/testbed.yaml +++ b/ansible/testbed.yaml @@ -4,9 +4,9 @@ group-name: ptf1 topo: ptf32 ptf_image_name: docker-ptf - ptf: ptf-unknown + ptf: ptf_ptf1 ptf_ip: 10.255.0.188/24 - ptf_ipv6: + ptf_ipv6: 2001:db8:1::1/64 server: server_1 vm_base: dut: @@ -19,9 +19,9 @@ group-name: ptf2 topo: ptf64 ptf_image_name: docker-ptf - ptf: ptf-unknown + ptf: ptf_ptf2 ptf_ip: 10.255.0.189/24 - ptf_ipv6: + ptf_ipv6: 2001:db8:1::2/64 server: server_1 vm_base: dut: @@ -34,9 +34,9 @@ group-name: vms1-1 topo: t1 ptf_image_name: docker-ptf - ptf: ptf-unknown + ptf: ptf_vms1-1 ptf_ip: 10.255.0.178/24 - ptf_ipv6: + ptf_ipv6: 2001:db8:1::3/64 ptf_extra_mgmt_ip: [] server: server_1 vm_base: VM0100 @@ -47,11 +47,11 @@ comment: Tests Mellanox SN2700 vms - conf-name: vms-sn2700-t1-lag - group-name: vms1-1 + group-name: vms1-2 topo: t1-lag ptf_image_name: docker-ptf - ptf: ptf-unknown - ptf_ip: 10.255.0.178/24 + ptf: ptf_vms1-2 + ptf_ip: 10.255.0.179/24 ptf_ipv6: server: server_1 vm_base: VM0100 @@ -62,11 +62,11 @@ comment: Tests Mellanox SN2700 vms - conf-name: vms-sn2700-t0 - group-name: vms1-1 + group-name: vms1-3 topo: t0 ptf_image_name: docker-ptf - ptf: ptf-unknown - ptf_ip: 10.255.0.178/24 + ptf: ptf_vms1-3 + ptf_ip: 10.255.0.180/24 ptf_ipv6: server: server_1 vm_base: VM0100 @@ -80,8 +80,8 @@ group-name: vms2-1 topo: t0 ptf_image_name: docker-ptf - ptf: ptf-unknown - ptf_ip: 10.255.0.179/24 + ptf: ptf_vms2-1 + ptf_ip: 10.255.1.179/24 ptf_ipv6: server: server_1 vm_base: VM0100 @@ -95,8 +95,8 @@ group-name: vms3-1 topo: t0-116 ptf_image_name: docker-ptf - ptf: ptf-unknown - ptf_ip: 10.255.0.180/24 + ptf: ptf_vms3-1 + ptf_ip: 10.255.2.180/24 ptf_ipv6: server: server_1 vm_base: VM0100 @@ -110,7 +110,7 @@ group-name: vms4-1 topo: t0-64 ptf_image_name: docker-ptf - ptf: ptf-unknown + ptf: ptf_vms4-1 ptf_ip: 10.255.0.181/24 ptf_ipv6: server: server_1 @@ -122,10 +122,10 @@ comment: Tests Dell S6100 vms - conf-name: vms-s6100-t1 - group-name: vms4-1 + group-name: vms4-2 topo: t1-64 ptf_image_name: docker-ptf - ptf: ptf-unknown + ptf: ptf_vms4-2 ptf_ip: 10.255.0.182/24 ptf_ipv6: server: server_1 @@ -140,7 +140,7 @@ group-name: vms5-1 topo: t1-64-lag ptf_image_name: docker-ptf - ptf: ptf-unknown + ptf: ptf_vms5-1 ptf_ip: 10.255.0.183/24 ptf_ipv6: server: server_1 @@ -155,7 +155,7 @@ group-name: vms1-duts topo: ptf64 ptf_image_name: docker-ptf - ptf: ptf-unknown + ptf: ptf_vms1-duts ptf_ip: 10.255.0.184/24 ptf_ipv6: server: server_1 @@ -186,7 +186,7 @@ group-name: anvl topo: ptf32 ptf_image_name: docker-ptf-anvl - ptf: ptf-unknown + ptf: ptf_anvl ptf_ip: 10.250.0.100/24 ptf_ipv6: server: server_1 @@ -201,7 +201,7 @@ group-name: vms-dummy-dut topo: t2 ptf_image_name: docker-ptf - ptf: ptf-unknown + ptf: ptf_vms-dummy-dut ptf_ip: 1.1.1.1/23 ptf_ipv6: server: dummy_1 @@ -212,3 +212,34 @@ - lab-msft-lc2-1 - lab-msft-sup-1 comment: Chasiss Testbed + +- conf-name: vms-snappi-sonic + group-name: vms6-1 + topo: ptf64 + ptf_image_name: docker-ptf-snappi + ptf: snappi-sonic-ptf + ptf_ip: 10.251.0.232 + ptf_ipv6: + server: Server_6 + vm_base: + dut: + - sonic-s6100-dut1 + inv_name: snappi-sonic + auto_recover: 'True' + comment: Batman + +- conf-name: vms-snappi-sonic-multidut + group-name: vms6-1 + topo: ptf64 + ptf_image_name: docker-ptf-snappi + ptf: snappi-sonic-ptf + ptf_ip: 10.251.0.232 + ptf_ipv6: + server: Server_6 + vm_base: + dut: + - sonic-s6100-dut1 + - sonic-s6100-dut2 + inv_name: snappi-sonic + auto_recover: 'True' + comment: Batman diff --git a/ansible/testbed_start_VMs.yml b/ansible/testbed_start_VMs.yml index 26d96d837f..449dec9547 100644 --- a/ansible/testbed_start_VMs.yml +++ b/ansible/testbed_start_VMs.yml @@ -17,8 +17,6 @@ - hosts: servers:&vm_host gather_facts: no - vars_files: - - vars/azure_storage.yml tasks: roles: - { role: vm_set, action: 'start' } diff --git a/ansible/testbed_start_k8s_VMs.yml b/ansible/testbed_start_k8s_VMs.yml index 974f4a5da9..001fa2dd91 100644 --- a/ansible/testbed_start_k8s_VMs.yml +++ b/ansible/testbed_start_k8s_VMs.yml @@ -5,7 +5,6 @@ - hosts: k8s_servers:&k8s_vm_host gather_facts: no vars_files: - - vars/azure_storage.yml - group_vars/all/creds.yml roles: - { role: vm_set, action: 'start_k8s' } diff --git a/ansible/vars/azure_storage.yml b/ansible/vars/azure_storage.yml index 1d56022581..3f00283213 100644 --- a/ansible/vars/azure_storage.yml +++ b/ansible/vars/azure_storage.yml @@ -1,6 +1,5 @@ ## saskey are treated as credential in Credscan vmimage_saskey: use_own_value cdimage_saskey: use_own_value -ceosimage_saskey: use_own_value k8s_vmimage_saskey: use_own_value vciscoimage_saskey: use_own_value diff --git a/ansible/vars/topo_dualtor-64-breakout.yml b/ansible/vars/topo_dualtor-64-breakout.yml new file mode 100644 index 0000000000..18208b1c99 --- /dev/null +++ b/ansible/vars/topo_dualtor-64-breakout.yml @@ -0,0 +1,359 @@ +topology: + dut_num: 2 + host_interfaces: + - 0.0@0,1.0@0 + - 0.1@1,1.1@1 + - 0.2@2,1.2@2 + - 0.3@3,1.3@3 + - 0.4@4,1.4@4 + - 0.5@5,1.5@5 + - 0.6@6,1.6@6 + - 0.7@7,1.7@7 + - 0.8@8,1.8@8 + - 0.9@9,1.9@9 + - 0.10@10,1.10@10 + - 0.11@11,1.11@11 + - 0.12@12,1.12@12 + - 0.13@13,1.13@13 + - 0.14@14,1.14@14 + - 0.15@15,1.15@15 + - 0.16@16,1.16@16 + - 0.17@17,1.17@17 + - 0.18@18,1.18@18 + - 0.19@19,1.19@19 + - 0.20@20,1.20@20 + - 0.21@21,1.21@21 + - 0.22@22,1.22@22 + - 0.23@23,1.23@23 + - 0.40@40,1.40@40 + - 0.41@41,1.41@41 + - 0.42@42,1.42@42 + - 0.43@43,1.43@43 + - 0.44@44,1.44@44 + - 0.45@45,1.45@45 + - 0.46@46,1.46@46 + - 0.47@47,1.47@47 + - 0.48@48,1.48@48 + - 0.49@49,1.49@49 + - 0.50@50,1.50@50 + - 0.51@51,1.51@51 + - 0.52@52,1.52@52 + - 0.53@53,1.53@53 + - 0.54@54,1.54@54 + - 0.55@55,1.55@55 + - 0.56@56,1.56@56 + - 0.57@57,1.57@57 + - 0.58@58,1.58@58 + - 0.59@59,1.59@59 + - 0.60@60,1.60@60 + - 0.61@61,1.61@61 + - 0.62@62,1.62@62 + - 0.63@63,1.63@63 + disabled_host_interfaces: + - 0.1@1,1.1@1 + - 0.3@3,1.3@3 + - 0.5@5,1.5@5 + - 0.7@7,1.7@7 + - 0.9@9,1.9@9 + - 0.11@11,1.11@11 + - 0.13@13,1.13@13 + - 0.15@15,1.15@15 + - 0.17@17,1.17@17 + - 0.19@19,1.19@19 + - 0.21@21,1.21@21 + - 0.23@23,1.23@23 + - 0.41@41,1.41@41 + - 0.43@43,1.43@43 + - 0.45@45,1.45@45 + - 0.47@47,1.47@47 + - 0.49@49,1.49@49 + - 0.51@51,1.51@51 + - 0.53@53,1.53@53 + - 0.55@55,1.55@55 + - 0.57@57,1.57@57 + - 0.59@59,1.59@59 + - 0.61@61,1.61@61 + - 0.62@62,1.62@62 + - 0.63@63,1.63@63 + VMs: + ARISTA01T1: + vlans: + - "0.24@64" + - "0.26@65" + - "1.24@66" + - "1.26@67" + vm_offset: 0 + ARISTA02T1: + vlans: + - "0.28@68" + - "0.30@69" + - "1.28@70" + - "1.30@71" + vm_offset: 1 + ARISTA03T1: + vlans: + - "0.32@72" + - "0.34@73" + - "1.32@74" + - "1.34@75" + vm_offset: 2 + ARISTA04T1: + vlans: + - "0.36@76" + - "0.38@77" + - "1.36@78" + - "1.38@79" + vm_offset: 3 + DUT: + loopback: + ipv4: + - 10.1.0.32/32 + - 10.1.0.33/32 + ipv6: + - FC00:1:0:32::/128 + - FC00:1:0:33::/128 + loopback1: + ipv4: + - 10.1.0.34/32 + - 10.1.0.35/32 + ipv6: + - FC00:1:0:34::/128 + - FC00:1:0:35::/128 + loopback2: + ipv4: + - 10.1.0.36/32 + - 10.1.0.36/32 + ipv6: + - FC00:1:0:36::/128 + - FC00:1:0:36::/128 + loopback3: + ipv4: + - 10.1.0.38/32 + - 10.1.0.39/32 + ipv6: + - FC00:1:0:38::/128 + - FC00:1:0:39::/128 + vlan_configs: + default_vlan_config: one_vlan_a + one_vlan_a: + Vlan1000: + id: 1000 + intfs: [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60] + prefix: 192.168.0.1/21 + prefix_v6: fc02:1000::1/64 + tag: 1000 + mac: 00:aa:bb:cc:dd:ee + two_vlan_a: + Vlan100: + id: 100 + intfs: [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22] + prefix: 192.168.0.1/22 + prefix_v6: fc02:100::1/64 + tag: 100 + Vlan200: + id: 200 + intfs: [40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60] + prefix: 192.168.4.1/22 + prefix_v6: fc02:200::1/64 + tag: 200 + four_vlan_a: + Vlan1000: + id: 1000 + intfs: [0, 2, 4, 6, 8, 10] + prefix: 192.168.0.1/23 + prefix_v6: fc02:400::1/64 + tag: 1000 + Vlan2000: + id: 2000 + intfs: [12, 14, 16, 18, 20, 22] + prefix: 192.168.2.1/23 + prefix_v6: fc02:401::1/64 + tag: 2000 + Vlan3000: + id: 3000 + intfs: [40, 42, 44, 46, 48, 50] + prefix: 192.168.4.1/23 + prefix_v6: fc02:402::1/64 + tag: 3000 + Vlan4000: + id: 4000 + intfs: [52, 54, 56, 58, 60] + prefix: 192.168.6.1/23 + prefix_v6: fc02:403::1/64 + tag: 4000 + tunnel_configs: + default_tunnel_config: tunnel_ipinip + tunnel_ipinip: + MuxTunnel0: + type: IPInIP + attach_to: Loopback0 + dscp: uniform + ecn_encap: standard + ecn_decap: copy_from_outer + ttl_mode: pipe + +configuration_properties: + common: + dut_asn: 65100 + dut_type: ToRRouter + swrole: leaf + nhipv4: 10.10.246.254 + nhipv6: FC0A::FF + podset_number: 200 + tor_number: 16 + tor_subnet_number: 2 + max_tor_subnet_number: 16 + tor_subnet_size: 128 + spine_asn: 65534 + leaf_asn_start: 64600 + tor_asn_start: 65500 + failure_rate: 0 + +configuration: + ARISTA01T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.56 + - FC00::71 + - 10.0.1.56 + - FC00::1:71 + interfaces: + Loopback0: + ipv4: 100.1.0.29/32 + ipv6: 2064:100::1d/128 + Ethernet1: + lacp: 1 + dut_index: 0 + Ethernet2: + lacp: 1 + dut_index: 0 + Ethernet3: + lacp: 2 + dut_index: 1 + Ethernet4: + lacp: 2 + dut_index: 1 + Port-Channel1: + ipv4: 10.0.0.57/31 + ipv6: fc00::72/126 + Port-Channel2: + ipv4: 10.0.1.57/31 + ipv6: fc00::1:72/126 + bp_interface: + ipv4: 10.10.246.29/24 + ipv6: fc0a::1d/64 + + ARISTA02T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.58 + - FC00::75 + - 10.0.1.58 + - FC00::1:75 + interfaces: + Loopback0: + ipv4: 100.1.0.30/32 + ipv6: 2064:100::1e/128 + Ethernet1: + lacp: 1 + dut_index: 0 + Ethernet2: + lacp: 1 + dut_index: 0 + Ethernet3: + lacp: 2 + dut_index: 1 + Ethernet4: + lacp: 2 + dut_index: 1 + Port-Channel1: + ipv4: 10.0.0.59/31 + ipv6: fc00::76/126 + Port-Channel2: + ipv4: 10.0.1.59/31 + ipv6: fc00::1:76/126 + bp_interface: + ipv4: 10.10.246.30/24 + ipv6: fc0a::1e/64 + + ARISTA03T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.60 + - FC00::79 + - 10.0.1.60 + - FC00::1:79 + interfaces: + Loopback0: + ipv4: 100.1.0.31/32 + ipv6: 2064:100::1f/128 + Ethernet1: + lacp: 1 + dut_index: 0 + Ethernet2: + lacp: 1 + dut_index: 0 + Ethernet3: + lacp: 2 + dut_index: 1 + Ethernet4: + lacp: 2 + dut_index: 1 + Port-Channel1: + ipv4: 10.0.0.61/31 + ipv6: fc00::7a/126 + Port-Channel2: + ipv4: 10.0.1.61/31 + ipv6: fc00::1:7a/126 + bp_interface: + ipv4: 10.10.246.31/24 + ipv6: fc0a::1f/64 + + ARISTA04T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.62 + - FC00::7D + - 10.0.1.62 + - FC00::1:7D + interfaces: + Loopback0: + ipv4: 100.1.0.32/32 + ipv6: 2064:100::20/128 + Ethernet1: + lacp: 1 + dut_index: 0 + Ethernet2: + lacp: 1 + dut_index: 0 + Ethernet3: + lacp: 2 + dut_index: 1 + Ethernet4: + lacp: 2 + dut_index: 1 + Port-Channel1: + ipv4: 10.0.0.63/31 + ipv6: fc00::7e/126 + Port-Channel2: + ipv4: 10.0.1.63/31 + ipv6: fc00::1:7e/126 + bp_interface: + ipv4: 10.10.246.32/24 + ipv6: fc0a::20/64 diff --git a/ansible/vars/topo_dualtor-aa-64-breakout.yml b/ansible/vars/topo_dualtor-aa-64-breakout.yml new file mode 100644 index 0000000000..8531215b33 --- /dev/null +++ b/ansible/vars/topo_dualtor-aa-64-breakout.yml @@ -0,0 +1,384 @@ +topology: + dut_num: 2 + host_interfaces: + - 0.0@0,1.0@0 + - 0.1@1,1.1@1 + - 0.2@2,1.2@2 + - 0.3@3,1.3@3 + - 0.4@4,1.4@4 + - 0.5@5,1.5@5 + - 0.6@6,1.6@6 + - 0.7@7,1.7@7 + - 0.8@8,1.8@8 + - 0.9@9,1.9@9 + - 0.10@10,1.10@10 + - 0.11@11,1.11@11 + - 0.12@12,1.12@12 + - 0.13@13,1.13@13 + - 0.14@14,1.14@14 + - 0.15@15,1.15@15 + - 0.16@16,1.16@16 + - 0.17@17,1.17@17 + - 0.18@18,1.18@18 + - 0.19@19,1.19@19 + - 0.20@20,1.20@20 + - 0.21@21,1.21@21 + - 0.22@22,1.22@22 + - 0.23@23,1.23@23 + - 0.40@40,1.40@40 + - 0.41@41,1.41@41 + - 0.42@42,1.42@42 + - 0.43@43,1.43@43 + - 0.44@44,1.44@44 + - 0.45@45,1.45@45 + - 0.46@46,1.46@46 + - 0.47@47,1.47@47 + - 0.48@48,1.48@48 + - 0.49@49,1.49@49 + - 0.50@50,1.50@50 + - 0.51@51,1.51@51 + - 0.52@52,1.52@52 + - 0.53@53,1.53@53 + - 0.54@54,1.54@54 + - 0.55@55,1.55@55 + - 0.56@56,1.56@56 + - 0.57@57,1.57@57 + - 0.58@58,1.58@58 + - 0.59@59,1.59@59 + - 0.60@60,1.60@60 + - 0.61@61,1.61@61 + - 0.62@62,1.62@62 + - 0.63@63,1.63@63 + disabled_host_interfaces: + - 0.1@1,1.1@1 + - 0.3@3,1.3@3 + - 0.5@5,1.5@5 + - 0.7@7,1.7@7 + - 0.9@9,1.9@9 + - 0.11@11,1.11@11 + - 0.13@13,1.13@13 + - 0.15@15,1.15@15 + - 0.17@17,1.17@17 + - 0.19@19,1.19@19 + - 0.21@21,1.21@21 + - 0.23@23,1.23@23 + - 0.41@41,1.41@41 + - 0.43@43,1.43@43 + - 0.45@45,1.45@45 + - 0.47@47,1.47@47 + - 0.49@49,1.49@49 + - 0.51@51,1.51@51 + - 0.53@53,1.53@53 + - 0.55@55,1.55@55 + - 0.57@57,1.57@57 + - 0.59@59,1.59@59 + - 0.61@61,1.61@61 + - 0.62@62,1.62@62 + - 0.63@63,1.63@63 + host_interfaces_active_active: + - 0.0@0,1.0@0 + - 0.2@2,1.2@2 + - 0.4@4,1.4@4 + - 0.6@6,1.6@6 + - 0.8@8,1.8@8 + - 0.10@10,1.10@10 + - 0.12@12,1.12@12 + - 0.14@14,1.14@14 + - 0.16@16,1.16@16 + - 0.18@18,1.18@18 + - 0.20@20,1.20@20 + - 0.22@22,1.22@22 + - 0.40@40,1.40@40 + - 0.42@42,1.42@42 + - 0.44@44,1.44@44 + - 0.46@46,1.46@46 + - 0.48@48,1.48@48 + - 0.50@50,1.50@50 + - 0.52@52,1.52@52 + - 0.54@54,1.54@54 + - 0.56@56,1.56@56 + - 0.58@58,1.58@58 + - 0.60@60,1.60@60 + VMs: + ARISTA01T1: + vlans: + - "0.24@64" + - "0.26@65" + - "1.24@66" + - "1.26@67" + vm_offset: 0 + ARISTA02T1: + vlans: + - "0.28@68" + - "0.30@69" + - "1.28@70" + - "1.30@71" + vm_offset: 1 + ARISTA03T1: + vlans: + - "0.32@72" + - "0.34@73" + - "1.32@74" + - "1.34@75" + vm_offset: 2 + ARISTA04T1: + vlans: + - "0.36@76" + - "0.38@77" + - "1.36@78" + - "1.38@79" + vm_offset: 3 + DUT: + loopback: + ipv4: + - 10.1.0.32/32 + - 10.1.0.33/32 + ipv6: + - FC00:1:0:32::/128 + - FC00:1:0:33::/128 + loopback1: + ipv4: + - 10.1.0.34/32 + - 10.1.0.35/32 + ipv6: + - FC00:1:0:34::/128 + - FC00:1:0:35::/128 + loopback2: + ipv4: + - 10.1.0.36/32 + - 10.1.0.36/32 + ipv6: + - FC00:1:0:36::/128 + - FC00:1:0:36::/128 + loopback3: + ipv4: + - 10.1.0.38/32 + - 10.1.0.39/32 + ipv6: + - FC00:1:0:38::/128 + - FC00:1:0:39::/128 + vlan_configs: + default_vlan_config: one_vlan_a + one_vlan_a: + Vlan1000: + id: 1000 + intfs: [ 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60 ] + prefix: 192.168.0.1/21 + prefix_v6: fc02:1000::1/64 + tag: 1000 + mac: 00:aa:bb:cc:dd:ee + two_vlan_a: + Vlan100: + id: 100 + intfs: [ 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22 ] + prefix: 192.168.0.1/22 + prefix_v6: fc02:100::1/64 + tag: 100 + Vlan200: + id: 200 + intfs: [ 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60 ] + prefix: 192.168.4.1/22 + prefix_v6: fc02:200::1/64 + tag: 200 + four_vlan_a: + Vlan1000: + id: 1000 + intfs: [ 0, 2, 4, 6, 8, 10 ] + prefix: 192.168.0.1/23 + prefix_v6: fc02:400::1/64 + tag: 1000 + Vlan2000: + id: 2000 + intfs: [ 12, 14, 16, 18, 20, 22 ] + prefix: 192.168.2.1/23 + prefix_v6: fc02:401::1/64 + tag: 2000 + Vlan3000: + id: 3000 + intfs: [ 40, 42, 44, 46, 48, 50 ] + prefix: 192.168.4.1/23 + prefix_v6: fc02:402::1/64 + tag: 3000 + Vlan4000: + id: 4000 + intfs: [ 52, 54, 56, 58, 60 ] + prefix: 192.168.6.1/23 + prefix_v6: fc02:403::1/64 + tag: 4000 + tunnel_configs: + default_tunnel_config: tunnel_ipinip + tunnel_ipinip: + MuxTunnel0: + type: IPInIP + attach_to: Loopback0 + dscp: uniform + ecn_encap: standard + ecn_decap: copy_from_outer + ttl_mode: pipe + +configuration_properties: + common: + dut_asn: 65100 + dut_type: ToRRouter + swrole: leaf + nhipv4: 10.10.246.254 + nhipv6: FC0A::FF + podset_number: 200 + tor_number: 16 + tor_subnet_number: 2 + max_tor_subnet_number: 16 + tor_subnet_size: 128 + spine_asn: 65534 + leaf_asn_start: 64600 + tor_asn_start: 65500 + failure_rate: 0 + + +configuration: + ARISTA01T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.56 + - FC00::71 + - 10.0.1.56 + - FC00::1:71 + interfaces: + Loopback0: + ipv4: 100.1.0.29/32 + ipv6: 2064:100::1d/128 + Ethernet1: + lacp: 1 + dut_index: 0 + Ethernet2: + lacp: 1 + dut_index: 0 + Ethernet3: + lacp: 2 + dut_index: 1 + Ethernet4: + lacp: 2 + dut_index: 1 + Port-Channel1: + ipv4: 10.0.0.57/31 + ipv6: fc00::72/126 + Port-Channel2: + ipv4: 10.0.1.57/31 + ipv6: fc00::1:72/126 + bp_interface: + ipv4: 10.10.246.29/24 + ipv6: fc0a::1d/64 + + ARISTA02T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.58 + - FC00::75 + - 10.0.1.58 + - FC00::1:75 + interfaces: + Loopback0: + ipv4: 100.1.0.30/32 + ipv6: 2064:100::1e/128 + Ethernet1: + lacp: 1 + dut_index: 0 + Ethernet2: + lacp: 1 + dut_index: 0 + Ethernet3: + lacp: 2 + dut_index: 1 + Ethernet4: + lacp: 2 + dut_index: 1 + Port-Channel1: + ipv4: 10.0.0.59/31 + ipv6: fc00::76/126 + Port-Channel2: + ipv4: 10.0.1.59/31 + ipv6: fc00::1:76/126 + bp_interface: + ipv4: 10.10.246.30/24 + ipv6: fc0a::1e/64 + + ARISTA03T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.60 + - FC00::79 + - 10.0.1.60 + - FC00::1:79 + interfaces: + Loopback0: + ipv4: 100.1.0.31/32 + ipv6: 2064:100::1f/128 + Ethernet1: + lacp: 1 + dut_index: 0 + Ethernet2: + lacp: 1 + dut_index: 0 + Ethernet3: + lacp: 2 + dut_index: 1 + Ethernet4: + lacp: 2 + dut_index: 1 + Port-Channel1: + ipv4: 10.0.0.61/31 + ipv6: fc00::7a/126 + Port-Channel2: + ipv4: 10.0.1.61/31 + ipv6: fc00::1:7a/126 + bp_interface: + ipv4: 10.10.246.31/24 + ipv6: fc0a::1f/64 + + ARISTA04T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.62 + - FC00::7D + - 10.0.1.62 + - FC00::1:7D + interfaces: + Loopback0: + ipv4: 100.1.0.32/32 + ipv6: 2064:100::20/128 + Ethernet1: + lacp: 1 + dut_index: 0 + Ethernet2: + lacp: 1 + dut_index: 0 + Ethernet3: + lacp: 2 + dut_index: 1 + Ethernet4: + lacp: 2 + dut_index: 1 + Port-Channel1: + ipv4: 10.0.0.63/31 + ipv6: fc00::7e/126 + Port-Channel2: + ipv4: 10.0.1.63/31 + ipv6: fc00::1:7e/126 + bp_interface: + ipv4: 10.10.246.32/24 + ipv6: fc0a::20/64 diff --git a/ansible/vars/topo_ptp-256.yml b/ansible/vars/topo_ptp-256.yml new file mode 100644 index 0000000000..c042a5ab63 --- /dev/null +++ b/ansible/vars/topo_ptp-256.yml @@ -0,0 +1,270 @@ +topology: + host_interfaces: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + - 24 + - 25 + - 26 + - 27 + - 28 + - 29 + - 30 + - 31 + - 32 + - 33 + - 34 + - 35 + - 36 + - 37 + - 38 + - 39 + - 40 + - 41 + - 42 + - 43 + - 44 + - 45 + - 46 + - 47 + - 48 + - 49 + - 50 + - 51 + - 52 + - 53 + - 54 + - 55 + - 56 + - 57 + - 58 + - 59 + - 60 + - 61 + - 62 + - 63 + - 64 + - 65 + - 66 + - 67 + - 68 + - 69 + - 70 + - 71 + - 72 + - 73 + - 74 + - 75 + - 76 + - 77 + - 78 + - 79 + - 80 + - 81 + - 82 + - 83 + - 84 + - 85 + - 86 + - 87 + - 88 + - 89 + - 90 + - 91 + - 92 + - 93 + - 94 + - 95 + - 96 + - 97 + - 98 + - 99 + - 100 + - 101 + - 102 + - 103 + - 104 + - 105 + - 106 + - 107 + - 108 + - 109 + - 110 + - 111 + - 112 + - 113 + - 114 + - 115 + - 116 + - 117 + - 118 + - 119 + - 120 + - 121 + - 122 + - 123 + - 124 + - 125 + - 126 + - 127 + - 128 + - 129 + - 130 + - 131 + - 132 + - 133 + - 134 + - 135 + - 136 + - 137 + - 138 + - 139 + - 140 + - 141 + - 142 + - 143 + - 144 + - 145 + - 146 + - 147 + - 148 + - 149 + - 150 + - 151 + - 152 + - 153 + - 154 + - 155 + - 156 + - 157 + - 158 + - 159 + - 160 + - 161 + - 162 + - 163 + - 164 + - 165 + - 166 + - 167 + - 168 + - 169 + - 170 + - 171 + - 172 + - 173 + - 174 + - 175 + - 176 + - 177 + - 178 + - 179 + - 180 + - 181 + - 182 + - 183 + - 184 + - 185 + - 186 + - 187 + - 188 + - 189 + - 190 + - 191 + - 192 + - 193 + - 194 + - 195 + - 196 + - 197 + - 198 + - 199 + - 200 + - 201 + - 202 + - 203 + - 204 + - 205 + - 206 + - 207 + - 208 + - 209 + - 210 + - 211 + - 212 + - 213 + - 214 + - 215 + - 216 + - 217 + - 218 + - 219 + - 220 + - 221 + - 222 + - 223 + - 224 + - 225 + - 226 + - 227 + - 228 + - 229 + - 230 + - 231 + - 232 + - 233 + - 234 + - 235 + - 236 + - 237 + - 238 + - 239 + - 240 + - 241 + - 242 + - 243 + - 244 + - 245 + - 246 + - 247 + - 248 + - 249 + - 250 + - 251 + - 252 + - 253 + - 254 + - 255 + VMs: {} + DUT: + vlan_configs: + default_vlan_config: one_vlan_a + one_vlan_a: {} + +configuration_properties: + common: + dut_asn: 0 + dut_type: ToRRouter + +configuration: {} diff --git a/ansible/vars/topo_t0-28.yml b/ansible/vars/topo_t0-28.yml new file mode 100644 index 0000000000..870eff1227 --- /dev/null +++ b/ansible/vars/topo_t0-28.yml @@ -0,0 +1,199 @@ +topology: + host_interfaces: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + - 21 + - 22 + - 23 + disabled_host_interfaces: + - 0 + VMs: + ARISTA01T1: + vlans: + - 24 + vm_offset: 0 + ARISTA02T1: + vlans: + - 25 + vm_offset: 1 + ARISTA03T1: + vlans: + - 26 + vm_offset: 2 + ARISTA04T1: + vlans: + - 27 + vm_offset: 3 + DUT: + vlan_configs: + default_vlan_config: one_vlan_a + one_vlan_a: + Vlan1000: + id: 1000 + intfs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23] + prefix: 192.168.0.1/21 + prefix_v6: fc02:1000::1/64 + tag: 1000 + two_vlan_a: + Vlan100: + id: 100 + intfs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + prefix: 192.168.0.1/22 + prefix_v6: fc02:100::1/64 + tag: 100 + Vlan200: + id: 200 + intfs: [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23] + prefix: 192.168.4.1/22 + prefix_v6: fc02:200::1/64 + tag: 200 + four_vlan_a: + Vlan1000: + id: 1000 + intfs: [1, 2, 3, 4, 5, 6] + prefix: 192.168.0.1/23 + prefix_v6: fc02:400::1/64 + tag: 1000 + Vlan2000: + id: 2000 + intfs: [7, 8, 9, 10, 11, 12] + prefix: 192.168.2.1/23 + prefix_v6: fc02:401::1/64 + tag: 2000 + Vlan3000: + id: 3000 + intfs: [13, 14, 15, 16, 17, 18] + prefix: 192.168.4.1/23 + prefix_v6: fc02:402::1/64 + tag: 3000 + Vlan4000: + id: 4000 + intfs: [19, 20, 21, 22, 23] + prefix: 192.168.6.1/23 + prefix_v6: fc02:403::1/64 + tag: 4000 + +configuration_properties: + common: + dut_asn: 65100 + dut_type: ToRRouter + swrole: leaf + nhipv4: 10.10.246.254 + nhipv6: FC0A::FF + podset_number: 200 + tor_number: 16 + tor_subnet_number: 2 + max_tor_subnet_number: 16 + tor_subnet_size: 128 + spine_asn: 65534 + leaf_asn_start: 64600 + tor_asn_start: 65500 + failure_rate: 0 + +configuration: + ARISTA01T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.56 + - FC00::71 + interfaces: + Loopback0: + ipv4: 100.1.0.29/32 + ipv6: 2064:100::1d/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 10.0.0.57/31 + ipv6: fc00::72/126 + bp_interface: + ipv4: 10.10.246.29/24 + ipv6: fc0a::1d/64 + + ARISTA02T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.58 + - FC00::75 + interfaces: + Loopback0: + ipv4: 100.1.0.30/32 + ipv6: 2064:100::1e/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 10.0.0.59/31 + ipv6: fc00::76/126 + bp_interface: + ipv4: 10.10.246.30/24 + ipv6: fc0a::1e/64 + + ARISTA03T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.60 + - FC00::79 + interfaces: + Loopback0: + ipv4: 100.1.0.31/32 + ipv6: 2064:100::1f/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 10.0.0.61/31 + ipv6: fc00::7a/126 + bp_interface: + ipv4: 10.10.246.31/24 + ipv6: fc0a::1f/64 + + ARISTA04T1: + properties: + - common + bgp: + asn: 64600 + peers: + 65100: + - 10.0.0.62 + - FC00::7D + interfaces: + Loopback0: + ipv4: 100.1.0.32/32 + ipv6: 2064:100::20/128 + Ethernet1: + lacp: 1 + Port-Channel1: + ipv4: 10.0.0.63/31 + ipv6: fc00::7e/126 + bp_interface: + ipv4: 10.10.246.32/24 + ipv6: fc0a::20/64 diff --git a/ansible/vars/topo_t0-standalone-32.yml b/ansible/vars/topo_t0-standalone-32.yml index 7cd50fe01d..f4341eac1c 100644 --- a/ansible/vars/topo_t0-standalone-32.yml +++ b/ansible/vars/topo_t0-standalone-32.yml @@ -39,7 +39,7 @@ topology: one_vlan_a: Vlan1000: id: 1000 - intfs: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 0, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] + intfs: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] prefix: 192.168.0.1/21 prefix_v6: fc02:1000::1/64 tag: 1000 diff --git a/ansible/vars/topo_t2-ixia-2lc-4.yml b/ansible/vars/topo_t2-ixia-2lc-4.yml new file mode 100644 index 0000000000..e894355cb8 --- /dev/null +++ b/ansible/vars/topo_t2-ixia-2lc-4.yml @@ -0,0 +1,139 @@ +topology: + # 3 DUTs - 2 linecards (dut 0,1) and 1 Supervisor card (dut 2). + # + # - two ports(port32,port33) on dut0 asic2 connected to Ixia - both routed port + # - two ports(port34,port35) on dut1 asic2 connected to Ixia - both routed port + # + # No ptf ports, this is to setup the DUT for ixia testing. + + dut_num: 3 + VMs: + Ixia-Port1: + vlans: + - 0.32@32 + vm_offset: 0 + Ixia-Port2: + vlans: + - 0.33@33 + vm_offset: 1 + Ixia-Port3: + vlans: + - 1.34@34 + vm_offset: 2 + Ixia-Port4: + vlans: + - 1.35@35 + vm_offset: 3 + DUT: + loopback: + ipv4: + - 10.1.0.1/32 + - 10.1.0.2/32 + ipv6: + - FC00:10::1/128 + - FC00:11::1/128 + +configuration_properties: + common: + podset_number: 400 + tor_number: 16 + tor_subnet_number: 8 + max_tor_subnet_number: 32 + tor_subnet_size: 128 + dut_asn: 65100 + dut_type: SpineRouter + nhipv4: 10.10.246.254 + nhipv6: FC0A::FF + core: + swrole: core + leaf: + swrole: leaf + +configuration: + Ixia-Port1: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.1.1.0 + - FC00:1::1 + interfaces: + Loopback0: + ipv4: 100.1.0.1/32 + ipv6: 2064:100::1/128 + Ethernet1: + ipv4: 10.1.1.1/31 + ipv6: FC00:1::2/126 + dut_index: 0 + bp_interface: + ipv4: 10.10.246.1/24 + ipv6: fc0a::2/64 + + Ixia-Port2: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.1.2.0 + - FC00:2::1 + interfaces: + Loopback0: + ipv4: 100.1.0.3/32 + ipv6: 2064:100::3/128 + Ethernet1: + ipv4: 10.1.2.1/31 + ipv6: FC00:2::2/126 + dut_index: 0 + bp_interface: + ipv4: 10.10.246.3/24 + ipv6: fc0a::6/64 + + Ixia-Port3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.1.3.0 + - FC00:3::1 + interfaces: + Loopback0: + ipv4: 100.1.0.4/32 + ipv6: 2064:100::4/128 + Ethernet1: + ipv4: 10.1.3.1/31 + ipv6: FC00:3::2/126 + dut_index: 1 + bp_interface: + ipv4: 10.10.246.4/24 + ipv6: fc0a::9/64 + + Ixia-Port4: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.1.4.0 + - FC00:4::1 + interfaces: + Loopback0: + ipv4: 100.1.0.6/32 + ipv6: 2064:100::6/128 + Ethernet1: + ipv4: 10.1.4.1/31 + ipv6: FC00:4::2/126 + dut_index: 1 + bp_interface: + ipv4: 10.10.246.6/24 + ipv6: fc0a::d/64 diff --git a/ansible/vars/topo_t2-ixia-3lc-4.yml b/ansible/vars/topo_t2-ixia-3lc-4.yml new file mode 100644 index 0000000000..71b4d8df2e --- /dev/null +++ b/ansible/vars/topo_t2-ixia-3lc-4.yml @@ -0,0 +1,166 @@ +topology: + # 4 DUTs - 3 linecards (dut 0,1,2) and 1 Supervisor card (dut 2). + # + # - two ports(port32,port33) on dut0 asic2 connected to Ixia - both routed port + # - two ports(port34,port35) on dut1 asic2 connected to Ixia - both routed port + # + # One ptf port on dut2 to avoid empty minigraph being generated for dut2 + + dut_num: 4 + VMs: + Ixia-Port1: + vlans: + - 0.32@0 + vm_offset: 0 + Ixia-Port2: + vlans: + - 0.33@1 + vm_offset: 1 + Ixia-Port3: + vlans: + - 1.34@2 + vm_offset: 2 + Ixia-Port4: + vlans: + - 1.35@3 + vm_offset: 3 + ARISTA64T1: + vlans: + - 2.31@95 + vm_offset: 71 + DUT: + loopback: + ipv4: + - 10.1.0.1/32 + - 10.1.0.2/32 + - 10.1.0.3/32 + ipv6: + - FC00:10::1/128 + - FC00:11::1/128 + - FC00:12::1/128 + +configuration_properties: + common: + podset_number: 400 + tor_number: 16 + tor_subnet_number: 8 + max_tor_subnet_number: 32 + tor_subnet_size: 128 + dut_asn: 65100 + dut_type: SpineRouter + nhipv4: 10.10.246.254 + nhipv6: FC0A::FF + core: + swrole: core + leaf: + swrole: leaf + +configuration: + Ixia-Port1: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.1.1.0 + - FC00:1::1 + interfaces: + Loopback0: + ipv4: 100.1.0.1/32 + ipv6: 2064:100::1/128 + Ethernet1: + ipv4: 10.1.1.1/31 + ipv6: FC00:1::2/126 + dut_index: 0 + bp_interface: + ipv4: 10.10.246.1/24 + ipv6: fc0a::2/64 + + Ixia-Port2: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.1.2.0 + - FC00:2::1 + interfaces: + Loopback0: + ipv4: 100.1.0.3/32 + ipv6: 2064:100::3/128 + Ethernet1: + ipv4: 10.1.2.1/31 + ipv6: FC00:2::2/126 + dut_index: 0 + bp_interface: + ipv4: 10.10.246.3/24 + ipv6: fc0a::6/64 + + Ixia-Port3: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.1.3.0 + - FC00:3::1 + interfaces: + Loopback0: + ipv4: 100.1.0.4/32 + ipv6: 2064:100::4/128 + Ethernet1: + ipv4: 10.1.3.1/31 + ipv6: FC00:3::2/126 + dut_index: 1 + bp_interface: + ipv4: 10.10.246.4/24 + ipv6: fc0a::9/64 + + Ixia-Port4: + properties: + - common + - core + bgp: + asn: 65200 + peers: + 65100: + - 10.1.4.0 + - FC00:4::1 + interfaces: + Loopback0: + ipv4: 100.1.0.6/32 + ipv6: 2064:100::6/128 + Ethernet1: + ipv4: 10.1.4.1/31 + ipv6: FC00:4::2/126 + dut_index: 1 + bp_interface: + ipv4: 10.10.246.6/24 + ipv6: fc0a::d/64 + ARISTA64T1: + properties: + - common + - leaf + bgp: + asn: 65048 + peers: + 65100: + - 10.0.0.190 + - FC00::17d + interfaces: + Loopback0: + ipv4: 100.1.0.96/32 + ipv6: 2064:100::60/128 + Ethernet1: + ipv4: 10.0.0.191/31 + ipv6: FC00::17e/126 + dut_index: 2 + bp_interface: + ipv4: 10.10.246.96/24 + ipv6: fc0a::c0/64 diff --git a/ansible/veos_vtb b/ansible/veos_vtb index 16900d8c4e..81e5fb7892 100644 --- a/ansible/veos_vtb +++ b/ansible/veos_vtb @@ -19,6 +19,7 @@ all: - t1-backend - t0 - t0-16 + - t0-28 - t0-56 - t0-56-d48c8 - t0-52 diff --git a/ansible/verify_config.py b/ansible/verify_config.py new file mode 100644 index 0000000000..1af9c63f1e --- /dev/null +++ b/ansible/verify_config.py @@ -0,0 +1,779 @@ +import argparse +import csv +import glob +import logging +import os +import subprocess +from abc import ABC, abstractmethod +from collections import deque +from enum import Enum +from functools import lru_cache + +import yaml +from yaml.parser import ParserError + +""" +Script Usage Guide + +This script is designed to validate our configurations as part of our documentation process. + +NOTE: For now this script only supports yaml format + +General Usage: + +To verify the entire project’s configuration: +- python3 verify_config.py + +This will use the default testbed file (`testbed.yaml`) and the default vm file (`veos`) + +To check the configuration using a specific testbed and VM file: +- python3 verify_config.py -t -m + +Testbed-Specific Usage: + +To confirm connectivity with all Devices Under Test (DUTs) within a single testbed +(Note: This command must be executed within the management container): +- python3 verify_config.py -tb + +This will use the default testbed file (`testbed.yaml`) and the default vm file (`veos`) + +To verify a single testbed’s connectivity using specific testbed and VM files: +- python3 verify_config.py -t -m -tb + +Replace , , and with the actual file names and testbed identifier as required. +""" + + +class Formatting(Enum): + BOLD = "\033[1m" + YELLOW = "\033[33m" + UNDERLINE = '\033[4m' + END = "\033[0m" + RED = "\033[91m" + + @staticmethod + def bold(word): + return f"{Formatting.BOLD.value}{word}{Formatting.END.value}" + + @staticmethod + def yellow(word): + return f"{Formatting.YELLOW.value}{word}{Formatting.END.value}" + + @staticmethod + def red(word): + return f"{Formatting.RED.value}{word}{Formatting.END.value}" + + @staticmethod + def underline(word): + return f"{Formatting.UNDERLINE.value}{word}{Formatting.END.value}" + + +class Formatter(logging.Formatter): + def format(self, record): + if record.levelno == logging.INFO: + self._style._fmt = "%(message)s" + elif record.levelno == logging.WARNING: + self._style._fmt = f"{Formatting.bold(Formatting.yellow('[%(levelname)s]'))}: %(message)s" + elif record.levelno == logging.ERROR: + self._style._fmt = f"{Formatting.bold(Formatting.red('[%(levelname)s]'))}: %(message)s" + return super().format(record) + + +log = logging.getLogger(__name__) +handler = logging.StreamHandler() +handler.setFormatter(Formatter()) +log.setLevel(logging.INFO) +log.addHandler(handler) + + +class VMRange: + def __init__(self, vm_base, topo_name): + self.topo_name = topo_name + self.start = self._parse_start(vm_base) + self.end = self._parse_end(vm_base) + + def _parse_start(self, vm_base): + filtered = ''.join(filter(lambda x: x.isdigit(), vm_base)) + return int(filtered) + + def _parse_end(self, vm_base): + start = self._parse_start(vm_base) + return start + Utility.get_num_vm(self.topo_name) - 1 + + def __contains__(self, vm_range: 'VMRange'): + return not (self.end < vm_range.start or vm_range.end < self.start) + + +class Assertion: + def __init__(self, file): + self.file = file + self.pass_validation = True + self.error_details = deque() + + def assert_true(self, fn, reason): + try: + if not fn(): + self.pass_validation = False + self.log_error(reason, error_file=self.file, error_details=self.error_details) + except ParserError as err: + self.log_error(f"Error parsing yaml file: {err}", error_type="error") + + def add_error_details(self, detail): + self.error_details.append(detail) + + def log_error(self, reason, error_file=None, error_type='warning', error_details=None): + getattr(log, error_type)("{}{}{}".format( + reason, + ". Error file: " + error_file if error_file else "", + ". Details: " if error_details else "", + )) + while error_details: + log.info("\t- " + error_details.popleft()) + + +class Config: + DOCKER_REGISTRY_URL = "sonicdev-microsoft.azurecr.io:443" + DOCKER_IMAGE_NAME = "docker-sonic-mgmt" + DOCKER_REGISTRY_FILE = "vars/docker_registry.yml" + TESTBED_FILE = "testbed.yaml" + TOPO_FILE_PATTERN = "vars/topo*.yml" + VM_FILE = "veos" + FANOUT_LINKS_FILE = "files/sonic_*_links.csv" + FANOUT_DEVICES_FILE = "files/sonic_*_devices.csv" + FANOUT_BMC_LINKS_FILE = "files/sonic_*_bmc_links.csv" + FANOUT_PDU_LINKS_FILE = "files/sonic_*_pdu_links.csv" + FANOUT_CONSOLE_LINKS_FILE = "files/sonic_*_console_links.csv" + FANOUT_GRAPH_GROUP_FILE = "files/graph_groups.yml" + GROUP_VARS_ENV_FILE = "group_vars/all/env.yml" + + +class Utility: + @staticmethod + @lru_cache + def parse_yml(file): + with open(file, "r") as stream: + return yaml.safe_load(stream) + + @staticmethod + @lru_cache + def get_devices_from_links_file(file): + devices = set() + + try: + with open(file, "r") as stream: + for row in csv.DictReader(stream): + devices.add(row['StartDevice']) + devices.add(row['EndDevice']) + except FileNotFoundError: + log.error(f"Cannot find file {file} while getting devices information") + + return devices + + @staticmethod + @lru_cache + def get_devices_from_devices_file(file): + devices = set() + + try: + with open(file, "r") as stream: + for row in csv.DictReader(stream): + devices.add(row['Hostname']) + except FileNotFoundError: + log.error(f"Cannot find file {file} while getting devices information") + + return devices + + @staticmethod + @lru_cache + def get_topo_from_var_files(): + topo_name_set = set() + + for topo_file_path in glob.glob(os.path.abspath(Config.TOPO_FILE_PATTERN)): + file_name, _ = os.path.basename(topo_file_path).split(".") + topo_name = file_name[len("topo_"):] + topo_name_set.add(topo_name) + + return topo_name_set + + @staticmethod + @lru_cache + def get_inv_name_from_file(link_file_pattern): + inv_name_set = set() + + for inv_file_path in glob.glob(os.path.abspath(link_file_pattern)): + file_name, _ = os.path.basename(inv_file_path).split(".") + + inv_name = file_name[len("sonic_"): file_name.index("_", len("sonic_"))] + inv_name_set.add(inv_name) + + return inv_name_set + + @staticmethod + @lru_cache + def get_num_vm(topo_name): + topology = Utility.parse_yml(Config.TOPO_FILE_PATTERN.replace("*", f"_{topo_name}")) + if 'topology' not in topology or 'VMs' not in topology['topology']: + return 0 + + return len(topology['topology']['VMs']) + + @staticmethod + def execute_shell(cmd, is_shell=False): + try: + return subprocess.run( + cmd.split(" "), + shell=is_shell, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ).returncode + except FileNotFoundError: + return 1 + + +class Validator(ABC): + def __init__(self, validate_file=None): + self._file = validate_file + self.assertion = Assertion(self.file) + + @property + def file(self): + return self._file + + @file.setter + def file(self, file_name): + if file_name: + file_name = os.path.abspath(file_name) + self.assertion.file = file_name + self._file = file_name + + @abstractmethod + def validate(self): + pass + + +class DockerRegistryValidator(Validator): + def __init__(self): + super().__init__(Config.DOCKER_REGISTRY_FILE) + + def validate(self): + var = Utility.parse_yml(self.file) + self.assertion.assert_true(lambda: self._is_docker_registry_host_defined(var), + reason=f"Key '{Formatting.red('docker_registry_host')}' must be defined") + + def _is_docker_registry_host_defined(self, var): + required_key = 'docker_registry_host' + return required_key in var and var[required_key] is not None + + +class TestbedValidator(Validator): + def __init__(self): + super().__init__(Config.TESTBED_FILE) + + def validate(self): + conf_name_check_unique = set() + group_name_check = {} + conf_name_to_vm_range = {} + + for testbed in Utility.parse_yml(self.file): + conf_name = testbed['conf-name'] + + self.assertion.assert_true( + lambda: self._required_attributes_must_be_in_testbed(testbed), + reason=f"({Formatting.red(conf_name)}) Required attributes must be in testbed " + ) + + self.assertion.assert_true( + lambda: conf_name not in conf_name_check_unique, + reason=f"({Formatting.red(conf_name)}) Config name '{Formatting.red(conf_name)}' is not unique" + ) + + self.assertion.assert_true( + lambda: len(testbed['group-name']) <= 8, + reason=f"({Formatting.red(conf_name)}) Group name '{Formatting.red(testbed['group-name'])}' " + "must be up to 8 " + f"characters long. Actual length: {len(testbed['group-name'])}", + ) + + self.assertion.assert_true( + lambda: self._group_name_must_have_same_attributes(group_name_check, testbed), + reason=f"({Formatting.red(conf_name)}) The attributes of group name " + f"'{Formatting.red(testbed['group-name'])}'" + "are not consistent with those of other testbeds sharing the same group name.", + ) + self.assertion.assert_true( + lambda: testbed['topo'] in Utility.get_topo_from_var_files(), + reason=f"({Formatting.red(conf_name)}) Topology name '{Formatting.red(testbed['topo'])}' is " + f"not declared in '{Config.TOPO_FILE_PATTERN}'", + ) + + self.assertion.assert_true( + lambda: self._topo_name_must_be_in_vm_file(testbed), + reason=f"({Formatting.red(conf_name)}) Topology name '{Formatting.red(testbed['topo'])}' " + f"is not declared in '{Config.VM_FILE}'", + ) + + self.assertion.assert_true( + lambda: self._server_name_must_be_in_vm_file(testbed), + reason=f"({Formatting.red(conf_name)}) Server name '{Formatting.red(testbed['server'])}' is not " + f"declared in '{Config.VM_FILE}'", + ) + + if 'inv_name' in testbed and testbed['inv_name']: + self.assertion.assert_true( + lambda: self._ptf_information_must_aligns_with_inventory_file(testbed), + reason=f"({Formatting.red(conf_name)}) Ptf '{Formatting.red(testbed['ptf'])}' information does not " + f"align with its inventory file '{testbed['inv_name']}'", + ) + + if testbed['vm_base']: + self.assertion.assert_true( + lambda: Utility.get_num_vm(testbed['topo']) != 0, + reason=f"({Formatting.red(conf_name)}) Topology '{Formatting.red(testbed['topo'])}' " + f"is not declared to have VM in '{Config.TOPO_FILE_PATTERN}' but its VM base " + f"specified as '{testbed['vm_base']}'", + ) + + vm_range = VMRange(testbed['vm_base'], testbed['topo']) + self.assertion.assert_true( + lambda: self._vm_base_must_not_overlap(testbed, vm_range, conf_name_to_vm_range), + reason=f"({Formatting.red(conf_name)}) VM base of '{Formatting.red(conf_name)}' " + "must not overlap with other testbed" + ) + conf_name_to_vm_range[conf_name] = vm_range + + self.assertion.assert_true( + lambda: self._vm_base_must_be_in_the_correct_server(testbed), + reason=f"({Formatting.red(conf_name)}) VM base of '{Formatting.red(conf_name)}' " + "must be in the correct server") + + conf_name_check_unique.add(conf_name) + group_name_check[testbed['group-name']] = {"conf-name": conf_name, "ptf_ip": testbed["ptf_ip"], + "server": testbed["server"], "vm_base": testbed["vm_base"]} + + def _ptf_information_must_aligns_with_inventory_file(self, testbed): + try: + ptfs_information_from_inv_file = Utility.parse_yml(testbed['inv_name'])['all']['children']['ptf']['hosts'] + + if testbed['ptf'] not in ptfs_information_from_inv_file: + self.assertion.add_error_details( + f"Ptf '{Formatting.red(testbed['ptf'])}' is not declared in " + f"inventory file 'ansible/{testbed['inv_name']}'", + ) + return False + + ip, _ = testbed['ptf_ip'].split("/") + + ptf_testbed_information = ptfs_information_from_inv_file[testbed['ptf']] + if ip != ptf_testbed_information['ansible_host']: + self.assertion.add_error_details( + f"ptf_ip is not the same as its inventory file 'ansible/{testbed['inv_name']}' " + f"{Formatting.red(ip)} != {ptf_testbed_information['ansible_host']}", + ) + return False + + if 'ptf_ipv6' in testbed and testbed['ptf_ipv6']: + ipv6, _ = testbed['ptf_ipv6'].split("/") + if 'ansible_hostv6' in ptf_testbed_information and ipv6 != ptf_testbed_information['ansible_hostv6']: + self.assertion.add_error_details( + f"ptf_ipv6 is not the same as its inventory file 'ansible/{testbed['inv_name']}' " + f"{Formatting.red(ipv6)} != {ptf_testbed_information['ansible_hostv6']}", + ) + return False + + return True + + except FileNotFoundError: + self.assertion.log_error( + f"('{Formatting.red(testbed['conf-name'])}') does not have a corresponding inventory file " + f"for '{Formatting.red(testbed['inv_name'])}' consider creating it in " + f"'{Formatting.bold('ansible/' + testbed['inv_name'])}'", + error_file=self.file, + error_type="error", + ) + + def _vm_base_must_be_in_the_correct_server(self, testbed): + vm_base = testbed['vm_base'] + server = testbed['server'] + + vm_configuration = Utility.parse_yml(Config.VM_FILE) + + if server not in vm_configuration: + self.assertion.add_error_details( + f"Server '{Formatting.red(server)}' is not in file '{Config.VM_FILE}'", + ) + return False + + vms_server = next(filter( + lambda config: config.startswith("vms"), vm_configuration[server]['children']), + ) + + if vm_base not in vm_configuration[vms_server]['hosts']: + self.assertion.add_error_details( + f"VM base '{Formatting.red(vm_base)}' is not in server '{server}' from file '{Config.VM_FILE}'", + ) + return False + + return True + + def _vm_base_must_not_overlap(self, testbed, vm_range, conf_name_to_vm_range): + is_valid = True + + for conf_name in conf_name_to_vm_range: + occupied_range = conf_name_to_vm_range[conf_name] + + if vm_range in occupied_range: + self.assertion.add_error_details( + f"VM Range of '{testbed['conf-name']}' (start={vm_range.start}, end={vm_range.end}) is overlap " + f"with '{conf_name}' (start={occupied_range.start}, end={occupied_range.end})") + is_valid = False + + return is_valid + + def _group_name_must_have_same_attributes(self, group_name_check, testbed): + unique_attributes = ["ptf_ip", "server", "vm_base"] + is_valid = True + + group_name = testbed["group-name"] + + if group_name not in group_name_check: + return True + + group_name_attributes = group_name_check[group_name] + for attribute in unique_attributes: + if testbed[attribute] != group_name_attributes[attribute]: + self.assertion.add_error_details( + f"Attribute: {attribute}={testbed[attribute]} is not the same in group name '{group_name}' for " + f"conf-name='{testbed['conf-name']}'. Previously declared as {attribute}=" + f"{group_name_attributes[attribute]} in conf-name={group_name_attributes['conf-name']}") + is_valid = False + + return is_valid + + def _server_name_must_be_in_vm_file(self, testbed): + try: + return testbed['server'] in Utility.parse_yml(Config.VM_FILE)['all']['children']['servers']['children'] + except KeyError as unknown_key: + self.assertion.add_error_details(f"Key not found: {unknown_key}") + self.assertion.log_error( + "vm file is not in the correct format. Update the file or update this script", + error_file=Config.VM_FILE, + error_type="error", + ) + + def _topo_name_must_be_in_vm_file(self, testbed): + try: + topologies_from_file = Utility.parse_yml(Config.VM_FILE)['all']['children']['servers']['vars']['topologies'] + return testbed['topo'] in topologies_from_file + except KeyError as unknown_key: + self.assertion.add_error_details(f"Key not found: {unknown_key}") + self.assertion.log_error( + "vm file is not in the correct format. Update the file or update this script", + error_file=Config.VM_FILE, + error_type="error", + ) + + def _required_attributes_must_be_in_testbed(self, testbed): + is_valid = True + required_attributes = { + "conf-name", + "group-name", + "topo", + "ptf_image_name", + "ptf_ip", + "server", + "vm_base", + "dut", + "inv_name", + "auto_recover", + "comment" + } + + missing_keys = required_attributes - testbed.keys() + + if missing_keys: + self.assertion.add_error_details(f"Found missing required keys: {missing_keys}") + is_valid = False + + return is_valid + + +class InventoryNameValidator(Validator): + def __init__(self): + super().__init__(Config.FANOUT_GRAPH_GROUP_FILE) + + def validate(self): + self.assertion.assert_true(lambda: self._inv_name_from_devices_files_must_be_the_same_as_graph_group_yml_file(), + reason="Inventory name must be consistent between " + f"{Formatting.bold(Config.FANOUT_GRAPH_GROUP_FILE)} and " + f"{Formatting.bold(Config.FANOUT_DEVICES_FILE)}") + self.assertion.assert_true(lambda: self._check_if_inv_name_has_inv_file(), + reason="Inventory should have an inventory file") + + def _inv_name_from_devices_files_must_be_the_same_as_graph_group_yml_file(self): + inv_name_from_devices_files = Utility.get_inv_name_from_file(Config.FANOUT_DEVICES_FILE) + inv_name_from_graph_group_yml_file = set(Utility.parse_yml(self.file)) + + differences = inv_name_from_devices_files ^ inv_name_from_graph_group_yml_file + + if differences: + self.assertion.add_error_details( + "These are the group names that are not consistent between the 2 files: " + f"{Formatting.red(', '.join(differences))}") + return False + + return True + + def _check_if_inv_name_has_inv_file(self): + is_valid = True + inv_name_from_graph_group_yml_file = set(Utility.parse_yml(Config.FANOUT_GRAPH_GROUP_FILE)) + inv_files = set([f for f in os.listdir('.') if os.path.isfile(f)]) + + for inv_name in inv_name_from_graph_group_yml_file: + if inv_name not in inv_files: + is_valid = False + self.assertion.add_error_details( + f"'{Formatting.red(inv_name)}' is declared in " + f"{Formatting.bold(Config.FANOUT_GRAPH_GROUP_FILE)}" + f"but does not have inventory file. Consider creating '{Formatting.bold('ansible/' + inv_name)}'") + return is_valid + + +class FanoutLinkValidator(Validator): + def __init__(self): + super().__init__() + + def validate(self): + params = [ + {"name": "Link file", "file": Config.FANOUT_LINKS_FILE}, + {"name": "PDU link file", "file": Config.FANOUT_PDU_LINKS_FILE}, + {"name": "BMC link file", "file": Config.FANOUT_BMC_LINKS_FILE}, + {"name": "CONSOLE link file", "file": Config.FANOUT_CONSOLE_LINKS_FILE} + ] + + for param in params: + self.file = param["file"] + self.assertion.assert_true(lambda: self._links_file_should_have_equivalent_devices_file(param), + reason=f"{param['name']} should have its equivalent devices file") + self.assertion.assert_true(lambda: self._devices_in_links_file_should_be_in_devices_file(param), + reason=f"{param['name']} devices does not exist in its devices file") + + def _links_file_should_have_equivalent_devices_file(self, param): + inv_name_from_links_files = Utility.get_inv_name_from_file(param["file"]) + inv_name_from_devices_files = Utility.get_inv_name_from_file(Config.FANOUT_DEVICES_FILE) + + differences = inv_name_from_links_files ^ inv_name_from_devices_files + + if differences: + self.assertion.add_error_details( + f"These are the group names that do not have devices file: [{Formatting.red(', '.join(differences))}]. " + "Consider creating " + f"[{', '.join(Formatting.bold(f'files/sonic_{group_name}_devices.csv') for group_name in differences)}]" + ) + return False + + return True + + def _devices_in_links_file_should_be_in_devices_file(self, param): + is_valid = True + inv_name_from_links_files = Utility.get_inv_name_from_file(param["file"]) + + for group_name in inv_name_from_links_files: + device_file = Config.FANOUT_DEVICES_FILE.replace("*", group_name) + link_file = param['file'].replace("*", group_name) + devices_from_links_file = Utility.get_devices_from_links_file(link_file) + devices_from_devices_file = Utility.get_devices_from_devices_file(device_file) + + differences = devices_from_links_file - devices_from_devices_file + + if differences: + self.assertion.add_error_details( + "These are the devices that are in " + f"{Formatting.yellow(param['file'].replace('*', group_name))} " + f"but not in devices file: [{Formatting.red(', '.join(differences))}]. " + f"Consider adding in {Formatting.bold(device_file)}") + is_valid = False + + return is_valid + + +class HostNetworkValidation(Validator): + def __init__(self): + super().__init__() + + def validate(self): + self.assertion.assert_true( + lambda: self._check_if_bridge_is_up(), + reason=f"Interface 'br1' is not up. Consider running " + f"'{Formatting.yellow('./setup-management-network.sh')}'", + ) + + self.assertion.assert_true( + lambda: self._check_if_proxy_is_using(), + reason="Proxy setting is not correct ", + ) + + self.assertion.assert_true( + lambda: self._check_if_can_connect_to_docker_registry(), + reason="Cannot establish a connection to docker registry. Check your proxy setting, docker proxy setting" + ) + + self.assertion.assert_true( + lambda: self._check_if_can_connect_to_apt_repository(), + reason="Cannot establish a connection to apt repository. Please check your network connection." + ) + + def _check_if_can_connect_to_apt_repository(self): + return_code = Utility.execute_shell("apt-get --simulate upgrade") + return return_code == 0 + + def _check_if_bridge_is_up(self): + return_code = Utility.execute_shell("ifconfig br1") + return return_code == 0 + + def _check_if_can_connect_to_docker_registry(self): + is_docker_installed = Utility.execute_shell("command docker -v", is_shell=True) == 0 + + if not is_docker_installed: + self.assertion.add_error_details("Docker is not installed on the system. Please install docker") + return False + + docker_image_url = f"{Config.DOCKER_REGISTRY_URL}/{Config.DOCKER_IMAGE_NAME}:latest" + + # We need to check for pull here since only pull use the setting from docker-proxy. Unless in the future + # there are other alternatives. If fail exit code will be `1` otherwise will be `124` exit code for `timeout` + can_pull_image = Utility.execute_shell(f"timeout 1s docker pull {docker_image_url}") == 124 + + if not can_pull_image: + self.assertion.add_error_details("Not able to pull image from docker-registry. Please confirm your network " + "connectivity and configure proxy if needed") + return False + + return True + + def _check_if_proxy_is_using(self): + group_vars_env = Utility.parse_yml(Config.GROUP_VARS_ENV_FILE) + + if "proxy_env" not in group_vars_env: + return True + + env_vars = ["http_proxy", "https_proxy"] + + for var in env_vars: + if var not in os.environ: + continue + + if var not in group_vars_env or var not in group_vars_env['proxy_env']: + self.assertion.add_error_details( + f"'{Formatting.red(var)}' is detected in environment variables " + f"but not in '{Config.GROUP_VARS_ENV_FILE}'", + ) + return False + + if group_vars_env['proxy_env'][var] != os.environ[var]: + self.assertion.add_error_details( + f"Environment '{Formatting.red(var)}' is not the same " + f"as declared in '{Config.GROUP_VARS_ENV_FILE}': " + f"{os.environ[var]} != {group_vars_env['proxy_env'][var]}", + ) + return False + + return True + + +class TestbedConnectionValidator(Validator): + + def __init__(self, testbed_name): + super().__init__() + self.testbed_name = testbed_name + + def validate(self): + from devutil.devices.factory import init_testbed_sonichosts + + if not self.testbed_name: + return True + + yml_config = next( + filter(lambda tb: tb['conf-name'] == self.testbed_name, Utility.parse_yml(Config.TESTBED_FILE)) + ) + + sonic_hosts = init_testbed_sonichosts(yml_config['inv_name'], self.testbed_name) + + self.assertion.assert_true( + lambda: self._check_if_duts_are_reachable(sonic_hosts), + reason=f"Devices are not reachable for testbed '{Formatting.red(self.testbed_name)}'. " + "Please check your proxy configs", + ) + + def _check_if_duts_are_reachable(self, sonic_hosts): + _, result = sonic_hosts.reachable() + is_valid = True + + for dut_result in result.values(): + if not dut_result["reachable"] or dut_result["failed"]: + is_valid = False + self.assertion.add_error_details( + f"The following device is unreachable '{Formatting.red(dut_result['hostname'])}'. " + f"Message: '{Formatting.yellow(dut_result['msg'])}'", + ) + + return is_valid + + +def main(args): + if args.testbed_file: + Config.TESTBED_FILE = args.testbed_file + + if args.vm_file: + Config.VM_FILE = args.vm_file + + validators = [] + + if args.target: + validators.extend([ + TestbedConnectionValidator(args.target), + ]) + else: + validators.extend([ + DockerRegistryValidator(), + TestbedValidator(), + InventoryNameValidator(), + FanoutLinkValidator(), + HostNetworkValidation() + ]) + + for validator in validators: + validator.validate() + + if all(map(lambda _validator: _validator.assertion.pass_validation, validators)): + log.info("Successful! No validation error found") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Verify if configuration files are valid") + + parser.add_argument( + '-t', '--testbed-file', + type=str, + dest='testbed_file', + required=False, + help='Testbed file. Only yaml format testbed file is supported.', + ) + + parser.add_argument( + '-m', '--vm-file', + type=str, + dest='vm_file', + required=False, + help='VM files, typically it is the `veos` file', + ) + + parser.add_argument( + '-tb', '--testbed', + type=str, + dest='target', + required=False, + help='Only run check for this testbed. Note that running this options will use ansible ' + 'to check connectivity to all the DUTs for that testbed', + ) + + main(parser.parse_args()) diff --git a/ansible/vtestbed.yaml b/ansible/vtestbed.yaml index 14e31f2085..8be68705d5 100644 --- a/ansible/vtestbed.yaml +++ b/ansible/vtestbed.yaml @@ -336,3 +336,18 @@ inv_name: veos_vtb auto_recover: False comment: Tests virtual switch vm as DPU + +- conf-name: vms-kvm-t1 + group-name: vms6-2 + topo: t1 + ptf_image_name: docker-ptf + ptf: ptf-02 + ptf_ip: 10.250.0.106/24 + ptf_ipv6: fec0::ffff:afa:6/64 + server: server_1 + vm_base: VM0104 + dut: + - vlab-03 + inv_name: veos_vtb + auto_recover: 'False' + comment: Tests virtual switch vm diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ae2fb32cbb..90917d8919 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,8 +24,6 @@ name: $(TeamProject)_$(Build.DefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd stages: - stage: Pre_test - variables: - - group: GIT_SECRETS jobs: - job: static_analysis displayName: "Static Analysis" @@ -52,7 +50,6 @@ stages: value: veos_vtb - name: testbed_file value: vtestbed.yaml - - group: GIT_SECRETS - name: BUILD_BRANCH ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: value: $(System.PullRequest.TargetBranch) @@ -167,7 +164,7 @@ stages: MGMT_BRANCH: $(BUILD_BRANCH) - job: onboarding_elastictest_t0 - displayName: "onboarding testcases by Elastictest" + displayName: "onboarding t0 testcases by Elastictest - optional" timeoutInMinutes: 240 continueOnError: true pool: sonic-ubuntu-1c @@ -175,12 +172,47 @@ stages: - template: .azure-pipelines/run-test-elastictest-template.yml parameters: TOPOLOGY: t0 + STOP_ON_FAILURE: "False" + RETRY_TIMES: 0 MIN_WORKER: $(T0_ONBOARDING_SONIC_INSTANCE_NUM) MAX_WORKER: $(T0_ONBOARDING_SONIC_INSTANCE_NUM) KVM_IMAGE_BRANCH: $(BUILD_BRANCH) MGMT_BRANCH: $(BUILD_BRANCH) TEST_SET: onboarding_t0 + - job: onboarding_elastictest_t1 + displayName: "onboarding t1 testcases by Elastictest - optional" + timeoutInMinutes: 240 + continueOnError: true + pool: sonic-ubuntu-1c + steps: + - template: .azure-pipelines/run-test-elastictest-template.yml + parameters: + TOPOLOGY: t1-lag + STOP_ON_FAILURE: "False" + RETRY_TIMES: 0 + MIN_WORKER: $(T1_LAG_ONBOARDING_INSTANCE_NUM) + MAX_WORKER: $(T1_LAG_ONBOARDING_INSTANCE_NUM) + KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: $(BUILD_BRANCH) + TEST_SET: onboarding_t1 + + - job: onboarding_elastictest_dualtor + displayName: "onboarding dualtor testcases by Elastictest - optional" + timeoutInMinutes: 240 + continueOnError: true + pool: sonic-ubuntu-1c + steps: + - template: .azure-pipelines/run-test-elastictest-template.yml + parameters: + TOPOLOGY: dualtor + STOP_ON_FAILURE: "False" + RETRY_TIMES: 0 + MIN_WORKER: $(T0_DUALTOR_INSTANCE_NUM) + MAX_WORKER: $(T0_DUALTOR_INSTANCE_NUM) + KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: $(BUILD_BRANCH) + TEST_SET: onboarding_dualtor # - job: wan_elastictest # displayName: "kvmtest-wan by Elastictest" diff --git a/docs/README.md b/docs/README.md index 67754d6b53..4cc4cb3f60 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,7 +15,7 @@ Using of pytest does not mean that ansible will no long be used. Ansible is stil * `ansible`: This folder contains code for SONiC testbed deployment and setup. Legacy ansible playbook based automation code is also under this folder. * `docs`: Documentations * `spytest`: The SPyTest automation framework and tests for validating SONiC -* `test_peporting`: For parsing, uploading and processing junit xml test reports generated by pytest. Processed test data is uploaded to Kusto for query. +* `test_reporting`: For parsing, uploading and processing junit xml test reports generated by pytest. Processed test data is uploaded to Kusto for query. * `tests`: Pytest and pytest-ansible based test infrastructure code and test scripts * `api_wiki`: Information on how to communicate between localhost/dut/ptf. Useful for testwriting. diff --git a/docs/ansible/README.deploy.md b/docs/ansible/README.deploy.md index 203261e6ae..623100ddf4 100644 --- a/docs/ansible/README.deploy.md +++ b/docs/ansible/README.deploy.md @@ -20,6 +20,9 @@ and public [sonicdev Docker registry](https://sonicdev-microsoft.azurecr.io/). - Update `[ntp,syslog,dns]_servers` with a list of your server IPs for these services. - Update APT repository if you are using private repo. - Update Docker [registry](/ansible/vars/docker_registry.yml/) if you are using private registry. + + **NOTE:** for more information about credentials variables, see: [credentials management configuration](https://github.com/sonic-net/sonic-mgmt/blob/master/docs/testbed/README.new.testbed.Configuration.md#credentials-management). + - Update management IP of switch1 - Find the ManagementIPInterfaces xml block in [minigraph/switch1.xml](/ansible/minigraph/switch1.xml/) and change both IP addresses. diff --git a/docs/testbed/README.md b/docs/testbed/README.md index cafd95c978..b94129660a 100644 --- a/docs/testbed/README.md +++ b/docs/testbed/README.md @@ -9,8 +9,7 @@ - [cEOS](README.testbed.cEOS.md) - [Routing](README.testbed.Routing.md) - [Keysight](README.testbed.Keysight.md) -- [Configuration](README.testbed.Config.md) -- [Configuration - new](README.new.testbed.Configuration.md) +- [Configuration](README.new.testbed.Configuration.md) - [Minigraph](README.testbed.Minigraph.md) - [Command Line](README.testbed.Cli.md) - [Example](README.testbed.Example.md) @@ -19,3 +18,4 @@ - [Internal](README.testbed.Internal.md) - [Kubernetes Setup](README.testbed.k8s.Setup.md) - [SAI Test Setup](./sai_quality/README.md) +- [TACACS](README.testbed.TACACS.md) diff --git a/docs/testbed/README.new.testbed.Configuration.md b/docs/testbed/README.new.testbed.Configuration.md index 2140d01a39..1d4c03b86f 100644 --- a/docs/testbed/README.new.testbed.Configuration.md +++ b/docs/testbed/README.new.testbed.Configuration.md @@ -19,6 +19,32 @@ There are several devices needed to get sonic-mgmt up and running with the test Information for testbed and topology can be referenced at [Sonic-Mgmt Testbed Overview](/docs/testbed/README.testbed.Overview.md) +# Testbed inventory + +- [```ansible/lab```](/ansible/lab): Include all lab DUTs, fanout switches and testbed server topologies + +- [```ansible/veos```](/ansible/veos): all servers and VMs + + +## Testbed Physical Topology + +- [```ansible/files/sonic_lab_devices.csv```](/ansible/files/sonic_lab_devices.csv): Helper file helps you create lab_connection_graph.xml, list all devices that are physically connected to fanout testbed (all devices should be in ansible/lab) + +- [```ansible/files/sonic_lab_links.csv```](/ansible/files/sonic_lab_links.csv): Helper file helps you to create lab_connection_graph.xml, list all physical links between DUT, Fanoutleaf and Fanout root switches, servers and vlan configurations for each link + +- [```ansible/files/sonic_lab_pdu_links.csv```](/ansible/files/sonic_lab_pdu_links.csv): Helper file helps you to create lab_connection_graph.xml, list all pdu links between devices and pdu devices. For details about pdu configuraions, check doc [pdu wiring](./README.testbed.PDUWiring.md) + +- [```ansible/files/sonic_lab_bmc_links.csv```](/ansible/files/sonic_lab_bmc_links.csv): Helper file helps you to create lab_connection_graph.xml, list all bmc links between devices and management devices. + +- [```ansible/files/sonic_lab_console_links.csv```](/ansible/files/sonic_lab_console_links.csv): Helper file helps you to create lab_connection_graph.xml, list all console links between devices and management devices. + +- [```ansible/files/lab_connection_graph.xml```](/ansible/files/lab_connection_graph.xml): This is the lab graph file for library/conn_graph_facts.py to parse and get all lab fanout switch connections information. If you have only one fanout switch, you may go ahead and manually modify the sample lab_connection_graph.xml file to set both your fanout leaf and fanout root switch management IP point to the same fanout switch management IP and make sure all DUT and Fanout name and IP are matching your testbed. + +- [```ansible/files/creategraph.py```](/ansible/files/creategraph.py): Helper file helps you generate a lab_connection_graph.xml based on the device file and link file specified above. + + Based on ansible_facts, you may write ansible playbooks to deploy fanout switches or run test which requires to know the DUT physical connections to fanout switch + + # Modify Testbed.yaml Configuration File There are 7 main sections in testbed.yaml that need to be edited: 1. device_groups @@ -29,7 +55,7 @@ There are 7 main sections in testbed.yaml that need to be edited: 6. testbed 7. topology -Each of the sections above contribute to the files that need to be written into in order for the test cases to run. For more information about what each file does, please reference [Sonic-Mgmt Testbed Configuration](/docs/testbed/README.testbed.Config.md). +Each of the sections above contribute to the files that need to be written into in order for the test cases to run. For more information about what each file does, please reference [Testbed Inventory](#testbed-inventory) and [Testbed Physical Topology](#testbed-physical-topology). Within the testbed.yaml file: @@ -45,7 +71,7 @@ The device_groups section generates the lab file which is the inventory file nec ### devices section **USAGE**: files/sonic_lab_devices, group_vars/fanout/secrets, group_vars/lab/secrets, lab -The devices section is a dictionary that contains all devices and hosts. This section does not contain information on PTF containers. For more information on PTF containers, see the testbed.csv file. +The devices section is a dictionary that contains all devices and hosts. This section does not contain information on PTF containers. For more information on PTF containers, see the testbed.yaml file. For each device that you add, add the following: @@ -63,7 +89,7 @@ For each device that you add, add the following: - hwsku - this is the look up value for credentials in /group_vars/all/labinfo.json. Without this section, things will fail. Make sure this field is filled out and verify labinfo.json is accurate. - device type - the type of device. If you only have 4 devices, you can leave the provided labels alone -The lab server section requires different fields to be entered: ansible_become_pass, sonicadmin_user, sonicadmin_password, sonicadmin_initial_password. Sonicadmin_user is still just the username. The other fields is the password. These fields were selected because they are variables taken directly group_var/lab/secrets.yml. So for convenience, this section of the config file takes a copy of the variable labels. +The lab server section requires different fields to be entered: ansible_become_pass, sonicadmin_user, sonicadmin_password, sonicadmin_initial_password. Sonicadmin_user is still just the username. The other fields is the password. These fields were selected because they are variables taken directly group_var/lab/secrets.yml. For more information related to authentication, take a look at the [credentials section below](#credentials-management). So for convenience, this section of the config file takes a copy of the variable labels. ### host_vars section: **USAGE**: all host_var values @@ -107,15 +133,28 @@ Define: - define the IPs of the VMs (i.e." 10.250.1.0, 10.250.1.1, 10.250.1.2, etc...) ### testbed section: -**USAGE**: testbed.csv +**USAGE**: testbed.yaml +testbed.csv is deprecated, please use testbed.yaml instead. This is where the topology configuration file for the testbed will collect information from when running TestbedProcessing.py. -| #conf-name | group-name | topo | ptf_image_name | ptf|ptf_ip |ptf_ipv6 | server | vm_base | dut | inv_name | auto_recover | comment | -| ----------------- | ------------------ | ------- | -------------- |---------- |------------ |------------|-------------- | --------- | ----- | ---------- | -------------- | --------- | -| [ptf32 conf-name] | [ptf32 group-name] | [ptf32] | [docker-ptf] |[ptf-name] |[ip address] | [ipv6 address]|[server group] | [vm_base] | [dut] | [inv_name] | [auto_recover] | [comment] | -| [t0 conf-name] | [t0 group-name] | [t0] | [docker-ptf] |[ptf-name] |[ip address] | [ipv6 address]|[server group] | [vm_base] | [dut] | [inv_name] | [auto_recover] | [comment] | - +``` +- conf-name: vms-sn2700-t1 + group-name: vms1-1 + topo: t1 + ptf_image_name: docker-ptf + ptf: ptf_vms1-1 + ptf_ip: 10.255.0.178/24 + ptf_ipv6: 2001:db8:1::3/64 + ptf_extra_mgmt_ip: [] + server: server_1 + vm_base: VM0100 + dut: + - str-msn2700-01 + inv_name: lab + auto_recover: 'True' + comment: Tests Mellanox SN2700 vms +``` For each topology you use in your testbed environment, define the following: - conf-name - to address row in table @@ -140,6 +179,18 @@ For each topology you use in your testbed environment, define the following: - ansible_ssh_user - username to login to lab server - ansible_ssh_pass - password to login to lab server +#### Consistency Rule: +1. `conf-name` must be unique. +2. `group-name` must be up to 8 characters long. +3. All testbed with the same `group-name` must have the same: + - `ptf_ip` + - `server` + - `vm_base` +4. `topo` name must be valid and presented in [`/ansible/vars/topo_*.yml`](https://github.com/sonic-net/sonic-mgmt/tree/master/ansible/vars). +5. `ptf_image_name` must be valid. +6. `server` name must be valid and presented in [`veos`](https://github.com/sonic-net/sonic-mgmt/blob/master/ansible/veos) file. +7. `vm_base` must not overlap with testbed of different group names. + ### topology section: **USAGE**: files/sonic_lab_links.csv @@ -174,6 +225,193 @@ docker_registry_username: root docker_registry_password: root ``` +### inventory file: + +The inventory file contains all device host/IP information for testbeds within its inventory. +For example, the testbed `vms-sn2700-t1` uses the inventory file lab ( seecified by `inv_name: lab` in `testbed.yaml`). + +The `ansible/lab` inventory file includes three types of section for different devices: +- sonic +- fanout/pdu/mgmt/server +- ptf + +1. sonic Section + +The sonic section lists various SONiC platforms, such as `sonic_sn2700_40`, `sonic_a7260`, etc. +Ensure that the following fields are correctly filled for your Device Under Test (DUT): + +``` +sonic_sn2700_40: + vars: + hwsku: ACS-MSN2700 + iface_speed: 40000 + hosts: + str-msn2700-01: + ansible_host: 10.251.0.188 + model: MSN2700-CS2FO + serial: MT1234X56789 + base_mac: 24:8a:07:12:34:56 + syseeprom_info: + "0x21": "MSN2700" + "0x22": "MSN2700-CS2FO" + "0x23": "MT1234X56789" + "0x24": "24:8a:07:12:34:56" + "0x25": "12/07/2016" + "0x26": "0" + "0x28": "x86_64-mlnx_x86-r0" + "0x29": "2016.11-5.1.0008-9600" + "0x2A": "128" + "0x2B": "Mellanox" + "0xFE": "0xFBA1E964" +``` +- `iface_speed` is the speed of DUT's interface. For a 40G switch, `iface_speed` should be 40000; for a 100G switch, it should be 100000. The test `iface_namingmode/test_iface_namingmode.py` will fail if iface_speed is missing or incorrect. + +- For the fields under `syseeprom_info`, the EEPROM type descriptions are defined in `test_chassis.py` as follows: + +``` +# Valid OCP ONIE TlvInfo EEPROM type codes as defined here: +# https://opencomputeproject.github.io/onie/design-spec/hw_requirements.html +ONIE_TLVINFO_TYPE_CODE_PRODUCT_NAME = '0x21' # Product Name +ONIE_TLVINFO_TYPE_CODE_PART_NUMBER = '0x22' # Part Number +ONIE_TLVINFO_TYPE_CODE_SERIAL_NUMBER = '0x23' # Serial Number +ONIE_TLVINFO_TYPE_CODE_BASE_MAC_ADDR = '0x24' # Base MAC Address +ONIE_TLVINFO_TYPE_CODE_MFR_DATE = '0x25' # Manufacture Date +ONIE_TLVINFO_TYPE_CODE_DEVICE_VERSION = '0x26' # Device Version +ONIE_TLVINFO_TYPE_CODE_LABEL_REVISION = '0x27' # Label Revision +ONIE_TLVINFO_TYPE_CODE_PLATFORM_NAME = '0x28' # Platform Name +ONIE_TLVINFO_TYPE_CODE_ONIE_VERSION = '0x29' # ONIE Version +ONIE_TLVINFO_TYPE_CODE_NUM_MACS = '0x2A' # Number of MAC Addresses +ONIE_TLVINFO_TYPE_CODE_MANUFACTURER = '0x2B' # Manufacturer +ONIE_TLVINFO_TYPE_CODE_COUNTRY_CODE = '0x2C' # Country Code +ONIE_TLVINFO_TYPE_CODE_VENDOR = '0x2D' # Vendor +ONIE_TLVINFO_TYPE_CODE_DIAG_VERSION = '0x2E' # Diag Version +ONIE_TLVINFO_TYPE_CODE_SERVICE_TAG = '0x2F' # Service Tag +ONIE_TLVINFO_TYPE_CODE_VENDOR_EXT = '0xFD' # Vendor Extension +ONIE_TLVINFO_TYPE_CODE_CRC32 = '0xFE' # CRC-32 +``` + +- `show platform syseeprom` can get some of these values, +- The command `show platform syseeprom` can retrieve some of these values. The test case `tests/platform_tests/api/test_chassis.py` verifies the correctness of these fields. If some fields are unknown, running the test will show expected values for the unfilled fields, providing a summary of discrepancies. The summary looks like this: + +``` +Failed: 'base_mac' value is incorrect. Got '74:83:ef:63:e2:86', expected '74:83:ef:63:e2:87' +``` + +2. Fanout, PDU, Mgmt and Server Sections + +Those record the hostname and IP address of the respective fanout switch, PDU, or console server. +Those devices are also recorded in the following csv files: + +- `ansible/files/sonic_lab_devices.csv` +- `ansible/files/sonic_lab_pdu_links.csv` +- `ansible/files/sonic_lab_console_links.csv` + +3. `ptf` Section + +In `ptf` section the `ansible_ssh_user` and `ansible_ssh_pass` variables specify the credentials for the PTF container. +`ansible_host` should match the `ptf_ip` in `testbed.yaml`. + +``` + ptf: + vars: + ansible_ssh_user: root + ansible_ssh_pass: root + hosts: + ptf_ptf1: + ansible_host: 10.255.0.188 + ansible_hostv6: 2001:db8:1::1/64 +``` + +Ensure that these configurations are correct to facilitate proper communication and testing within the testbed environment. + + +# Credentials management + +This section briefly describes how sonic-mgmt manages credentials for authentication purposes. `Pytest` will also use these variables to execute tests. + + +Variables are stored in [`ansible/group_vars//*.(yml|json)`](https://github.com/sonic-net/sonic-mgmt/tree/master/ansible/group_vars) where `` is the name of the group declared in your inventory files. + +The default ansible group name `all` refers to all the groups. Therefore, we store the shared configs in `ansible/group_vars/all` folder. +For more information related to variable encryptions and how to use, please refer to [official Ansible variable documentation](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html) and [official Ansible encryption and decryption guide](https://docs.ansible.com/ansible/latest/vault_guide/vault_encrypting_content.html#encrypting-individual-variables-with-ansible-vault). + +Currently, the variables that we're using to authenticate are: + +## Fanout + +For explanations on how sonic-mgmt works with these variables, refer to [Fanout Credentials documentation](https://github.com/sonic-net/sonic-mgmt/blob/master/docs/testbed/README.testbed.Fanout.md). + + +### EOS (Arista) +- `fanout_admin_user` +- `fanout_admin_password` + +### Melanox +- `fanout_mlnx_user` +- `fanout_mlnx_password` + +### Sonic devices +- `fanout_sonic_user` +- `fanout_sonic_password` + +### Tacas credentials +- `fanout_tacacs_user` +- `fanout_tacacs_password` + +**Note:** when TACACS is enabled but different devices have different credentials, do not set `fanout_tacacs_user` and `fanout_tacacs_password` but do set `fanout_tacac_OS_user/password`, for example: +- `fanout_tacacs_mlnx_user` +- `fanout_tacacs_mlnx_password` +- `fanout_tacacs_eos_user` +- `fanout_tacacs_eos_password` +- `fanout_tacacs_sonic_user` +- `fanout_tacacs_sonic_password` + +### Local credentials +These local credentials can be used in substitution of TACACS account. + +#### Network credential +- `fanout_network_user` +- `fanout_network_password` + +#### Shell credential +- `fanout_shell_user` +- `fanout_shell_password` + + +## DUT credentails +- `sonicadmin_user`: os user for SONiC image +- `sonicadmin_password`: password for os user for SONiC image +- `sonicadmin_initial_password`: password you built into baseimage + +- `eos_default_login`: default credential for Eos +- `eos_default_password`: default credential for Eos +- `eos_login`: credential for eos +- `eos_password`: credential for eos + +- `junos_default_login`: default credential for Junos +- `junos_default_password`: default credential for Junos + +- `junos_login`: credential for Junos +- `junos_password`: credential for Junos + +- `cisco_login`: credential for Cisco +- `cisco_password`: credential for Cisco + +- `sonic_login`: credential for SONiC +- `sonic_password`: credential for SONiC +- `sonic_default_passwords`: default password for SONiC + +- `k8s_master_login`: credential for k8s +- `k8s_master_password`: credential for k8s + +- `ptf_host_user`: credential for PTF +- `ptf_host_pass`: credential for PTF + +- `vm_host_user`: credential for VM +- `vm_host_password`: credential for VM +- `vm_host_become_password`: root password for VM + + # Testbed Processing Script **NOTE**: - This section is an example of starting VMs, deploying topology and running test cases with ansible-playbook. However, it is old-fasioned. @@ -218,6 +456,9 @@ Run the following commands: > echo $TESTCASE_NAME
> ansible-playbook -i lab -l sonic-ag9032 test_sonic.yml -e testbed_name=$TESTBED_NAME -e testcase_name=$TESTCASE_NAME +### Additional steps before running QoS SAI Test Case + +Unlike other SONiC test case, QoS SAI requires additional setup and syncd RPC container image building. Please refer [here](https://github.com/sonic-net/sonic-mgmt/blob/master/docs/testbed/README.testbed.QosRpc.md) # Troubleshooting Issue: Testbed Command Line complains there is no password file available.
@@ -239,3 +480,37 @@ Resolution: There are a plethora of things that could be wrong here. Here are so 3. Does your device have the correct hwsku in files/sonic_lab_devices.csv? 4. Confirm that your lab file does not have "/"s after the IPs. "/"s are a way to denote port numbers which INI files do not recognize. 5. Recheck your testbed.yaml configuration file to see if you got the IPs and credentials correct + + +# Configuration Validation Script + +We have provided a script that cross-checks your configuration with the guidelines outlined in this document to ensure optimal functionality. The script is located at `ansible/verify_config.py`. + +To validate all configurations within your project, execute the following command: + +```bash +python3 verify_config.py +``` +The script will present any warnings or errors based on our validation rules, using the default testbed file `testbed.yaml` and the default VM file `veos`. + + +If you wish to use custom VM and testbed files, input the command as shown below, replacing and with your filenames: + +```bash +python3 verify_config.py -t -m +``` + +For validating your connection to a specific testbed listed in the testbed file, run the following command **within `sonic-mgmt` container**: + +```bash +python3 verify_config.py -tb +``` + +Lastly, to specify both custom testbed and VM files along with a specific testbed, use: + + +```bash +python3 verify_config.py -t -m -tb +``` + +Replace ``, ``, and `` with your respective file names and testbed name to proceed with the validation. diff --git a/docs/testbed/README.testbed.Config.md b/docs/testbed/README.testbed.Config.md deleted file mode 100644 index 129cf8e1ec..0000000000 --- a/docs/testbed/README.testbed.Config.md +++ /dev/null @@ -1,72 +0,0 @@ -# Testbed Configuration - -## Testbed Inventory - -- [```ansible/lab```](/ansible/lab): Include all lab DUTs, fanout switches and testbed server topologies - -- [```ansible/veos```](/ansible/veos): all servers and VMs - -## Testbed Physical Topology - -- [```ansible/files/sonic_lab_devices.csv```](/ansible/files/sonic_lab_devices.csv): Helper file helps you create lab_connection_graph.xml, list all devices that are physically connected to fanout testbed (all devices should be in ansible/lab) - -- [```ansible/files/sonic_lab_links.csv```](/ansible/files/sonic_lab_links.csv): Helper file helps you to create lab_connection_graph.xml, list all physical links between DUT, Fanoutleaf and Fanout root switches, servers and vlan configurations for each link - -- [```ansible/files/sonic_lab_pdu_links.csv```](/ansible/files/sonic_lab_pdu_links.csv): Helper file helps you to create lab_connection_graph.xml, list all pdu links between devices and pdu devices. For details about pdu configuraions, check doc [pdu wiring](./README.testbed.PDUWiring.md) - -- [```ansible/files/sonic_lab_bmc_links.csv```](/ansible/files/sonic_lab_bmc_links.csv): Helper file helps you to create lab_connection_graph.xml, list all bmc links between devices and management devices. - -- [```ansible/files/sonic_lab_console_links.csv```](/ansible/files/sonic_lab_console_links.csv): Helper file helps you to create lab_connection_graph.xml, list all console links between devices and management devices. - -- [```ansible/files/lab_connection_graph.xml```](/ansible/files/lab_connection_graph.xml): This is the lab graph file for library/conn_graph_facts.py to parse and get all lab fanout switch connections information. If you have only one fanout switch, you may go head manually modify the sample lab_connection_graph.xml file to set bot your fanout leaf and fanout root switch management IP point to the same fanout switch management IP and make sure all DUT and Fanout name and IP are matching your testbed. - -- [```ansible/files/creategraph.py```](/ansible/files/creategraph.py): Helper file helps you generate a lab_connection_graph.xml based on the device file and link file specified above. - - Based on ansible_facts, you may write ansible playbooks to deploy fanout switches or run test which requires to know the DUT physical connections to fanout switch - - -## Testbed Logical Topology - -[```testbed.csv```](/ansible/testbed.csv) is the topology configuration file for the testbed. - -### ```testbed.csv``` format -``` -# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,inv_name,auto_recover,comment -ptf1-m,ptf1,ptf32,docker-ptf,ptf-1,10.255.0.188/24,,server_1,,str-msn2700-01,lab,False,Tests ptf -vms-t1,vms1-1,t1,docker-ptf,ptf-2,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests vms -vms-t1-lag,vms1-1,t1-lag,docker-ptf,ptf-3,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests vms - -``` - -- conf-name - to address row in table -- group-name – used in interface names, up to 8 characters -- topo – name of topology -- ptf_imagename – defines PTF image -- ptf_ip – ip address for mgmt interface of PTF container -- server – server where the testbed resides -- vm_base – first VM for the testbed. If empty, no VMs are used -- dut – target dut name -- inv_name - inventory file name that contains the definition of the target DUTs -- auto_recover - (`yes`|`True`|`true`) to recover this testbed when runnings serve recovery script, (`no`|`False`|`false`) otherwise -- comment – any text here - -### ```testbed.csv``` consistency rules -``` -# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,comment -vms2-2-b,vms2-2,t1,docker-ptf,ptf-1,10.0.10.7/23,,server_1,VM0100,str-d6000-05,brcm test -vms2-2-m,vms2-2,t1,docker-ptf,ptf-2,10.0.10.7/23,,server_1,VM0100,str-msn2700-5,mlnx test - -``` -Must be strictly checked in code reviews - - conf-name must be unique - - All testbed records with the same testbed-name must have the same: - - ptf_ip - - server - - vm_base - - testbed-name must be up to 8 characters long - - topo name must be valid (topo registered in ```veos``` and topo file presented in vars/topo_*.yml - - ptf_imagename must be valid - - server name must be valid and presented in veos inventory file - - vm_base must not overlap with testbeds from different groups (different test-name) - -TODO: check this constraints in testbed-cli.sh diff --git a/docs/testbed/README.testbed.Example.Config.md b/docs/testbed/README.testbed.Example.Config.md index 4e5b144323..98c72a38d8 100644 --- a/docs/testbed/README.testbed.Example.Config.md +++ b/docs/testbed/README.testbed.Example.Config.md @@ -62,6 +62,21 @@ grep 'vlab-01' -A 2 ./veos_vtb ``` this is the IP for the DUT. +#### DUT user name and password +The DUT may enable TACACS AAA, if you can't login DUT with local user, please find TACACS user name and password by following steps: + +##### User name: +``` + 1. If secret_group_vars['str']['ansible_ssh_user'] defined, the DUT user name is the value of secret_group_vars['str']['ansible_ssh_user'] + 2. If secret_group_vars['str']['ansible_ssh_user'] not defined, the DUT user name is the value of sonicadmin_user variable defined in group_vars/lab/secrets.yml +``` + +##### Password: +``` + 1. If secret_group_vars['str']['altpasswords'] defined, the DUT user name is the value of secret_group_vars['str']['altpasswords'][0] + 2. If secret_group_vars['str']['altpasswords'] not defined, the DUT user name is the value of sonicadmin_password variable defined in group_vars/lab/secrets.yml +``` + Then, you can use this IP `10.250.0.101` to access that DUT. @@ -153,3 +168,4 @@ For this article, some of the reference docs as: - [```Testbed Configuration```](/docs/testbed/README.testbed.Config.md): Introduction about Testbed configuration, mainly about the testbed.csv (Will be replaced by testbed.yaml). - [```New Testbed Configuration```](/docs/testbed/README.new.testbed.Configuration.md): Introduction about Testbed configuration, mainly about the Testbed.yaml. - [```KVM Testbed Setup```](/docs/testbed/README.testbed.VsSetup.md) +- [```Testbed TACACS server```](/docs/testbed/README.testbed.TACACS.md) diff --git a/docs/testbed/README.testbed.Overview.md b/docs/testbed/README.testbed.Overview.md index bb8f832278..979607e8f8 100644 --- a/docs/testbed/README.testbed.Overview.md +++ b/docs/testbed/README.testbed.Overview.md @@ -49,12 +49,13 @@ Please be noted that the number of test servers, fanout switches and SONiC DUTs ## Logical topologies -Mainly 4 types of testbed topologies can be simulated based on the physical topology. +Mainly 5 types of testbed topologies can be simulated based on the physical topology. * T0 * T1 * T2 * PTF +* PTP Details of the logical topologies are defined in `ansible/vars/topo_*.yml` files. @@ -220,6 +221,19 @@ The PTF type topology does not have VMs. All the DUT ports are connected to a PT * Requires no VM. * All the DUT ports are connected to the PTF docker. +### PTP type topology +The point-to-point (PTP) topology is used to validate transceivers and their link stability over L2 control traffic such as LLDP. It does not involve ports connected to PTF docker or VMs. Instead, both SONiC DUTs are connected through multiple ports. + +```text ++-----------------+ +-----------------+ +| Port 1|<--->|Port 1 | +| Port 2|<--->|Port 2 | +| Device 1 | | Device 2 | +| | | | +| Port N|<--->|Port N | ++-----------------+ +-----------------+ +``` + ## Build testbed ### Hardware Requirements diff --git a/docs/testbed/README.testbed.QosRpc.md b/docs/testbed/README.testbed.QosRpc.md new file mode 100644 index 0000000000..645fda1623 --- /dev/null +++ b/docs/testbed/README.testbed.QosRpc.md @@ -0,0 +1,66 @@ +# Background + +To run the QoS SAI test, an additional syncd RPC container image and relevant configuration are required. You have two options for building, uploading, and accessing this image: + +- [Build your own syncd RPC container image](#1. Build your own syncd RPC container image) +- [Utilize SONiC public image](#2. Utilize image build by SONiC public service) + + +# 1. Build your own syncd RPC container image + +## 1.1 How to build the syncd RPC Container Image + +Familiarize yourself with the SONiC build system by reviewing [doc](https://github.com/sonic-net/sonic-buildimage/blob/master/README.md) +Follow the build steps in above documentation. Before proceeding to the last step "build sonic image", build the syncd RPC container image for your platforms using below command: + +``` +# Build syncd RPC container iamge +make ENABLE_SYNCD_RPC=y target/docker-syncd-${platform_rpc}-rpc.gz +``` + +Replace ${platform_rpc} with the appropriate value for your platform: + +- Barefoot: bfn +- Broadcom: brcm +- Centec: centec +- Mellanox: mlnx +- Nephos: nephos + +## 1.2 How to setup and manage your docker registry + +For detailed instructions on establishing and managing your Docker registry, refer to [Docker Registry Documentation](https://docs.docker.com/registry/) + +## 1.3 How to access docker registry in QoS SAI test code + +For detailed guidance, refer to the "docker_registry section" within the [Sonic-Mgmt Testbed Setup](https://github.com/sonic-net/sonic-mgmt/blob/master/docs/testbed/README.new.testbed.Configuration.md). +To configure access to the Docker registry, navigate to the `ansible/vars/docker_registry.yml` file. and then enter the appropriate Docker registry URL, your username, and password in the specified format: + +``` +docker_registry_host: "soniccr1.azurecr.io" + +docker_registry_username: "read-only-user" +docker_registry_password: "password-of-user" +``` + +# 2. Utilize image build by SONiC public service + +If you do not require a private image, recommend to use the SONiC public service to get SONiC images and container image, avoid to build image and setuping docker registry yourself. + +The artifacts download link is structured as follows: + +``` +https://sonic-build.azurewebsites.net/ui/sonic/pipelines/1/builds/{buildid}/artifacts?branchName={branchname} +``` + +For instance, to download Broadcom SONiC image for master branch build in pipeline: +"https://dev.azure.com/mssonic/build/_build/results?buildId=547766&view=results" to replace {buildid} with 547766 and {branchname} with master: + +``` +https://sonic-build.azurewebsites.net/ui/sonic/pipelines/1/builds/547766/artifacts?branchName=master +``` + +As this is a public service, there is no need to modify `ansible/vars/docker_registry.yml`. The QoS SAI test script will automatically pull the syncd rpc container image based on the configuration in `ansible/group_vars/all/public_docker_registry.yml`: + +``` +public_docker_registry_host: sonicdev-microsoft.azurecr.io +``` diff --git a/docs/testbed/README.testbed.Setup.md b/docs/testbed/README.testbed.Setup.md index 7452407cb4..f26f162bef 100644 --- a/docs/testbed/README.testbed.Setup.md +++ b/docs/testbed/README.testbed.Setup.md @@ -219,12 +219,14 @@ Once you are in the docker container, you need to modify the testbed configurati ``` ansible -m ping -i veos vm_host_1 ``` + - (Optional) The connectivity to the public internet is necessary during the setup, if the lab env of your organization requires http/https proxy server to reach out to the internet, you need to configure to use the proxy server. It will automatically be leveraged on required steps (e.g. Docker daemon config for image pulling, APT configuration for installing packages). You can configure it in [`ansible/group_vars/all/env.yml`](https://github.com/sonic-net/sonic-mgmt/blob/master/ansible/group_vars/all/env.yml) - VMs - Update /ansible/group_vars/vm_host/main.yml with the location of the veos files or veos file name if you downloaded a different version - Update the VM IP addresses in the [`ansible/veos`](/ansible/veos) inventory file. These IP addresses should be in the management subnet defined above. - - Update the VM credentials in `ansible/group_vars/eos/creds.yml`. + - Update the VM credentials in `ansible/group_vars/eos/creds.yml`. For more information on how to configure credentials, see [credentials management configuration](https://github.com/sonic-net/sonic-mgmt/blob/master/docs/testbed/README.new.testbed.Configuration.md#credentials-management). + ``` cat <> /data/sonic-mgmt/ansible/group_vars/eos/creds.yml --- @@ -232,14 +234,14 @@ Once you are in the docker container, you need to modify the testbed configurati ansible_user: admin EOT ``` - -update the cEOS vars in [`ansible/group_vars/all/ceos.ymal`](/ansible/group_vars/all/ceos.yml). + - Update the cEOS vars in [`ansible/group_vars/all/ceos.ymal`](/ansible/group_vars/all/ceos.yml). ``` ceos_image_filename: cEOS64-lab-4.25.10M.tar ceos_image_orig: ceosimage:4.25.10M ceos_image: ceosimage:4.25.10M skip_ceos_image_downloading: true ``` - **NOTE: We are using local ceos image, hence the skip ceos image downloading should be set as true. + **NOTE**: We are using local ceos image, hence the skip ceos image downloading should be set as true. ## Deploy physical Fanout Switch VLAN @@ -250,7 +252,7 @@ Please follow the "Testbed Physical Topology" section of the [Configuration Guid We are using Arista switches as the fanout switches in our lab. So, the playbook under `roles/fanout` is for deploying fanout (leaf) switch Vlan configurations on Arista devices only. If you are using other types of fanout switches, you can manually configure the Vlan configurations on the switch, or you can deploy a regular Layer-2 switch configuration. -Our fanout switches deploy using the Arista switch's eosadmin shell login. If you have an Arista switch as your fanout and you want to run `fanout/tasks/main.yml` to deploy the switch, please `scp` the `roles/fanout/template/rc.eos` file to the Arista switch flash, and make sure that you can login to the shell with `fanout_admin_user/fanout_admin_password`. +Our fanout switches deploy using the Arista switch's eosadmin shell login. If you have an Arista switch as your fanout and you want to run `fanout/tasks/main.yml` to deploy the switch, please `scp` the `roles/fanout/template/rc.eos` file to the Arista switch flash, and make sure that you can login to the shell with `fanout_admin_user/fanout_admin_password`. For more information regarding credentials management for fanout, see: [fanout management configuration](https://github.com/sonic-net/sonic-mgmt/blob/master/docs/testbed/README.new.testbed.Configuration.md#fanout). **`TODO:`** - Improve testbed root fanout switch configuration method. diff --git a/docs/testbed/README.testbed.TACACS.md b/docs/testbed/README.testbed.TACACS.md new file mode 100644 index 0000000000..038699d3af --- /dev/null +++ b/docs/testbed/README.testbed.TACACS.md @@ -0,0 +1,209 @@ +# Testbed TACACS server + +## TACACS intro +- RFC: https://www.rfc-editor.org/rfc/rfc8907.html + +- HLD: + - https://github.com/sonic-net/SONiC/blob/master/doc/aaa/TACACS%2B%20Authentication.md?plain=1 + - https://github.com/sonic-net/SONiC/blob/master/doc/aaa/TACACS%2B%20Design.md?plain=1 + +- TACACS protocol + - Autehntication: Identify user + - On SONiC device, this will check if user can login + - Authorization: Check if user have permission + - On SONiC device, this will check if user can run command + - Accounting: Record what user do + - On SONiC device, this will record user command to syslog and remote TACACS server + +- TACACS related config on SONiC + - TACACS server config +``` +admin@vlab-01:~$ show tacacs +TACPLUS global auth_type login <== TACACS packet authentication method +TACPLUS global timeout 5 (default) <== TACACS server timeout in second +TACPLUS global passkey testing123 <== TACACS passkey, used for encrypt TACACS packet + +TACPLUS_SERVER address 10.250.0.102 <== TACACS server address + priority 1 <== TACACS server priority, will use TACACS server with bigger value first. + tcp_port 49 <== TACACS server port +``` + + - AAA config +``` +admin@vlab-01:~$ show aaa +AAA authentication login tacacs+ <== How to verify user login, default/local means check local account, tacacs+ means check with remote TACACS server +AAA authentication failthrough False (default) <== Reject user login when authentication failed +AAA authorization login tacacs+ <== How to check user have permission to run every command, tacacs+ means every command will verify with remote TACACS server, local means will verify with linux permission control +AAA accounting login tacacs+,local <== How to record user command history, tacacs+ means send to remote TACACS server, local means write to syslog +``` + +## Testbed TACACS server setup +On testbed, there will be 2 TACACS server running on PTF host: +- TACACS server for dailywork and none TACACS test cases. + - Bind to TCP port 49 + - Auto restart by /root/tacacs_daily_daemon +``` +root@0ce3f7bbb316:~# ps -auxww | grep tacacs_daily_daemon +root 488 0.0 0.0 2388 464 ? S Jul01 0:00 /bin/sh /root/tacacs_daily_daemon + +root@0ce3f7bbb316:~# ps -auxww | grep "/usr/sbin/tac_plus .* -p 49" +root 502 0.0 0.0 5600 1880 ? S Jul01 0:00 /usr/sbin/tac_plus -d 88 -l /var/log/tac_plus_daily.log -C /etc/tac_plus_daily.conf -p 49 -G +``` + +- TACACS server for TACACS test cases. + - Bind to TCP port 59 + - Only start when TACACS test case running, will stop after TACACS test finished. +``` +root@0ce3f7bbb316:~# ps -auxww | grep "/usr/sbin/tac_plus .* -p 59" +root 3303 8.3 0.0 5600 224 ? S 05:08 0:00 /usr/sbin/tac_plus -d 2058 -l /var/log/tac_plus.log -C /etc/tacacs+/tac_plus.conf -p 59 +``` + +- The reason of having 2 TACACS server are: +1. All test case may running in parallel on multiple DUTs. +2. TACACS test case need change TACACS config and start/stop TACACS server, which will conflict with none TACACS test cases. +3. TACACS server for dailywork need auto restart and allow all RW command, which conflict with TACACS test case. + +### How PTF host TACACS server for dailywork deployed +- TACACS server deployed when add-topo, in ansible/roles/vm_set/tasks/add_topo.yml: +``` + - name: Start tacacs+ daily daemon + include_tasks: start_tacacs_daily_daemon.yml +``` + +- Deploy step defined here ansible/roles/vm_set/tasks/start_tacacs_daily_daemon.yml + - For TACACS server passkey, please check "Include tacacs_passkey" step +``` + 1. Try use tacacs_passkey defined in /group_vars/{{ inventory }}/{{ inventory }}.yml first, if not defined, try next step + 2. use default tacacs_passkey defined in group_vars/lab/lab.yml +``` + + - For DUT login account, please check "Include duthost user name" step +``` + 1. If secret_group_vars['str']['ansible_ssh_user'] defined, the DUT user name is the value of secret_group_vars['str']['ansible_ssh_user'] + 2. If secret_group_vars['str']['ansible_ssh_user'] not defined, the DUT user name is the value of sonicadmin_user variable defined in group_vars/lab/secrets.yml +``` + + - For DUT login password, please check "Include duthost password" step +``` + 1. If secret_group_vars['str']['ansible_ssh_user'] defined, the DUT user name is the value of secret_group_vars['str']['ansible_ssh_user'] + 2. If secret_group_vars['str']['ansible_ssh_user'] not defined, the DUT user name is the value of sonicadmin_user variable defined in group_vars/lab/secrets.yml +``` + +### How PTF host TACACS server for TACACS test case deployed +- TACACS server for TACACS test case pre-installed in ptf host docker image. + - PTF host will run 2 TACACS server, for different, please check "Testbed TACACS server setup" section. + +- The check_tacacs fixture will setup TACACS server before TACACS test case, and shutdown TACACS server after TACACS test case finish: + - check_tacacs defined here: tests/tacacs/conftest.py + - check_tacacs will invoke setup_tacacs_server method render and start TACACS server. + - TACACS server config file render by template: tests/tacacs/tac_plus.conf.j2 + - The username and password are defined in this file: tests/tacacs/tacacs_creds.yaml + + +## DUT host TACACS config + +### How DUT host TACACS server for dailywork setup +- TACACS server and AAA config deployed by ansible/config_sonic_basedon_testbed.yml + - Following code will assign PTF IP address as TACACS server IP address when generate minigraph.xml +``` + - name: Enable PTF tacacs server by default + set_fact: + use_ptf_tacacs_server: true + tacacs_enabled_by_default: true + when: use_ptf_tacacs_server is not defined + + - debug: msg="use_ptf_tacacs_server {{ use_ptf_tacacs_server }}" + + - block: + - name: saved original minigraph file in SONiC DUT(ignore errors when file does not exist) + shell: mv /etc/sonic/minigraph.xml /etc/sonic/minigraph.xml.orig + become: true + ignore_errors: true + + - name: Update TACACS server address to PTF IP + set_fact: + tacacs_servers: ["{{ testbed_facts['ptf_ip'] }}"] + when: use_ptf_tacacs_server is defined and use_ptf_tacacs_server|bool == true + + - debug: msg="tacacs_servers {{ tacacs_servers }}" +``` + + - Following code will change DUT host AAA config to enable TACACS AAA feature, because per-command AAA feature does not exist on older release, so this step will ignore all error: +``` + - name: Configure TACACS with PTF TACACS server + become: true + shell: "{{ tacacs_config_cmd }}" + loop: + - config tacacs authtype login + - config aaa authorization tacacs+ + - config aaa accounting "tacacs+ local" + loop_control: + loop_var: tacacs_config_cmd + ignore_errors: true + when: use_ptf_tacacs_server is defined and use_ptf_tacacs_server|bool == true +``` + + - On DUT host, user can check TACACS related config with following command: +``` +admin@vlab-01:~$ show aaa +AAA authentication login tacacs+ +AAA authentication failthrough False (default) +AAA authorization login tacacs+,local +AAA accounting login tacacs+,local + +admin@vlab-01:~$ show tacacs +TACPLUS global auth_type login +TACPLUS global timeout 5 (default) +TACPLUS global passkey testing123 + +TACPLUS_SERVER address 10.250.0.102 + priority 1 + tcp_port 49 +``` + +### How DUT host TACACS server for TACACS test case setup +- This TACACS server been setup by setup_tacacs_client method: tests/tacacs/utils.py +- When TACACS test case finish, restore_tacacs_servers will restore the dut host TACACS config to use the TACACS server for dailywork. + +## Debug testbed TACACS server + +- How to stop testbed TACACS server: + - I can't login DUT host, user can stop testbed TACACS server and login with local account. + + - Find PID of /root/tacacs_daily_daemon with following command, in this case PID is 488 +``` +root@0ce3f7bbb316:~# ps -auxww | grep tacacs_daily_daemon +root 488 0.0 0.0 2388 464 ? S Jul01 0:00 /bin/sh /root/tacacs_daily_daemon +``` + + - Stop /root/tacacs_daily_daemon with kill command and PID +``` +root@0ce3f7bbb316:~# kill -9 488 +``` + + - Find PID of tac_plus server with following command, in this case PID is 502 +``` +root@0ce3f7bbb316:~# ps -auxww | grep "/usr/sbin/tac_plus .* -p 49" +root 502 0.0 0.0 5600 1880 ? S Jul01 0:00 /usr/sbin/tac_plus -d 88 -l /var/log/tac_plus_daily.log -C /etc/tac_plus_daily.conf -p 49 -G +``` + + - Stop tac_plus with kill command and PID +``` +root@0ce3f7bbb316:~# kill -9 502 +``` + +- How to start testbed TACACS server: + - If tacacs_daily_daemon not running, start it with following command: +``` +root@0ce3f7bbb316:~# /bin/sh /root/tacacs_daily_daemon & +[1] 3865 +root@0ce3f7bbb316:~# starting tac_plus for daily work +``` + - After tacacs_daily_daemon started, it will monitor tac_plus server bind to port 49, if not running will auto restart tac_plus. + +- How to find TACACS log + - testbed TACACS server log is saved in /var/log/tac_plus_daily.log + +- How to find TACACS config + - testbed TACACS server log is saved in /etc/tac_plus_daily.conf + - for the config file format, please check: https://shrubbery.net/tac_plus/ diff --git a/docs/testbed/README.testbed.VsSetup.md b/docs/testbed/README.testbed.VsSetup.md index 1485bff5df..f6eea3fab0 100644 --- a/docs/testbed/README.testbed.VsSetup.md +++ b/docs/testbed/README.testbed.VsSetup.md @@ -38,50 +38,110 @@ sudo -H ./setup-management-network.sh ## Download an VM image We currently support EOS-based or SONiC VMs to simulate neighboring devices in the virtual testbed, much like we do for physical testbeds. To do so, we need to download the image to our testbed host. +**Prepare folder for image files on testbed host** + +The location for storing image files on the testbed host is specified by the `root_path` variable in the `ansible/group_vars/vm_host/main.yml` file. Please update this variable to reflect the location you have planned for the testbed host. You can use +either an absolute path or a relative path. If you choose a relative path, it will be relative to the home directory of the user accessing the testbed host. + +For example, if `root_path` is set to `veos-vm`, the image location would be `/home/$USER/veos-vm/images/`. If `root_path` is set to `/data/veos-vm`, the image location would be `/data/veos-vm/images`. + +As you may have noticed, image files are usually stored under subfolder `images` of location determined by `root_path`. + +Example 1: +``` yaml +root_path: veos-vm +``` + +Example 2: +```yaml +root_path: /data/veos-vm +``` + ### Option 1: vEOS (KVM-based) image 1. Download the [vEOS image from Arista](https://www.arista.com/en/support/software-download) -2. Copy below image files to `~/veos-vm/images` on your testbed host: +2. Copy below image files to location determined by `root_path` on your testbed host: - `Aboot-veos-serial-8.0.0.iso` - `vEOS-lab-4.20.15M.vmdk` -### Option 2: cEOS (container-based) image (experimental) -#### Option 2.1: Download and import cEOS image manually -1. Download the [cEOS image from Arista](https://www.arista.com/en/support/software-download) -2. Import the cEOS image (it will take several minutes to import, so please be patient!) -``` -docker import cEOS-lab-4.25.5.1M.tar.xz ceosimage:4.25.5.1M -``` -After imported successfully, you can check it by 'docker images' -``` -$ docker images -REPOSITORY TAG IMAGE ID CREATED SIZE -ceosimage 4.25.5.1M fa0df4b01467 9 seconds ago 1.62GB -``` -**Note**: *For time being, the image might be updated, in that case you can't download the same version of image as in the instruction, -please download the corresponding version(following [Arista recommended release](https://www.arista.com/en/support/software-download#datatab300)) of image and import it to your local docker repository. -The actual image version that is needed in the installation process is defined in the file [ansible/group_vars/all/ceos.yml](../../ansible/group_vars/all/ceos.yml), make sure you modify locally to keep it up with the image version you imported.* +### Option 2: cEOS (container-based) image (recommended) + +1. **Prepare folder for image files on test server** + + Create a subfolder called `images` inside the `root_path` directory defined in `ansible/group_vars/vm_host/main.yml` file. For instance, if `root_path` is set to `veos-vm`, you should run the following command: + + ```bash + mkdir -p ~/veos-vm/images + ``` + +2. **Prepare the cEOS image file** + + #### Option 2.1: Manually download cEOS image + + 1. Obtain the cEOS image from [Arista's software download page](https://www.arista.com/en/support/software-download). + 2. Place the image file in the `images` subfolder located within the directory specified by the `root_path` variable in the `ansible/group_vars/vm_host/main.yml` file. + + Assuming you set `root_path` to `veos-vm`, you should run the following command: -**Note**: *Please also notice the type of the bit for the image, in the example above, it is a standard 32-bit image. Please import the right image as your needs.* -#### Option 2.2: Pull cEOS image automatically -1. Alternatively, you can host the cEOS image on a http server. Specify `vm_images_url` for downloading the image [here](https://github.com/sonic-net/sonic-mgmt/blob/master/ansible/group_vars/vm_host/main.yml#L2). + ```bash + cp cEOS64-lab-4.29.3M.tar ~/veos-vm/images/ + ``` + The Ansible playbook for deploying testbed topology will automatically use the manually prepared image file from this location. -2. If a SAS key is required for downloading the cEOS image, specify `ceosimage_saskey` in `sonic-mgmt/ansible/vars/azure_storage.yml`. + #### Option 2.2: Host the cEOS image file on a HTTP server + If you need to deploy VS setup on multiple testbed hosts, this option is more recommended. -If you want to skip downloading the image when the cEOS image is not imported locally, set `skip_ceos_image_downloading` to `true` in `sonic-mgmt/ansible/group_vars/all/ceos.yml`. Then, when the cEOS image is not locally available, the scripts will not try to download it and will fail with an error message. Please use option 1 to download and import the cEOS image manually. + 1. **Download the cEOS Image** + + Obtain the cEOS image from [Arista's software download page](https://www.arista.com/en/support/software-download). + + 2. **Host the cEOS Image** + + Host the cEOS image file on an HTTP server. Ensure that the image file is accessible via HTTP from the `sonic-mgmt` container running the testbed deployment code. For example, the URL might look like `http://192.168.1.10/cEOS64-lab-4.29.3M.tar`. + + 3. **Update the Ansible Configuration** + + Update the `ceos_image_url` variable in `ansible/group_vars/all/ceos.yml` with the URL of the cEOS image. This variable can be a single string for one URL or a list of strings for multiple URLs. + + The Ansible playbook will attempt to download the image from each URL in the list until it succeeds. Downloaded file is stored to `images` subfolder of the location determined by `root_path` variable in `ansible/group_vars/vm_host/main.yml`. For example if `root_path` is `/data/veos-vm`, then the downloaded image file is put to `/data/veso-vm/images` + + Variable `skip_ceos_image_downloading` in `ansible/group_vars/all/ceos.yml` also must be set to `false` if you wish ansible playbook to automatically try downloading cEOS image file. For example + ```yaml + ceos_image_url: http://192.168.1.10/cEOS64-lab-4.29.3M.tar + skip_ceos_image_downloading: false + ``` + Or: + ```yaml + ceos_image_url: + - http://192.168.1.10/cEOS64-lab-4.29.3M.tar + skip_ceos_image_downloading: false + ``` ### Option 3: Use SONiC image as neighboring devices -You need to prepare a sound SONiC image `sonic-vs.img` in `~/veos-vm/images/`. We don't support to download sound sonic image right now, but for testing, you can also follow the section [Download the sonic-vs image](##download-the-sonic-vs-image) to download an available image and put it into the directory `~/veos-vm/images` +You need to create a valid SONiC image named `sonic-vs.img` in the `~/veos-vm/images/` directory. Currently, we don’t support downloading a pre-built SONiC image. However, for testing purposes, you can refer to the section Download the sonic-vs image to obtain an available image and place it in the `~/veos-vm/images/` directory. ## Download the sonic-vs image -To run the tests with a virtual SONiC device, we need a virtual SONiC image. The simplest way to do so is to download the latest succesful build. -1. Download the sonic-vs image from [here](https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=master&platform=vs&target=target/sonic-vs.img.gz) +### 1. To run the tests with a virtual SONiC device, we need a virtual SONiC image. + +#### Option 1: Download sonic-vs image + +The simplest way to do so is to download the latest succesful build. Download the sonic-vs image from [here](https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=master&platform=vs&target=target/sonic-vs.img.gz) ``` wget "https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=master&platform=vs&target=target/sonic-vs.img.gz" -O sonic-vs.img.gz ``` -2. Unzip the image and copy it into `~/sonic-vm/images/` and also `~/veos-vm/images` +#### Option 2: Build sonic-vpp image + +Follow the instructions from [sonic-platform-vpp](https://github.com/sonic-net/sonic-platform-vpp?tab=readme-ov-file#building-a-kvm-vm-image) and build a **kvm** vm image. + +__Note: make sure you rename the vpp image to `sonic-vs.img`.__ + +``` +mv sonic-vpp.img.gz sonic-vs.img.gz +``` + +### 2. Unzip the image and copy it into `~/sonic-vm/images/` and also `~/veos-vm/images` ``` gzip -d sonic-vs.img.gz @@ -96,11 +156,12 @@ All testbed configuration steps and tests are run from a `sonic-mgmt` docker con 1. Run the `setup-container.sh` in the root directory of the sonic-mgmt repository: -``` +```bash cd sonic-mgmt ./setup-container.sh -n -d /data ``` + 2. (Required for IPv6 test cases): Follow the steps [IPv6 for docker default bridge](https://docs.docker.com/config/daemon/ipv6/#use-ipv6-for-the-default-bridge-network) to enable IPv6 for container. For example, edit the Docker daemon configuration file located at `/etc/docker/daemon.json` with the following parameters to use ULA address if no special requirement. Then restart docker daemon by running `sudo systemctl restart docker` to take effect. ```json @@ -112,6 +173,7 @@ cd sonic-mgmt } ``` + 3. From now on, **all steps are running inside the sonic-mgmt docker**, unless otherwise specified. @@ -147,7 +209,7 @@ become_user='root' become_ask_pass=False ``` -3. Modify `/data/sonic-mgmt/ansible/group_vars/vm_host/creds.yml` to use the username (e.g. `foo`) and password (e.g. `foo123`) you want to use to login to the host machine (this can be your username and sudo password on the host) +3. Modify `/data/sonic-mgmt/ansible/group_vars/vm_host/creds.yml` to use the username (e.g. `foo`) and password (e.g. `foo123`) you want to use to login to the host machine (this can be your username and sudo password on the host). For more information about credentials variables, see: [credentials management configuration](https://github.com/sonic-net/sonic-mgmt/blob/master/docs/testbed/README.new.testbed.Configuration.md#credentials-management). ``` vm_host_user: foo @@ -321,6 +383,8 @@ cd /data/sonic-mgmt/ansible ## Deploy minigraph on the DUT Once the topology has been created, we need to give the DUT an initial configuration. +(Optional) The connectivity to the public internet is necessary during the setup, if the lab env of your organization requires http/https proxy server to reach out to the internet, you need to configure to use the proxy server. It will automatically be leveraged on required steps (e.g. Docker daemon config for image pulling, APT configuration for installing packages). You can configure it in [`ansible/group_vars/all/env.yml`](https://github.com/sonic-net/sonic-mgmt/blob/master/ansible/group_vars/all/env.yml) + 1. Deploy the `minigraph.xml` to the DUT and save the configuration: ``` diff --git a/docs/testbed/README.testbed.WANSetup.md b/docs/testbed/README.testbed.WANSetup.md index 6f0e609dfa..a5b38c5b4c 100644 --- a/docs/testbed/README.testbed.WANSetup.md +++ b/docs/testbed/README.testbed.WANSetup.md @@ -30,33 +30,63 @@ First, we need to prepare the host where we will be configuring the virtual test ## Download an VM image We currently support IOS-XR-based, EOS-based or SONiC VMs to simulate neighboring devices in the virtual testbed, much like we do for physical testbeds. To do so, we need to download the image to our testbed host. -### Option 1: cEOS (container-based) image (experimental) -#### Option 1.1: Download and import cEOS image manually -1. Download the [cEOS image from Arista](https://www.arista.com/en/support/software-download) -2. Import the cEOS image (it will take several minutes to import, so please be patient!) +### Option 1: cEOS (container-based) image (recommended) -``` -docker import cEOS-lab-4.25.5.1M.tar.xz ceosimage:4.25.5.1M -``` -After imported successfully, you can check it by 'docker images' -``` -$ docker images -REPOSITORY TAG IMAGE ID CREATED SIZE -ceosimage 4.25.5.1M fa0df4b01467 9 seconds ago 1.62GB -``` -**Note**: *For time being, the image might be updated, in that case you can't download the same version of image as in the instruction, -please download the corresponding version(following [Arista recommended release](https://www.arista.com/en/support/software-download#datatab300)) of image and import it to your local docker repository. -The actual image version that is needed in the installation process is defined in the file [ansible/group_vars/all/ceos.yml](../../ansible/group_vars/all/ceos.yml), make sure you modify locally to keep it up with the image version you imported.* +1. **Prepare folder for image files on test server** -**Note**: *Please also notice the type of the bit for the image, in the example above, it is a standard 32-bit image. Please import the right image as your needs.* + Create a subfolder called `images` inside the `root_path` directory defined in `ansible/group_vars/vm_host/main.yml` file. For instance, if `root_path` is set to `veos-vm`, you should run the following command: -#### Option 1.2: Pull cEOS image automatically -1. Alternatively, you can host the cEOS image on a http server. Specify `vm_images_url` for downloading the image [here](https://github.com/sonic-net/sonic-mgmt/blob/master/ansible/group_vars/vm_host/main.yml#L2). + ```bash + mkdir -p ~/veos-vm/images + ``` -2. If a SAS key is required for downloading the cEOS image, specify `ceosimage_saskey` in `sonic-mgmt/ansible/vars/azure_storage.yml`. +2. **Prepare the cEOS image file** -If you want to skip downloading the image when the cEOS image is not imported locally, set `skip_ceos_image_downloading` to `true` in `sonic-mgmt/ansible/group_vars/all/ceos.yml`. Then, when the cEOS image is not locally available, the scripts will not try to download it and will fail with an error message. Please use option 1.1 to download and import the cEOS image manually. + #### Option 1.1: Manually download cEOS image + 1. Obtain the cEOS image from [Arista's software download page](https://www.arista.com/en/support/software-download). + 2. Place the image file in the `images` subfolder located within the directory specified by the `root_path` variable in the `ansible/group_vars/vm_host/main.yml` file. + + Assuming you set `root_path` to `veos-vm`, you should run the following command: + + ```bash + cp cEOS64-lab-4.29.3M.tar ~/veos-vm/images/ + ``` + The Ansible playbook for deploying testbed topology will automatically use the manually prepared image file from this location. + + #### Option 1.2: Host the cEOS image file on a HTTP server + If you need to deploy VS setup on multiple testbed hosts, this option is more recommended. + + 1. **Download the cEOS Image** + + Obtain the cEOS image from [Arista's software download page](https://www.arista.com/en/support/software-download). + + 2. **Host the cEOS Image** + + Host the cEOS image file on an HTTP server. Ensure that the image file is accessible via HTTP from the `sonic-mgmt` container running the testbed deployment code. For example, the URL might look like `http://192.168.1.10/cEOS64-lab-4.29.3M.tar`. + + 3. **Update the Ansible Configuration** + + Update the `ceos_image_url` variable in `ansible/group_vars/vm_host/ceos.yml` with the URL of the cEOS image. This variable can be a single string for one URL or a list of strings for multiple URLs. + + The Ansible playbook will attempt to download the image from each URL in the list until it succeeds. Downloaded file is stored to `images` subfolder of the location determined by `root_path` variable in `ansible/group_vars/vm_host/main.yml`. For example if `root_path` is `/data/veos-vm`, then the downloaded image file is put to `/data/veso-vm/images` + + Variable `skip_ceos_image_downloading` in `ansible/group_vars/vm_host/ceos.yml` also must be set to `false` if you wish ansible playbook to automatically try downloading cEOS image file. For example + ```yaml + ceos_image_url: http://192.168.1.10/cEOS64-lab-4.29.3M.tar + skip_ceos_image_downloading: false + ``` + Or: + ```yaml + ceos_image_url: + - http://192.168.1.10/cEOS64-lab-4.29.3M.tar + skip_ceos_image_downloading: false + ``` + +**Note** When downloading, the version specified above might be outdated or unavailable. Please check the [Arista recommended release](https://www.arista.com/en/support/software-download#datatab300) to obtain the latest recommended image and import it into your local Docker registry. +The actual image version that is needed in the installation process is defined in the file [ansible/group_vars/vm_host/ceos.yml](../../ansible/group_vars/vm_host/ceos.yml), make sure you modify locally to keep it up with the image version you imported.* + +**Note**: Please be aware of the image's CPU architecture (32 vs 64-bit). In the example above, it is a standard 64-bit cEOS image. Ensure you import the correct image according to your requirements. ### Option 2: Use Cisco image as neighboring devices You need to prepare a Cisco IOS-XR image `cisco-vs.img` in `~/veos-vm/images/`.The actual image version that is needed in the installation process is defined in the file [ansible/group_vars/vm_host/main.yml:cisco_image_filename](../../ansible/group_vars/vm_host/main.yml). We don't support to download cisco image automatically, you can download an available image from [Download Cisco image](https://software.cisco.com/download/home/282414851/type/280805694/release/7.6.2) and put it into the directory `~/veos-vm/images` @@ -147,6 +177,7 @@ In order to configure the testbed on your host automatically, Ansible needs to b ## Deploy multiple devices topology Now we're finally ready to deploy the topology for our testbed! Run the following command: +(Optional) The connectivity to the public internet is necessary during the setup, if the lab env of your organization requires http/https proxy server to reach out to the internet, you need to configure to use the proxy server. It will automatically be leveraged on required steps (e.g. Docker daemon config for image pulling, APT configuration for installing packages). You can configure it in [`ansible/group_vars/all/env.yml`](https://github.com/sonic-net/sonic-mgmt/blob/master/ansible/group_vars/all/env.yml) ### cEOS ``` diff --git a/docs/testbed/sai_quality/DeploySAITestTopologyWithSONiC-MGMT.md b/docs/testbed/sai_quality/DeploySAITestTopologyWithSONiC-MGMT.md index a01db5b59b..9acf1deae9 100644 --- a/docs/testbed/sai_quality/DeploySAITestTopologyWithSONiC-MGMT.md +++ b/docs/testbed/sai_quality/DeploySAITestTopologyWithSONiC-MGMT.md @@ -3,6 +3,11 @@ In this article, you will get to know how to use the sonic-mgmt docker to set up **Those commands need to be run within a sonic-mgmt docker, or you need to run them within a similar environment.** This section of the document described how to build a sonic-mgmt docker https://github.com/sonic-net/sonic-mgmt/blob/master/docs/testbed/README.testbed.VsSetup.md#setup-sonic-mgmt-docker + + + +(Optional) The connectivity to the public internet is necessary during the setup, if the lab env of your organization requires http/https proxy server to reach out to the internet, you need to configure to use the proxy server. It will automatically be leveraged on required steps (e.g. Docker daemon config for image pulling, APT configuration for installing packages). You can configure it in [`ansible/group_vars/all/env.yml`](https://github.com/sonic-net/sonic-mgmt/blob/master/ansible/group_vars/all/env.yml) + 1. install the sonic image in the DUT(device under test) for example ``` @@ -32,7 +37,7 @@ For example, we want to use the config `vms-sn2700-t1-lag`, then we need to chan - topo: t1 + topo: ptf32 ptf_image_name: docker-ptf-sai-mlnx -- ptf: ptf-unknown +- ptf: ptf_vms1-1 + ptf: ptf-docker-name ptf_ip: 10.255.0.178/24 ptf_ipv6: @@ -41,6 +46,7 @@ For example, we want to use the config `vms-sn2700-t1-lag`, then we need to chan **for the topo, if it ends with 64, then the topo should be ptf64, please change it according to the actual device port.** 4. deploy the new topology + ``` ./testbed-cli.sh -t testbed.yaml add-topo vms-sn2700-t1 password.txt ``` @@ -48,4 +54,4 @@ For example, we want to use the config `vms-sn2700-t1-lag`, then we need to chan 5. push the minigraph to dut ``` ./testbed-cli.sh -t testbed.yaml deploy-mg vms-sn2700-t1 str password.txt -``` +``` diff --git a/docs/testplan/BGP-4-Byte_AS_Translation.md b/docs/testplan/BGP-4-Byte_AS_Translation.md new file mode 100644 index 0000000000..496e9643eb --- /dev/null +++ b/docs/testplan/BGP-4-Byte_AS_Translation.md @@ -0,0 +1,56 @@ +- [Overview](#overview) + - [Scope](#scope) + - [Testbed](#testbed) +- [Setup configuration](#setup-configuration) +- [Test cases](#test-cases) + +# Test name + +BGP 4-Byte AS Translation + +## Overview + +The goal of this test is to verify that the 4-byte to 2-byte AS translation feature operates as expected. + +### Scope + +The test is targeting a running SONIC system with fully functioning configuration. The purpose of the test is to test for 4-Byte AS translation. + +### Related DUT CLI commands + +| Command | Comment | +| ------- | ------- | +|Configuration commands| +| N/A | | +|Show commands| +| show ip bgp summary | Dispaly current memory statistics, can be done with ipv6 too | + +### Related DUT configuration files + +N/A + +### Related SAI APIs + +N/A + +## Test structure +### Setup configuration + +This test requires BGP neighbors to be configured and established. + +### Configuration scripts + +N/A + +## Test cases +### Test case #1 - 4-byte AS Translation + +#### Test objective + +Have a single neighbor configured with 4-byte ASN. +Step 1: Configure DUT and neighbor with 4-Byte ASN +Step 2: Verify 4-byte BGP session between DUT and neighbor is established +Step 3: Verify BGP is established for 4-byte neighbor and down for 2-byte neighbors +Step 3: Configure DUT to use 2-byte local-ASN for 2-byte neighbors +Step 4: Verify BGP is now established for 4-byte neighbor AND 2-byte neighbors +Step 5: Verify 2-byte neighbors receive routes from upstream 4-byte routers diff --git a/docs/testplan/BGP-BBR.md b/docs/testplan/BGP-BBR.md index 243bc9868e..e3aad65ef1 100644 --- a/docs/testplan/BGP-BBR.md +++ b/docs/testplan/BGP-BBR.md @@ -8,7 +8,7 @@ The goal of the test to check that BBR feature works correctly. The feature is implemented on bgpcfgd. The bgpcfgd dynamicaly changes BGP configuration, which either enable or disabled BBR functionality. The BBR functionality is enablement to see DUT ASN in the routes aspath not more than once. ### Scope -The test is targeting a running SONIC system with fully functioning configuration. The purpose of the test is to test BBR feature, which includes bgpcfgd implementation and BGP. +The test is targeting a running SONIC system with fully functioning configuration. The purpose of the test is to test BBR feature and configuration, which includes bgpcfgd implementation and BGP. ### Testbed The test could run on t1 testbed in virtual switch environment. @@ -38,3 +38,10 @@ The test announces ipv4 and ipv6 routes from one of the T0s, and checks when DUT 2. Announce ipv4 and ipv6 routes fron one of the T0s to DUT. Each route must have patched aspath which contains DUT ASN once. 3. Check that DUT BGP rejected both routes to the routing table 4. Restore the BBR state how it was been before the test + +### Test case # 4 - BBR status remains consistent after config reload +1. Set the BBR status in the config_db to the value specified by the `bbr_status` parameters using the redis-cli command. +2. Save the configuration using the `sudo config save -y` command. +3. Reload the configuration using the `config_reload` function. +4. Retrieve the BBR status and verified it matches the expected `bbr_status` value. +5. Restore the BBR status how it was been before the test. diff --git a/docs/testplan/DHCP-relay-stress-test.md b/docs/testplan/DHCP-relay-stress-test.md new file mode 100644 index 0000000000..62be2fff18 --- /dev/null +++ b/docs/testplan/DHCP-relay-stress-test.md @@ -0,0 +1,77 @@ +# DHCP Relay Stress Test Plan +- [Overview](#overview) +- [Scope](#scope) +- [Scale / Performance](#scale--performance) +- [Related DUT CLI commands](#related-dut-cli-commands) +- [Test structure](#test-structure) + - [Configuration](#configuration) + - [Test cases](#test-cases) + - [Test case](#test-case) + + + +## Overview + +The purpose is to test the DHCP relay service can survive the max load that we lift to CPU. + +### Scope +--------- + +CoPP is to limit the rate of traffic sent to CPU, and then generating packets above this maximum rate, the test is to ensure the DHCP relay service can handle the maximum load without failure. + +### Scale / Performance +------------------- +1. Verfy DHCP Container Status. +2. CPU should remain within acceptable range, indicating the DUT is not overwhelmed. +3. Check whether the function is still normal. +4. No errors in the logs related to DHCP or CoPP. + +### Related **DUT** CLI commands +---------------------------- + + +| **Command** | **Comment** | +|------------------------------------------------------------------|-------------| +| docker ps \| grep dhcp | check the status of DHCP container | +| show processes cpu --verbose \| grep dhcp \| awk '{print $9}' | check the resource utilisation | +| docker exec dhcp_relay supervisorctl status \| grep dhcp \| awk '{print $2}' | verify the status of processes in dhcp are running | +| loganalyzer | check no error keywords related to DHCP and CoPP | + +## Test structure +=============== + +### Configuration +------------------- + +1) def setup_copp_policy(): define a CoPP policy on the test device, and limit DHCP traffic to a maximum rate 600. +2) def client_send_discover(): simulate client sending DHCPDISCOVER message from different source MAC by broadcast. Duration: 120s. e.g. PTF will send 10,000 packets per second. +3) def server_send_offer(): simulate the server sending DHCPOFFER message to the client. At a rate up to 600 packets per second with a tolerance of 10%. +4) def client_send_request(): simulate client sending DHCPREQUEST message from different source MAC or through different interface. +5) def server_send_ack(): simulate the server sending DHCPACK message to the client under the same conditions (up tp 600 packets per second with a tolerance of 10%). +6) def verify_dhcp_container_alive(): verify the DHCP container is still alive. +7) def verify_cpu_utilisation(): verify the %cpu is within an acceptable range, the acceptable range will be adjusted after checking the device. +8) def verify_supervisor_process_running(): verify the status of processes managed by Supervisor is normal. +9) def verify_relay_packet_count_with_delay(): check DHCP relay packet count within a reasonable delay time range. Note that the packet count for DHCP offer and ack messages is around 72,000 packets (600 packets per second with 2 minutes) with a tolerance of 10%, and the packet count for DHCP discover and request messages should be 1,200,000 packets (10,000 packets per second for 120 seconds). +10) No errors in the logs related to DHCP or CoPP, just using loganalyzer. + +Test cases +---------- + +### Test case + +Test objective: To test the DHCP relay service can survive the max load that we lift to CPU. + +Test steps: +1) Configure CoPP policy +2) Clients broadcast a discover message +3) The DHCP server sends a offer message to client +4) Clients broadcast a request message +5) The DHCP server sends a offer message + +| **\#** | **Test Description** | **Expected Result** | +|--------|----------------------|---------------------| +| a. | verify_dhcp_container_alive() | Ensure the DHCP container is running by using the command ```docker ps \| grep dhcp```. | +| b. | verify_cpu_utilisation() | Verify the cpu is less than the maximum allowed CPU usage percentage. | +| c. | verify_supervisor_process_running()| Verify that all DHCP-related processes are running normally.| +| d. | verify_relay_packet_count_with_delay() | The function checks the packet count for DHCP messages within a reasonable delay. | +| e. | loganaylzer | Ensure there are no errors in the logs. | diff --git a/docs/testplan/LLDP-syncd-test-plan.md b/docs/testplan/LLDP-syncd-test-plan.md new file mode 100644 index 0000000000..94512ed36a --- /dev/null +++ b/docs/testplan/LLDP-syncd-test-plan.md @@ -0,0 +1,51 @@ +# Test Plan: Verification of `LLDP_ENTRY_TABLE` in SONiC `APPL_DB` + +## Objective +To verify that the `LLDP_ENTRY_TABLE` entries in the SONiC `APPL_DB` correctly reflect the LLDP information for all interfaces and are consistent with the output of `lldpctl -f json` under various conditions. +`LLDP_ENTRY_TABLE` will be used for SONiC SNMP, the data accuracy is important. + +## Test Scenarios + +### 1. Verify Presence of All Interfaces in `LLDP_ENTRY_TABLE` +- **Objective**: Ensure that all interfaces present in the system have corresponding entries in the `LLDP_ENTRY_TABLE`. +- **Steps**: + 1. Execute the command `sonic-db-cli APPL_DB keys 'LLDP_ENTRY_TABLE:*'`. + 2. Compare the list of interfaces in `LLDP_ENTRY_TABLE` with the expected list of system interfaces. +- **Expected Result**: Every active interface in the system should have a corresponding entry in the `LLDP_ENTRY_TABLE`. + +### 2. Verify `LLDP_ENTRY_TABLE` Content Against `lldpctl` Output +- **Objective**: Ensure that the content of each interface's `LLDP_ENTRY_TABLE` entry matches the output of `lldpctl -f json`. +- **Steps**: + 1. For each interface, retrieve the LLDP information using `sonic-db-cli APPL_DB hgetall LLDP_ENTRY_TABLE:`. + 2. Retrieve the LLDP information using `lldpctl -f json` and parse the output. + 3. Compare the data from `LLDP_ENTRY_TABLE` with the corresponding data in the `lldpctl -f json` output. +- **Expected Result**: The data in `LLDP_ENTRY_TABLE` should match the data from `lldpctl -f json` for each interface. + +### 3. Verify Interface Flap Handling +- **Objective**: Ensure that `LLDP_ENTRY_TABLE` entries are correctly updated after an interface flap. +- **Steps**: + 1. Simulate an interface flap by running `shutdown` and `no shutdown` commands on an interface. + 2. Repeat tests from scenarios 1 and 2. +- **Expected Result**: The `LLDP_ENTRY_TABLE` should update correctly after the interface flap, and the entries should still match the output of `lldpctl -f json`. + +### 4. Verify Behavior After LLDP Service Restart +- **Objective**: Ensure that `LLDP_ENTRY_TABLE` entries are correctly updated after restarting the LLDP service. +- **Steps**: + 1. Restart the LLDP service using the appropriate command. + 2. Repeat tests from scenarios 1 and 2 after the LLDP service has restarted. +- **Expected Result**: The `LLDP_ENTRY_TABLE` entries should be updated correctly after the LLDP service restart and should match the output of `lldpctl -f json`. + +### 5. Verify Behavior After System Reboot +- **Objective**: Ensure that `LLDP_ENTRY_TABLE` entries are preserved and accurate after a system reboot. +- **Steps**: + 1. Reboot the SONiC device. + 2. Repeat tests from scenarios 1 and 2 after the system has fully rebooted. +- **Expected Result**: The `LLDP_ENTRY_TABLE` entries should persist across reboots and match the `lldpctl -f json` output. + +## Test Data +- **APPL_DB Commands**: `sonic-db-cli APPL_DB keys`, `sonic-db-cli APPL_DB hgetall` +- **LLDP Command**: `lldpctl -f json` +- **Interfaces**: List of interfaces to be tested, retrieved dynamically from the device. + +## Conclusion +This test plan outlines the steps required to verify that the `LLDP_ENTRY_TABLE` in SONiC's `APPL_DB` is correctly populated, updated, and persistent under various conditions. The expected outcomes should confirm that the `LLDP_ENTRY_TABLE` is in sync with the LLDP information reported by the `lldpctl` command. diff --git a/docs/testplan/PFC_Snappi_Additional_Testcases.md b/docs/testplan/PFC_Snappi_Additional_Testcases.md new file mode 100644 index 0000000000..f5cca953fe --- /dev/null +++ b/docs/testplan/PFC_Snappi_Additional_Testcases.md @@ -0,0 +1,65 @@ +This document describes the list of additional system testcases. + +- [Background](#background) +- [Setup](#setup) +- [Testcases](#testcases) + + +### Background + +Intent of these testcases is to test throughput at the interface speed, PFC, PFC-WD, ECN congestion, port-channels and MACSEC. These testcases will be executed for 100 and 400Gbps ports on single line-card single asic, single line-card multiple asic and multiple line-card. The packet-size will vary from 128 to 1400 bytes. + +Test cases are specifically meant for Multi-ASIC, DNX based chassis platform. + +Traffic pattern includes at least one lossless priority flow and with/without lossy background traffic. Some of the testcases use pause flows to simulate congestion. + +Additionally, the testcases also intend to capture the utilization, packet-drops and/or latency. + + +### Setup + +Example for 100Gbps: + + ________________ + | | + IXIA(100Gbps) <---------> | DUT-LC1 |<---------> IXIA(100Gbps) + IXIA(100Gbps) <---------> | ASIC0 |<---------> IXIA(100Gbps) + |_______________| + |_______________| + | | + IXIA(100Gbps) <---------> | DUT-LC1 | + IXIA(100Gbps) <---------> | ASIC1 | + |_______________| + + ________________ + | | + IXIA(100Gbps) <---------> | DUT-LC2 |<---------> IXIA(100Gbps) + | ASIC0 | + |_______________| + + +There will be similar setup on a different line-cards for 400Gbps in the same chassis. + +### Testcases + +| Test case ID | Testcase Groups | Description | Topology | Traffic Pattern and Settings | LC combination | Pass-Fail Criteria | +|-------------:|:-------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 01 | NON-CONGESTION(NORMAL) | **NO CONGESTION AT LINE RATE TEST:**
Aim of the test is to ensure that there are no packet drops with equal speed ingress and egress links and mix of lossless and lossy traffic. No packet drops for both lossless and lossy priority traffic. | - Single 100Gbps Ingress and egress
- Repeat with 400Gbps ingress and egress port
- Frame size: IMIX packet-profile. | - 90% of ingress interface speed - Lossless Priority 3 and 4
- 10% of ingress interface speed - Lossy Background Flow [Priority 0, 1, and 2] | - Single LC - Multiple ASIC
- Single LC - Single ASIC
- Multiple LC | 1. There should be NO packet drops on the DUT (Tool Tx packets == Tool Rx packets)
2. There should no PFCs on tool Tx port or on the DUT.
3. Measure latency of the packets traversing the DUT.
4. Expected throughput of linerate on the egress port of DUT. | +| 02 | NON-CONGESTION(MACSEC) | **NO CONGESTION LINE-RATE OVER MACSEC ENABLED LINK:**
Test case to ensure that there are no packet drops with equal speed ingress and egress links with MACSEC enabled and mix of lossless and lossy traffic. | - Single 100Gbps Ingress and single 100Gbps egress - MACSEC
- Repeat with Single 400Gbps ingress and Single 400Gbps egress - MACSEC.
- Repeat with IPv6
- Frame size: IMIX packet-profile. | - 90% - Lossless Priority 3 and 4
- 10% - Lossy Background Flow [Priority 0, 1, and 2] | - Single LC - Multiple ASIC
- Multiple LC | 1. There should be NO packet drops on the DUT (IXIA Tx packets == IXIA Rx packets)
2. There should no PFCs on IXIA Tx port or on the DUT.
3. Measure latency of the packets traversing the DUT.
4. Expected throughput of linerate on the egress port of DUT. | +| 03 | CONGESTION | **DETECT CONGESTION WITH MISMATCHED INGRESS AND EGRESS:**
Aim of the test is ensure that congestion with ingress 400Gbps link and single 100Gbps egress link and mix of lossless and lossy traffic.No losses for lossless priorities (Priority 3 and 4) traffic streams. | - Single 400Gbps Ingress and single 100Gbps egress
- Repeat with IPv6
- 1024B packet-size | - 90% of 400Gbps - Lossless Priority 3 and 4
- 10% of 400Gbps - Lossy Background Flow [Priority 0, 1, and 2] | - Single LC - Multiple ASIC
- Multiple LC | 1. The single 100Gbps egress should see 100Gbps lossy+lossless traffic
2. The 40Gbps lossy traffic should be continue to flow.
3. The traffic transmitter port should be receiving PFCs for the lossless priority traffic.
4. On receiving the PFCs, the IXIA-ingress should reduce the lossless traffic to around 60Gbps.
5. No drop for lossy and lossless traffic. | +| 04 | CONGESTION | **DETECT CONGESTION WITH REAL-LIFE TRAFFIC PATTERN - 90% LOSSLESS and 10% LOSSY:**
Aim of the testcase is to ensure that congestion is detected with two ingress and single egresses of same speed and mix of lossless and lossy traffic. No losses for lossless priority traffic (Priority 3 and priority 4 streams). | - Two 100Gbps Ingress and single 100Gbps egress
- Repeat test with 2x400Gbps ingress and 1x400Gbps egress
- IMIX packet-profile
- Unequal traffic lossy and lossless priorities. | - 90% - Lossless Priority 3 and 4 per ingress LC
- 10% - Lossy Background Flow [Priority 0, 1, and 2] per ingress LC. | - Single LC - Multiple ASIC
- Single LC - Single ASIC
- Multiple LC | 1. No packet drops for Lossy Traffic.
2. PFCs for lossless traffic sent from DUT ingress to traffic transmitter. No packet drops for losslesss traffic.
3. Expected throughput of (Lossless + lossy traffic only) on egress interface line rate of DUT. | +| 05 | CONGESTION | **DETECT CONGESTION WITH EQUAL DISTRIBUTION OF LOSSLESS AND LOSSY TRAFFIC:**
Purpose of the testcase is to determine that amount of congestion faced by both lossless and lossy priority traffic. No losses for priority 3 and priority 4 traffic streams. | - Two 100Gbps Ingress and single 100Gbps egress
- Repeat test with 2x400Gbps ingress and 1x400Gbps egress
- IMIX packet-profile
- Equal traffic for all priorities | - 24% - Priority 3 and 4 per ingress LC
- 36% Background flows [Priority 0, 1, and 2] per ingress LC
- Enabled PFCWD and credit-watchdog on ingress and egress. | - Single LC - Multiple ASIC
- Multiple LC | 1. Packet drops for Lossy Traffic.
2. PFCs for lossless traffic. No packet drops for losslesss traffic.
3. Expected throughput of egress interface line-rate (Lossless + lossy traffic only) on egress interface of DUT.
4. Expected close to equal throughput for all priorities. | +| 06 | CONGESTION (MACSEC) | **CONGESTION DETECTION WITH PAUSE FRAMES OVER MACSEC ENABLED LINK:**
Purpose of the test is to ensure that congestion is detected on DUT egress with MACSEC enabled links on ingress and egress and mix of lossless and lossy traffic. No drops for lossless priority traffic. | - Single 100Gbps Ingress and Egress - PFC over MACSEC.
- Repeat with 400Gbps Ingress and Egress
- 1024B packet-size | - 90% - Lossless Priority 3 and 4 per ingress LC
- 10% - Lossy Background Flow [Priority 0, 1, and 2]
- Disable PFCWD before the test
- Send PAUSE frames to egress interface of DUT. | - Single LC - Multiple ASIC
- Single LC - Single ASIC
- Multiple LC | 1. There should be NO packet drops on the DUT (IXIA Tx packets == IXIA Rx packets)
2. There should be PFCs on IXIA and DUT - Tx and Rx port.
3. No Lossless Priority traffic on IXIA Rx Interface.
4. Only 10% lossy priority traffic allowed from the DUT.
5. No loss for Lossless traffic. | +| 07 | CONGESTION (MACSEC) | **CONGESTION OVER MISMATCHED INGRESS AND EGRESS MACSEC ENABLED LINK:**
Test case to check if the DUT detects congestion due to mismatched MACSEC enabled link speeds and sends PFCs to rate-limit the lossless priority traffic. | - Single 400Gbps Ingress and single 100Gbps egress
- Repeat with IPv6
- 1024B packet-size. | - 90% - ingress line rate Priority 3 and 4
- 10% of ingress line-rate -- Background Flow[Priority 0, 1, and 2] | - Single LC - Multiple ASIC
- Multiple LC | 1. There should be NO packet drops on the DUT for both Lossless and lossy priority traffic.
2. DUT should send PFCs to the traffic generator to rate-limit the lossless priority traffic.
3. 100Gbps Egress port should receive Lossless Priority 3 and 4 traffic. | +| 08 | CONGESTION (PC) | **UNEQUAL SPEED INGRESS-EGRESS PORTCHANNEL CONGESTION TEST:**
Test case to check that DUT detects congestion due to mismatched ingress and egress port-channel speeds and send PFCs to the ingress to rate-limit the lossless priority traffic. No drops for the lossless priority traffic. | - Port-channel - N x 100Gbps Ingress - 400Gbps Egress with N > 4.
- 1024B packet-size. | - 90% of total ingress port-channel link- Priority 3 and 4
- 10% of ingress port-channel link- Background Flow[Prio0, 1, and 2] | - Single LC - Multiple ASIC
- Multiple LC | 1. The lossless Prio3-4 traffic should not experience drop.
2. Ensure that background flows experience drop.
3. DUT sends PFCs to traffic generator to rate-limit the lossless prio traffic. | +| 09 | CONGESTION (PC + MACSEC) | **CONGESTION DETECTION ON PORT FAILURE ON PORTCHANNEL WITH MACSEC ENABLED:**
Test case to verify that DUT detects congestion on egress port-channel link on port-failure with MACSEC enabled on port-channel. | - Minimum of Two x 100Gbps ingress and egress port with MACSEC and PFC-WD enabled.
- Min port set to 1 for this test
- Repeat with minimum 2x400Gbps ingress and egress ports as port-channel members.
- 1024B packet-size. | - 90% - Lossless Priority 3 and 4
- 10% - Lossy Background Flow[Prio0, 1, and 2]
- Shutdown one of the egress ports.
- Shutting one of egress port causing congestion on other egress port.
- Bring back the egress port online again
Repeat in loop. | - Single LC - Multiple ASIC
Multiple LC | 1. Line-rate should be 200Gbps without any PFCs or packet drops.
2. When one of egress ports is shut, PFCs should be sent to the ingress to slow down the incoming traffic.
3. The other egress port should receive all the Lossless Priority 3 and 4 traffic.
4. Miniscule drop for both lossless and lossy traffic while traffic is shifting to other egress port.
5. When egress port is back-online, the traffic on egress ports should switch back with no loss or drops. | +| 10 | PFCWD | **PFCWD ENABLED DROP MODE TEST:**
Purpose of the test case is to ensure that lossless priorities are dropped when PFCWD is enabled and egress ports faces PFC storm. DUT does not send PFCs out of ingress port and no drops for lossy traffic. | - Two 100Gbps Ingress and single 100Gbps egress
- Repeat test with 2x400Gbps ingress and 1x400Gbps egress
-Repeat with IPv6.
- 1024B packet-size | - 30% - Lossless Priority 3 and 4 per ingress LC
- 10% Lossy Background Flow [Priority 0, 1, and 2] per ingress LC.
- Enable PFCWD with "action" set to DROP(default)
- Send continuous PFC for lossless Priorities for configured interval. | - Single LC - Multiple ASIC
- Single LC - Single ASIC
- Multiple LC | 1. No lossy background flows dropped on DUT when the PFC is sent to DUT egress. Egress port is in 'stormed' mode.
2. Lossless flows are dropped on DUT.
3. PFCWD drop counter should increment.
4. No PFCs are sent from DUT ingress to traffic transmitter.
5. After the PFC packets are stopped, the egress port is moves to "operational state" and storm-restored counter increments. | +| 11 | PFCWD | **PFCWD ENABLED FORWARD MODE TEST:**
Test case to ensure that both lossless and lossy priority traffic are forwarded when PFCWD is enabled in forward mode and egress ports faces PFC storm. DUT does not send PFCs out of ingress port and no drops for both lossy and lossless traffic. | - Two 100Gbps Ingress and single 100Gbps egress
- Repeat test with 2x400Gbps ingress and 1x400Gbps egress
- 1024B packet-size | - 30% - Lossless Priority 3 and 4 per ingress LC
- 10% Lossy Background Flow [Priority 0, 1, and 2] per ingress LC.
- Enable PFCWD with "action" set to FORWARD
- Send continuous PFC for lossless Priorities for configured interval. | - Single LC - Multiple ASIC
- Single LC - Single ASIC
- Multiple LC | 1. There should be no packet drops on DUT for lossy background flows.
2. The Priority 3 and 4 packets are forwarded to the Traffic Receiver.
3. After the PFC packets are stopped, the lossless PRIO traffic should continue without any drops.
4. No PFCs sent from DUT ingress to traffic transmitter.
5. PFCWD drop counters should not increment. | +| 12 | PFCWD (MACSEC) | **TEST PFCWD BEHAVIOR OVER MACSEC ENABLED LINK:**
Aim of the test case is to ensure that PFCWD drops the lossless priority traffic on detecting PFC storm on MACSEC enabled DUT and mix of lossless and lossy traffic. | - Single 100Gbps Ingress and single 100Gbps egress - PFCWD(Drop) over MACSEC
- Repeat with Single 400Gbps ingress and Single 400Gbps egress.
- 1024B packet-size. | - 90% - Lossless Priority 3 and 4
- 10% - Lossy Background Flow [Priority 0, 1, and 2]
3. Enabled PFCWD in default mode before the test.
4. Send pause to the egress interface of DUT from IXIA Rx. | - Single LC - Multiple ASIC
- Multiple LC | 1. There should be packet drops on the DUT (IXIA Tx packets > IXIA Rx packets).
2. There should PFCs on DUT egress but no PFC sent from DUT ingress to traffic transmitter.
3. Only 10% of lossy traffic should flow.
4. PFCWD drop counter should increment
5. DUT Rx port should be in 'stormed' mode. | +| 13 | ECN | **APPROPRIATE ECN MARKING ON CONGESTION RESTORATION:**
Aim of the test case is to ensure that DUT marks the lossless priority traffic with appropriate ECN flags when congestion is detection and restoration. | - Single 100Gbs ingress and single 100Gbps egress
- Repeat with 400Gbps ingress and egress.
- 1024B packet-size
- ECN congestion and packet-marking test.
| - Disable PFCWD
- 90% - Lossless Priority 3 and 4
- 10% - Lossy Background Flow [Priority 0, 1, and 2]
- Send PFC storm to the egress implying ECN congestion.
- Stop the PFC storm.
- Monitor set of packets to ensure packets are ECN-marked. | - Single LC - Multiple ASIC
- Multiple LC | 1. PFC storm should stop the packets coming DUT ingress.
2. No packets going out of egress port during storm.
3. The captured packet on egress, once storm is stopped should be ECN-marked. | + + + + + + diff --git a/docs/testplan/Smartswitch-test-plan.md b/docs/testplan/Smartswitch-test-plan.md new file mode 100644 index 0000000000..3083f9f8a7 --- /dev/null +++ b/docs/testplan/Smartswitch-test-plan.md @@ -0,0 +1,821 @@ +# Test plan for Smartswitch + +- [Introduction](#introduction) +- [Scope](#scope) +- [Definitions and Abbreviations](#definitions-and-abbreviations) +- [Objectives of CLI Test Cases](#objectives-of-cli-test-cases) +- [CLI Test Cases](#cli-test-cases) + - [1.1 Check DPU Status](#11-check-dpu-status) + - [1.2 Check platform voltage](#12-check-platform-voltage) + - [1.3 Check platform temperature](#13-check-platform-temperature) + - [1.4 Check DPU console](#14-check-DPU-console) + - [1.5 Check midplane ip address between NPU and DPU](#15-check-midplane-ip-address-between-npu-and-dpu) + - [1.6 Check DPU shutdown and power up individually](#16-check-DPU-shutdown-and-power-up-individually) + - [1.7 Check removal of pcie link between NPU and DPU](#17-check-removal-of-pcie-link-between-npu-and-dpu) + - [1.8 Check the NTP date and timezone between DPU and NPU](#18-check-the-ntp-date-and-timezone-between-dpu-and-npu) + - [1.9 Check the State of DPUs](#19-check-the-state-of-dpus) + - [1.10 Check the Health of DPUs](#110-check-the-health-of-dpus) + - [1.11 Check reboot cause history](#111-check-reboot-cause-history) + - [1.12 Check the DPU state after OS reboot](#112-check-the-dpu-state-after-os-reboot) +- [Objectives of API Test Cases](#objectives-of-api-test-cases) +- [API Test Cases](#api-test-cases) + - [1.1 Check SmartSwitch specific ChassisClass APIs](#11-check-smartswitch-specific-chassisclass-apis) + - [1.2 Check modified ChassisClass APIs for smartswitch](#12-check-modified-chassisclass-apis-for-smartswitch) + - [1.3 Check DpuModule APIs for SmartSwitch](#13-check-dpumodule-apis-for-smartswitch) + - [1.4 Check modified ModuleClass APIs](#14-check-modified-moduleclass-apis) + +## Introduction + +The purpose is to test the functionality of Smartswitch. +Smartswitch is connected to DPUs via pcie links. + +## Scope + +The test is targeting a running SONIC on Switch and SONIC-DASH system on each DPUs. +Purpose of the test is to verify smartswich platform related functionalities/features for each DPUs and PMON APIs. +For every test cases, all DPUs need to be powered on unless specified in any of the case. +General convention of DPU0, DPU1, DPU2 and DPUX has been followed to represent DPU modules and the number of DPU modules can vary. + +## Definitions and Abbreviations + +| **Term** | **Meaning** | +| ---------- | ---------------------------------------- | +| DPU | Data Processing Unit | +| NPU | Network Processing Unit | +| NTP | Network Time Protocol | +| SWITCH | Refers to NPU and the anything other than DPUs | +| SS | SmartSwitch | + +## Objectives of CLI Test Cases + +| | **Test Case** | **Intention** | **Comments** | +| ---------- | ---------- | ---------------------------------------- | ---------- | +| 1.1 | Check DPU Status | To verify the DPU Status shown in the cli | | +| 1.2 | Check platform voltage | To verify the Voltage sensor values and and functionality of alarm by changing the threshold values | | +| 1.3 | Check platform temperature | To Verify the Temperature sensor values and functionality of alarm by changing the threshold values | | +| 1.4 | Check DPU console | To Verify console access for all DPUs | | +| 1.5 | Check midplane ip address between NPU and DPU | To Verify PCIe interface created between NPU and DPU according to bus number | | +| 1.6 | Check DPU shutdown and power up individually | To Verify DPU shutdown and DPUs power up | | +| 1.7 | Check removal of pcie link between NPU and DPU | To Verify the PCie hot plug functinality | | +| 1.8 | Check the NTP date and timezone between DPU and NPU | To Verify NPU and DPU are in sync with respect to timezone and logs timestamp | | +| 1.9 | Check the State of DPUs | To Verify DPU state details during online and offline | | +| 1.10 | Check the Health of DPUs | To Verify overall health (LED, process, docker, services and hw) of DPU | Phase:2 | +| 1.11 | Check reboot cause history | To Verify reboot cause history cli | | +| 1.12 | Check the DPU state after OS reboot | To Verify DPU state on host reboot | | + + +## CLI Test Cases + +### 1.1 Check DPU Status + +#### Steps + * Use command `show chassis modules status` to get DPU status + * Get the number of DPU modules from ansible inventory file for the testbed. + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: +root@sonic:/home/cisco# show chassis modules status + Name Description Physical-Slot Oper-Status Admin-Status Serial +------ -------------------- --------------- ------------- -------------- -------------- + DPU0 Data Processing Unit N/A Online up 154226463179136 + DPU1 Data Processing Unit N/A Online up 154226463179152 + DPU2 Data Processing Unit N/A Online up 154226463179168 + DPUX Data Processing Unit N/A Online up 154226463179184 +``` +#### Pass/Fail Criteria + * Verify number of DPUs from inventory file for the testbed and number of DPUs shown in the cli output. + + +### 1.2 Check platform voltage + +#### Steps + * Use command `show platform voltage` to get platform voltage + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: + +root@sonic:/home/cisco# show platform voltage + Sensor Voltage High TH Low TH Crit High TH Crit Low TH Warning Timestamp +------------------------ --------- --------- -------- -------------- ------------- --------- ----------------- + A1V2_BB 1211 mV 1308 1092 1320 1080 False 20230619 11:31:08 + A1V8_BB 1810 mV 1962 1638 1980 1620 False 20230619 11:31:07 + A1V_BB 1008 mV 1090 910 1100 900 False 20230619 11:31:06 + A1V_CPU 1001 mV 1090 910 1100 900 False 20230619 11:31:51 + A1_2V_CPU 1209 mV 1308 1092 1320 1080 False 20230619 11:31:52 + A1_8V_CPU 1803 mV 1962 1638 1980 1620 False 20230619 11:31:51 + A2_5V_CPU 2517 mV 2725 2275 2750 2250 False 20230619 11:31:52 + A3P3V_CPU 3284 mV 3597 3003 3603 2970 False 20230619 11:31:53 + A3V3_BB 3298 mV 3597 3003 3630 2970 False 20230619 11:31:08 + GB_CORE_VIN_L1_BB 12000 mV 13800 10200 14400 9600 False 20230619 11:31:06 + GB_CORE_VOUT_L1_BB 824 mV N/A 614 918 608 False 20230619 11:31:50 + GB_P1V8_PLLVDD_BB 1812 mV 1962 1638 1980 1620 False 20230619 11:31:11 + GB_P1V8_VDDIO_BB 1815 mV 1962 1638 1980 1620 False 20230619 11:31:11 + GB_PCIE_VDDACK_BB 755 mV 818 683 825 675 False 20230619 11:31:12 + GB_PCIE_VDDH_BB 1208 mV 1308 1092 1320 1080 False 20230619 11:31:12 + P3_3V_BB 3330 mV 3597 3003 3630 2970 False 20230619 11:31:12 + P5V_BB 5069 mV 5450 4550 5500 4500 False 20230619 11:31:07 + P12V_CPU 12103 mV 13080 10920 13200 10800 False 20230619 11:31:54 + P12V_SLED1_VIN 12048 mV N/A N/A 12550 11560 False 20230619 11:31:13 + P12V_SLED2_VIN 12048 mV N/A N/A 12550 11560 False 20230619 11:31:13 + P12V_SLED3_VIN 12079 mV N/A N/A 12550 11560 False 20230619 11:31:13 + P12V_SLED4_VIN 12043 mV N/A N/A 12550 11560 False 20230619 11:31:13 + P12V_STBY_CPU 12103 mV 13080 10920 13200 10800 False 20230619 11:31:54 + P12V_U1_VR3_CPU 11890 mV 13800 10200 14400 9600 False 20230619 11:31:54 + P12V_U1_VR4_CPU 11890 mV N/A N/A N/A N/A False 20230619 11:31:54 + P12V_U1_VR5_CPU 11890 mV 13800 10200 14400 9600 False 20230619 11:31:54 + TI_3V3_L_VIN_BB 12015 mV N/A 10200 14400 9600 False 20230619 11:31:06 + TI_3V3_L_VOUT_L1_BB 3340 mV N/A 2839 4008 2672 False 20230619 11:31:13 + TI_3V3_R_VIN_BB 12078 mV N/A 10200 14400 9600 False 20230619 11:31:06 + TI_3V3_R_VOUT_L1_BB 3340 mV N/A 2839 4008 2672 False 20230619 11:31:13 + TI_GB_VDDA_VOUT_L2_BB 960 mV N/A 816 1152 768 False 20230619 11:31:13 + TI_GB_VDDCK_VIN_BB 12031 mV N/A 10200 14400 9600 False 20230619 11:31:06 + TI_GB_VDDCK_VOUT_L1_BB 1150 mV N/A 978 1380 920 False 20230619 11:31:13 + TI_GB_VDDS_VIN_BB 12046 mV N/A 10200 14400 9600 False 20230619 11:31:50 + TI_GB_VDDS_VOUT_L1_BB 750 mV N/A 638 900 600 False 20230619 11:31:12 + VP0P6_DDR0_VTT_DPU0 599 mV 630 570 642 558 False 20230619 11:31:55 + VP0P6_DDR0_VTT_DPU1 597 mV 630 570 642 558 False 20230619 11:31:56 + VP0P6_DDR0_VTT_DPU2 600 mV 630 570 642 558 False 20230619 11:31:58 + VP0P6_DDR0_VTT_DPUX 600 mV 630 570 642 558 False 20230619 11:31:59 + VP0P6_DDR1_VTT_DPU0 600 mV 630 570 642 558 False 20230619 11:31:56 + VP0P6_DDR1_VTT_DPU1 602 mV 630 570 642 558 False 20230619 11:31:57 + VP0P6_DDR1_VTT_DPU2 601 mV 630 570 642 558 False 20230619 11:31:58 + VP0P6_DDR1_VTT_DPUX 601 mV 630 570 642 558 False 20230619 11:32:00 + VP0P6_VTT_DIMM_CPU 597 mV 654 546 660 540 False 20230619 11:31:51 + VP0P8_AVDD_D6_DPU0 801 mV 840 760 856 744 False 20230619 11:31:16 + VP0P8_AVDD_D6_DPU1_ADC 806 mV 840 760 856 744 False 20230619 11:31:20 + VP0P8_AVDD_D6_DPU2 804 mV 840 760 856 744 False 20230619 11:31:25 + VP0P8_AVDD_D6_DPU3_ADC 805 mV 840 760 856 744 False 20230619 11:31:29 + VP0P8_AVDD_D6_DPU4 806 mV 840 760 856 744 False 20230619 11:31:34 + VP0P8_AVDD_D6_DPU5_ADC 801 mV 840 760 856 744 False 20230619 11:31:39 + VP0P8_AVDD_DX_DPUX 805 mV 840 760 856 744 False 20230619 11:31:44 + VP0P8_AVDD_D6_DPU7_ADC 806 mV 840 760 856 744 False 20230619 11:31:48 + VP0P8_NW_DPU0 803 mV 840 760 856 744 False 20230619 11:31:17 + VP0P8_NW_DPU1 804 mV 840 760 856 744 False 20230619 11:31:21 + VP0P8_NW_DPU2 803 mV 840 760 856 744 False 20230619 11:31:26 + VP0P8_NW_DPUX 804 mV 840 760 856 744 False 20230619 11:31:31 +VP0P8_PLL_AVDD_PCIE_DPU0 802 mV 840 760 856 744 False 20230619 11:31:56 +VP0P8_PLL_AVDD_PCIE_DPU1 804 mV 840 760 856 744 False 20230619 11:31:57 +VP0P8_PLL_AVDD_PCIE_DPU2 801 mV 840 760 856 744 False 20230619 11:31:59 +VP0P8_PLL_AVDD_PCIE_DPUX 802 mV 840 760 856 744 False 20230619 11:32:00 + VP0P9_AVDDH_D6_DPU0 906 mV 945 855 963 837 False 20230619 11:31:15 + VP0P9_AVDDH_D6_DPU1 908 mV 945 855 963 837 False 20230619 11:31:19 + VP0P9_AVDDH_D6_DPU2 907 mV 945 855 963 837 False 20230619 11:31:24 + VP0P9_AVDDH_D6_DPUX 908 mV 945 855 963 837 False 20230619 11:31:29 + VP0P9_AVDDH_PCIE_DPU0 901 mV 945 855 963 837 False 20230619 11:31:17 + VP0P9_AVDDH_PCIE_DPU1 903 mV 945 855 963 837 False 20230619 11:31:22 + VP0P9_AVDDH_PCIE_DPU2 901 mV 945 855 963 837 False 20230619 11:31:26 + VP0P9_AVDDH_PCIE_DPUX 903 mV 945 855 963 837 False 20230619 11:31:31 + VP0P75_PVDD_DPU0 752 mV 788 713 803 698 False 20230619 11:31:15 + VP0P75_PVDD_DPU1 756 mV 788 713 802 698 False 20230619 11:31:20 + VP0P75_PVDD_DPU2 756 mV 788 713 803 698 False 20230619 11:31:24 + VP0P75_PVDD_DPUX 755 mV 788 713 802 698 False 20230619 11:31:29 + VP0P75_RTVDD_DPU0 753 mV 788 713 803 698 False 20230619 11:31:14 + VP0P75_RTVDD_DPU1 755 mV 788 713 802 698 False 20230619 11:31:19 + VP0P75_RTVDD_DPU2 752 mV 788 713 803 698 False 20230619 11:31:24 + VP0P75_RTVDD_DPUX 755 mV 788 713 802 698 False 20230619 11:31:28 + VP0P82_AVDD_PCIE_DPU0 823 mV 861 779 877 763 False 20230619 11:31:18 + VP0P82_AVDD_PCIE_DPU1 823 mV 861 779 877 763 False 20230619 11:31:22 + VP0P82_AVDD_PCIE_DPU2 822 mV 861 779 877 763 False 20230619 11:31:27 + VP0P82_AVDD_PCIE_DPUX 822 mV 861 779 877 763 False 20230619 11:31:31 +root@sonic:/home/cisco# + +``` +#### Pass/Fail Criteria + * Verify warnings are all false. + * Verify changing the threshold values (high, low, critical high and critical low) and check alarm warning changing into True + * Verify changing back the threshold values to original one and check alarm warning changing into False + + +### 1.3 Check platform temperature + +#### Steps + * Use command `show platform temperature` to get platform temperature + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: + +root@sonic:~#show platform temperature + + Sensor Temperature High TH Low TH Crit High TH Crit Low TH Warning Timestamp +--------------- ------------- --------- -------- -------------- ------------- --------- ----------------- + DPU_0_T 37.438 100.0 -5.0 105.0 -10.0 False 20230728 06:39:18 + DPU_1_T 37.563 100.0 -5.0 105.0 -10.0 False 20230728 06:39:18 + DPU_2_T 38.5 100.0 -5.0 105.0 -10.0 False 20230728 06:39:18 + DPU_3_T 38.813 100.0 -5.0 105.0 -10.0 False 20230728 06:39:18 + FAN_Sensor 23.201 100.0 -5.0 102.0 -10.0 False 20230728 06:39:18 + MB_PORT_Sensor 21.813 97.0 -5.0 102.0 -10.0 False 20230728 06:39:18 +MB_TMP421_Local 26.25 135.0 -5.0 140.0 -10.0 False 20230728 06:39:18 + SSD_Temp 40.0 80.0 -5.0 83.0 -10.0 False 20230728 06:39:18 + X86_CORE_0_T 37.0 100.0 -5.0 105.0 -10.0 False 20230728 06:39:18 + X86_PKG_TEMP 41.0 100.0 -5.0 105.0 -10.0 False 20230728 06:39:18 +``` +#### Pass/Fail Criteria + * Verify warnings are all false + * Verify changing the threshold values (high, low, critical high and critical low) and check alarm warning changing into True + * Verify changing back the threshold values to original one and check alarm warning changing into False + + +### 1.4 Check DPU Console + +#### Steps + * Use serial port utility to access console for given DPU. + * Get the mapping of serial port to DPU number from platform.json file. + * Get the number of DPU modules from ansible inventory file for the testbed. Test is to check for console access for all DPUs. + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: (shows connection to dpu-4 console and offset is 4 for this case) + +root@sonic:/home/cisco# cat /proc/tty/driver/serial +serinfo:1.0 driver revision: +0: uart:16550A port:000003F8 irq:16 tx:286846 rx:6096 oe:1 RTS|DTR|DSR|CD|RI +1: uart:16550A port:00006000 irq:19 tx:0 rx:0 CTS|DSR|CD +2: uart:16550A port:000003E8 irq:18 tx:0 rx:0 DSR|CD|RI +3: uart:16550A port:00007000 irq:16 tx:0 rx:0 CTS|DSR|CD +**4: uart:16550 mmio:0x94040040 irq:94 tx:0 rx:0 +5: uart:16550 mmio:0x94040060 irq:94 tx:20 rx:68 +6: uart:16550 mmio:0x94040080 irq:94 tx:0 rx:0 +7: uart:16550 mmio:0x940400A0 irq:94 tx:0 rx:0 +8: uart:16550 mmio:0x940400C0 irq:94 tx:0 rx:0 +9: uart:16550 mmio:0x940400E0 irq:94 tx:0 rx:0 +10: uart:16550 mmio:0x94040100 irq:94 tx:0 rx:0 +11: uart:16550 mmio:0x94040120 irq:94 tx:0 rx:0** +12: uart:16550 mmio:0x94040140 irq:94 tx:0 rx:0 CTS|DSR +13: uart:16550 mmio:0x94040160 irq:94 tx:0 rx:0 CTS|DSR +14: uart:16550 mmio:0x94040180 irq:94 tx:0 rx:0 CTS|DSR +15: uart:16550 mmio:0x940401A0 irq:94 tx:0 rx:0 CTS|DSR + +root@sonic:/home/cisco# /usr/bin/picocom -b 115200 /dev/ttyS8 +picocom v3.1 + +port is : /dev/ttyS8 +flowcontrol : none +baudrate is : 115200 +parity is : none +databits are : 8 +stopbits are : 1 +escape is : C-a +local echo is : no +noinit is : no +noreset is : no +hangup is : no +nolock is : no +send_cmd is : sz -vv +receive_cmd is : rz -vv -E +imap is : +omap is : +emap is : crcrlf,delbs, +logfile is : none +initstring : none +exit_after is : not set +exit is : no + +Type [C-a] [C-h] to see available commands +Terminal ready + +sonic login: admin +Password: +Linux sonic 6.1.0-11-2-arm64 #1 SMP Debian 6.1.38-4 (2023-08-08) aarch64 +You are on + ____ ___ _ _ _ ____ + / ___| / _ \| \ | (_)/ ___| + \___ \| | | | \| | | | + ___) | |_| | |\ | | |___ + |____/ \___/|_| \_|_|\____| + +-- Software for Open Networking in the Cloud -- + +Unauthorized access and/or use are prohibited. +All access and/or use are subject to monitoring. + +Help: https://sonic-net.github.io/SONiC/ + +Last login: Fri Jan 26 21:49:12 UTC 2024 from 169.254.143.2 on pts/1 +admin@sonic:~$ +admin@sonic:~$ +Terminating... +Thanks for using picocom +root@sonic:/home/cisco# + +``` +#### Pass/Fail Criteria + * Verify Login access is displayed. + * cntrl+a and then cntrl+x to come out of the DPU console. + + +### 1.5 Check midplane ip address between NPU and DPU + +#### Steps + * Get the number of DPU modules from from ansible inventory file for the testbed. + * Get mid plane ip address for each DPU module from ansible inventory file for the testbed. + +#### Verify in + * Switch + +#### Sample Output +``` + On Switch: + + root@sonic:/home/cisco# show ip interface + Interface Master IPv4 address/mask Admin/Oper BGP Neighbor Neighbor IP + ------------ -------- ------------------- ------------ -------------- ------------- + eth0 172.25.42.65/24 up/up N/A N/A + bridge-midplane 169.254.200.254/24 up/up N/A N/A + lo 127.0.0.1/16 up/up N/A N/A + root@sonic:/home/cisco# +``` +#### Pass/Fail Criteria + * Verify Ping works to all the mid plane ip listed in the ansible inventory file for the testbed. + + +### 1.6 Check DPU shutdown and power up individually + +#### Steps + * Get the number of DPU modules from Ansible inventory file for the testbed. + * Use command `config chassis modules shutdown ` to shut down individual DPU + * Use command `show chassis modules status` to show DPU status + * Use command `config chassis modules startup ` to power up individual DPU + * Use command `show chassis modules status` to show DPU status + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: + +root@sonic:/home/cisco# config chassis modules shutdown DPU3 +root@sonic:/home/cisco# +root@sonic:/home/cisco# show chassis modules status + Name Description Physical-Slot Oper-Status Admin-Status Serial +------ -------------------- --------------- ------------- -------------- ---------------- + DPU0 Data Processing Unit N/A Online up 154226463179136 + DPU1 Data Processing Unit N/A Online up 154226463179152 + DPU2 Data Processing Unit N/A Online up 154226463179168 + DPU3 Data Processing Unit N/A Offline down 154226463179184 + +root@sonic:/home/cisco# config chassis modules startup DPU3 +root@sonic:/home/cisco# show chassis modules status + Name Description Physical-Slot Oper-Status Admin-Status Serial +------ ------------------- --------------- ------------- -------------- ---------------- + DPU0 Data Processing Unit N/A Online up 154226463179136 + DPU1 Data Processing Unit N/A Online up 154226463179152 + DPU2 Data Processing Unit N/A Online up 154226463179168 + DPU3 Data Processing Unit N/A Online up 154226463179184 + +``` +#### Pass/Fail Criteria + * Verify DPU offline in show chassis modules status after DPU shut down + * Verify DPU is shown in show chassis modules status after DPU powered on + + +### 1.7 Check removal of pcie link between NPU and DPU + +#### Steps + * Use `show platform pcieinfo -c` to run the pcie info test to check everything is passing + * Use command `config chassis modules shutdown DPU` to bring down the dpu (This will bring down the pcie link between npu and dpu) + * Use `show platform pcieinfo -c` to run the pcie info test to check pcie link has been removed + * Use command `config chassis modules startup DPU` to bring up the dpu (This will rescan pcie links) + * Use `show platform pcieinfo -c` to run the pcie info test to check everything is passing + * This test is to check the PCie hot plug functinality since there is no OIR possible + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: Showing example of one DPU pcie link + + +root@sonic:/home/cisco# show platform pcieinfo -c + +root@sonic:/home/cisco# echo 1 > /sys/bus/pci/devices/0000:1a:00.0/remove +root@sonic:/home/cisco# +root@sonic:/home/cisco# echo 1 > /sys/bus/pci/rescan +root@sonic:/home/cisco# +root@sonic:/home/cisco# show platform pcieinfo -c + +``` +#### Pass/Fail Criteria + * Verify after removing pcie link, pcie info test fail only for that pcie link. + * Verify pcieinfo test pass for all after bringing back up the link + + +### 1.8 Check the NTP date and timezone between DPU and NPU + +#### Steps + * Use command `date` to get date and time zone on Switch + * Use command `ssh admin@169.254.x.x` to enter into required dpu. + * Use command `date` to get date and time zone on DPU + +#### Verify in + * Switch and DPU + +#### Sample Output +``` +On Switch: + +root@sonic:/home/cisco# date +Tue 23 Apr 2024 11:46:47 PM UTC +root@sonic:/home/cisco# +. +root@sonic:/home/cisco# ssh admin@169.254.200.1 +root@sonic:/home/cisco# +. +On DPU: +root@sonic:/home/admin# date +Tue 23 Apr 2024 11:46:54 PM UTC +root@sonic:/home/cisco# + +``` +#### Pass/Fail Criteria + * Verify both the date and time zone are same + * Verify the syslogs on both switch and DPU to be same + * Verify by changing time intentionally in DPU and restart the DPU. Verify again for time sync + + +### 1.9 Check the State of DPUs + +#### Steps + * Use command `show system-health DPU all` to get DPU health status. + +#### Verify in + * Switch and DPU + +#### Sample Output +``` +On Switch: + +root@sonic:~#show system-health DPU all + +Name ID Oper-Status State-Detail State-Value Time Reason +DPU0 1 Online dpu_midplane_link_state up Wed 20 Oct 2023 06:52:28 PM UTC + dpu_booted_state up Wed 20 Oct 2023 06:52:28 PM UTC + dpu_control_plane_state up Wed 20 Oct 2023 06:52:28 PM UTC + dpu_data_plane_state up Wed 20 Oct 2023 06:52:28 PM UTC + + +DPU1 2 Online dpu_midplane_link_state up Wed 20 Oct 2023 06:52:28 PM UTC + dpu_booted_state up Wed 20 Oct 2023 06:52:28 PM UTC + dpu_control_plane_state up Wed 20 Oct 2023 06:52:28 PM UTC + dpu_data_plane_state up Wed 20 Oct 2023 06:52:28 PM UTC + +root@sonic:~#show system-health DPU 0 + +Name ID Oper-Status State-Detail State-Value Time Reason +DPU0 1 Offline dpu_midplane_link_state down Wed 20 Oct 2023 06:52:28 PM UTC + dpu_booted_state down Wed 20 Oct 2023 06:52:28 PM UTC + dpu_control_plane_state down Wed 20 Oct 2023 06:52:28 PM UTC + dpu_data_plane_state down Wed 20 Oct 2023 06:52:28 PM UTC + +root@sonic:~#show system-health DPU 0 + +Name ID Oper-Status State-Detail State-Value Time Reason +DPU0 1 Partial Online dpu_midplane_link_state up Wed 20 Oct 2023 06:52:28 PM UTC + dpu_booted_state up Wed 20 Oct 2023 06:52:28 PM UTC + dpu_control_plane_state up Wed 20 Oct 2023 06:52:28 PM UTC + dpu_data_plane_state down Wed 20 Oct 2023 06:52:28 PM UTC Pipeline failure + + +``` +#### Pass/Fail Criteria + * Verify the following criteria for Pass/Fail: + * Online : All states are up + * Offline: dpu_midplane_link_state or dpu_booted_state is down + * Partial Online: dpu_midplane_link_state is up and dpu_booted_state is up and dpu_control_plane_state is up and dpu_data_plane_state is down + * Verify powering down DPU and check for status and powering up again to check the status to show online. + + +### 1.10 Check the Health of DPUs + +#### NOTE + * This Test case is to be covered in Phase 2 + +#### Steps + * Use command `show system-health detail ` to check the health of the DPU. + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: + +root@sonic:/home/cisco# show system-health detail + +Device: DPU0 + +System status summary + + System status LED green + Services: + Status: Not OK + Not Running: container_checker, lldp + Hardware: + Status: OK + +system services and devices monitor list + +Name Status Type +------------------------- -------- ---------- +mtvr-r740-04-bf3-sonic-01 OK System +rsyslog OK Process + +``` +#### Pass/Fail Criteria + * Verify System Status - Green, Service Status - OK, Hardware Status - OK + * Stop any docker in DPU and check for Service Status - Not OK and that docker as Not running + * Start the docker again and Verify System Status - Green, Service Status - OK, Hardware Status - OK + + +### 1.11 Check reboot cause history + +#### Steps + * The "show reboot-cause" CLI on the switch shows the most recent rebooted device, time and the cause. + * The "show reboot-cause history" CLI on the switch shows the history of the Switch and all DPUs + * The "show reboot-cause history module-name" CLI on the switch shows the history of the specified module + * Use `config chassis modules shutdown ` + * Use `config chassis modules startup ` + * Wait for 5 minutes for Pmon to update the DPU states + * Use `show reboot-cause ` to check the latest reboot is displayed + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: + +root@sonic:~#show reboot-cause + +Device Name Cause Time User Comment + +switch 2023_10_20_18_52_28 Watchdog:1 expired; Wed 20 Oct 2023 06:52:28 PM UTC N/A N/A +DPU3 2023_10_03_18_23_46 Watchdog: stage 1 expired; Mon 03 Oct 2023 06:23:46 PM UTC N/A N/A +DPU2 2023_10_02_17_20_46 reboot Sun 02 Oct 2023 05:20:46 PM UTC admin User issued 'reboot' + +root@sonic:~#show reboot-cause history + +Device Name Cause Time User Comment + +switch 2023_10_20_18_52_28 Watchdog:1 expired; Wed 20 Oct 2023 06:52:28 PM UTC N/A N/A +switch 2023_10_05_18_23_46 reboot Wed 05 Oct 2023 06:23:46 PM UTC user N/A +DPU3 2023_10_03_18_23_46 Watchdog: stage 1 expired; Mon 03 Oct 2023 06:23:46 PM UTC N/A N/A +DPU3 2023_10_02_18_23_46 Host Power-cycle Sun 02 Oct 2023 06:23:46 PM UTC N/A Host lost DPU +DPU3 2023_10_02_17_23_46 Host Reset DPU Sun 02 Oct 2023 05:23:46 PM UTC N/A N/A +DPU2 2023_10_02_17_20_46 reboot Sun 02 Oct 2023 05:20:46 PM UTC admin User issued 'reboot' + +"show reboot-cause history module-name" + +root@sonic:~#show reboot-cause history dpu3 + +Device Name Cause Time User Comment + +DPU3 2023_10_03_18_23_46 Watchdog: stage 1 expired; Mon 03 Oct 2023 06:23:46 PM UTC N/A N/A +DPU3 2023_10_02_18_23_46 Host Power-cycle Sun 02 Oct 2023 06:23:46 PM UTC N/A Host lost DPU +DPU3 2023_10_02_17_23_46 Host Reset DPU Sun 02 Oct 2023 05:23:46 PM UTC N/A N/A +``` + +#### Pass/Fail Criteria + * Verify the output to check the latest reboot cause with the date time stamp at the start of reboot + * Verify all the reboot causes - Watchdog, reboot command, Host Reset + + +### 1.12 Check the DPU state after OS reboot + +#### Steps + +Existing Test case for NPU: + * Reboot using a particular command (sonic reboot, watchdog reboot, etc) + * All the timeout and poll timings are read from platform.json + * Wait for ssh to drop + * Wait for ssh to connect + * Database check + * Check for uptime – (NTP sync) + * Check for critical process + * Check for transceiver status + * Check for pmon status + * Check for reboot cause + * Reboot is successful + +Reboot Test Case for DPU: + * Save the configurations of all DPU state before reboot + * Power on all the DPUs that were powered on before reboot using `config chassis modules startup ` + * Wait for DPUs to be up + * Use command `show chassis modules status` to get DPU status + * Get the number of DPU modules from ansible inventory file for the testbed. + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: + +root@sonic:/home/cisco# reboot +root@sonic:/home/cisco# +root@sonic:/home/cisco# config chassis modules startup +root@sonic:/home/cisco# +root@sonic:/home/cisco# +root@sonic:/home/cisco# show chassis modules status + Name Description Physical-Slot Oper-Status Admin-Status Serial +------ -------------------- --------------- ------------- -------------- --------------- + DPU0 Data Processing Unit N/A Online up 154226463179136 + DPU1 Data Processing Unit N/A Online up 154226463179152 + DPU2 Data Processing Unit N/A Online up 154226463179168 + DPUX Data Processing Unit N/A Online up 154226463179184 +``` + +#### Pass/Fail Criteria + * Verify number of DPUs from inventory file for the testbed and number of DPUs shown in the cli output. + + +## Objectives of API Test Cases + +| | **Test Case** | **Intention** | **Comments** | +| ---------- | ---------- | ---------------------------------------- | ---------- | +| 1.1 | Check SmartSwitch specific ChassisClass APIs | To verify the newly implemented SmartSwitch specific ChassisClass APIs | | +| 1.2 | Check modified ChassisClass APIs for SmartSwitch | To verify the existing ChassisClass APIs that undergo minor changes with the addition of SmartSwitch| | +| 1.3 | Check DpuModule APIs for SmartSwitch | To verify the newly implemented DpuModule APIs for SmartSwitch| | +| 1.4 | Check modified ModuleClass APIs for SmartSwitch | To verify the existing ModuleClass APIs that undergo minor changes with the addition of SmartSwitch| + +## API Test Cases + +### 1.1 Check SmartSwitch specific ChassisClass APIs + +#### Steps + * Execute the following APIs on SmartSwitch + * get_dpu_id(self, name): + * Provide name (Example: DPU0 - Get it from platform.json file) + * This API should return an integer from 0- (check it against platform.json) + * is_smartswitch(self): + * This API should return True + * get_module_dpu_data_port(self, index): + * It will return a dict as shown below. + + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: + get_dpu_id(self, DPU3) + Output: 4 + is_smartswitch(self): + Output: True + get_module_dpu_data_port(self, DPU0): + Output: { + "interface": {"Ethernet224": "Ethernet0"} + } +``` +#### Pass/Fail Criteria + * The test result is a pass if the return value matches the expected value as shown in the "steps" and "Sample Output". + + +### 1.2 Check modified ChassisClass APIs for SmartSwitch + +#### Steps + * is_modular_chassis(self): + * Should return False + * get_num_modules(self): + * Should return number of DPUs + * get_module(self, index): + * Make sure for each index this API returns an object and has some content and not None + * Check that the object's class is inherited from the ModuleBase class + * get_all_modules(self): + * This should return a list of items + * get_module_index(self, module_name): + * Given the module name say “DPU0” should return the index of it “1” + + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: + is_modular_chassis(self): + Output: False + get_num_modules(self): + Output: number of DPUs + get_module(self, DPU0): + Output: DPU0 object + get_all_modules(self): + Output: list of objects (one per DPU + 1 switch object) + get_module_index(self, DPU0): + Output: could be any value from 0 to modules count -1 +``` +#### Pass/Fail Criteria + * The test result is a pass if the return value matches the expected value as shown in the "steps" and "Sample Output" + + +### 1.3 Check DpuModule APIs for SmartSwitch + +#### Steps + * get_dpu_id(self): + * Should return ID of the DpuModule Ex: 1 on DPU0 +* get_reboot_cause(self): + * Reboot the module and then execute the "show reboot-cause ..." CLIs + * Verify the output string shows the correct Time and Cause + * Limit the testing to software reboot + * get_state_info(self): + * This should return an object + * Stop Syncd container on this DPU + * Execute the CLI and check the dpu-control-plane value should be down + * Check also dpu-data-plane is also down + * This will be enhanced in Phase - 2 to test multiple failure scenarios and transistions. + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: + get_dpu_id(self): + Output: When on module DPUx should return x+1 + get_reboot_cause(self): + Output: {"Device": "DPU0", "Name": 2024_05_31_00_33_30, "Cause": "reboot", "Time": "Fri 31 May 2024 12:29:34 AM UTC", "User": "NA", "Comment": "NA"} + get_state_info(self): + Output: get_state_info() { + 'dpu_control_plane_reason': 'All containers are up and running, host-ethlink-status: Uplink1/1 is UP', + 'dpu_control_plane_state': 'UP', + 'dpu_control_plane_time': '20240626 21:13:25', + 'dpu_data_plane_reason': 'DPU container named polaris is running, pdsagent running : OK, pciemgrd running : OK', + 'dpu_data_plane_state': 'UP', + 'dpu_data_plane_time': '20240626 21:10:07', + 'dpu_midplane_link_reason': 'INTERNAL-MGMT : admin state - UP, oper_state - UP, status - OK, HOST-MGMT : admin state - UP, oper_state - UP, status - OK', + 'dpu_midplane_link_state': 'UP', + 'dpu_midplane_link_time': '20240626 21:13:25', + 'id': '0' + } +``` +#### Pass/Fail Criteria + * Verify that all the APIs mentioned return the expected output + + ### 1.4 Check modified ModuleClass APIs + +#### Steps + * get_base_mac(self): + * Should return the base mac address of this DPU + * Read all DPUs mac and verify if they are unique and not None + * get_system_eeprom_info(self): + * Verify the returned dictionary key:value + * get_name(self): + * Verify if this API returns “DPUx" on each of them + * get_description(self): + * Should return a string + * get_type(self): + * Should return “DPU” which is “MODULE_TYPE_DPU” + * get_oper_status(self): + * Should return the operational status of the DPU + * Stop one ore more containers + * Execute the CLI and see if it is down + * Power down the dpu and check if the operational status is down. + * reboot(self, reboot_type): + * Issue this CLI with input “ + * verify if the module reboots + * The reboot type should be updated based on SmartSwitch reboot HLD sonic-net/SONiC#1699 + * get_midplane_ip(self): + * should return the midplane IP + +#### Verify in + * Switch + +#### Sample Output +``` +On Switch: + get_base_mac(self): + Output: BA:CE:AD:D0:D0:01 + get_system_eeprom_info(self): + Output: eeprom info object + get_name(self): + Output: DPU2 + get_description(self): + Output "Pensando DSC" + get_type(self): + Output: DPU + get_oper_status(self): + Output: Online + reboot(self, reboot_type): + Result: the DPU should reboot + get_midplane_ip(self): + Output: 169.254.200.1 +``` +#### Pass/Fail Criteria + * The test result is a pass if the return value matches the expected value as shown in the "steps" and "Sample Output". diff --git a/docs/testplan/dash/Dash-Relaxed-Match-Support.md b/docs/testplan/dash/Dash-Relaxed-Match-Support.md new file mode 100644 index 0000000000..9e234d6381 --- /dev/null +++ b/docs/testplan/dash/Dash-Relaxed-Match-Support.md @@ -0,0 +1,79 @@ +# DASH Relaxed Match Support test plan + +* [Overview](#Overview) + * [Scope](#Scope) + * [Testbed](#Testbed) + * [Setup configuration](#Setup%20configuration) +* [Test](#Test) +* [Test cases](#Test%20cases) +* [TODO](#TODO) +* [Open questions](#Open%20questions) + +## Overview + The feature Relaxed Match is to support dynamic change of VXLAN UDP port on the DPU. + The configuration is made via swssconfig in the swss container, and it takes effect immediatly after the configuration is applied. + The purpose of this test is to verify the functionality of dynamic VXLAN UDP port changing on DPU. + +### Scope +The test is targeting on the vetification of functionality on a standalone DPU testbed. +The test has two parts: +- Part1: Integrate the functionality test into the existing dash vnet test(https://github.com/sonic-net/sonic-mgmt/blob/master/tests/dash/test_dash_vnet.py). +- Part2: Add a new test file for the negative test cases. + +The configration is not persistent, it disappears after reload/reboot. So, the reload/reboot test is not in the scope. + +### Testbed +The test will run on standalone DPU testbed. + +### Setup configuration +Common tests configuration: +- Test will apply the basic DASH configration for vnet to vnet scenario + +Common tests cleanup: +- Remove the vnet to vnet configration + +Configuration example to change the port: +``` +[​ + {​ + "SWITCH_TABLE:switch": { "vxlan_port": "12345" },​ + "OP": "SET"​ + }​ +]​ +``` + +## Test +## Part1 - Functionality test integrated to dash vnet test +### Test objective +Verify VXLAN udp port can be changed dynamically +### Test steps +* The validation of the VxLAN port is integrated to the dash vnet test. +* In each test case, randomly choose a UDP port from list ["default", 4789, 1024, 13330, random_port, 65535]. + * "default" means no vxlan port will be explicitly configured, the traffic will use the default port 4789. + * 4789 means the default port 4789 will be explicitly configured. + * 0 to 1023 are the well known ports, they are not tested to avoid unexpected issues. + * The tested port range is from 1024 to 65535, the first 1024 and last 65535 are always in the list, and randomly add another port from the range. + * 13330 is the vnet VXLAN_PORT which is used also in the VXLAN vnet test. + * The port can also be specified by a pytest option. +* In each test case, after the dash configuration, change the VxLAN UDP dst port via swssconfig. +* Run the following traffic validations of the test case with the specified port. + +## Part2 - Negative test +### Test case # 1 – Negative traffic validation +#### Test objective +Verify VXLAN udp port can be changed dynamically, and the traffic with the original port is dropped. +#### Test steps +* Configure basic dash vnet configuration. +* Send the traffic with default port 4789, check it is received. +* Change the port to 13330 +* Send the traffic with default port 4789, check it is dropped. +* Send the traffic with default port 4789 and verify it can be received. +* Change the port back to 4789 +* Send the traffic with port 4789, check it is received. +* Send the traffic with port 13330, check it is dropped. +* Restore the configuration + +## TODO +The test is only for standalone DPU testbed. Need to align it to SmartSwitch DPU testbed after the test infra is ready. + +## Open questions diff --git a/docs/testplan/sensors-test-plan.md b/docs/testplan/sensors-test-plan.md new file mode 100644 index 0000000000..3b59a7c856 --- /dev/null +++ b/docs/testplan/sensors-test-plan.md @@ -0,0 +1,67 @@ +# Test Plan - Dynamic PSU Support in Sensor Test + +## Table of Content + +- [Revision](#revision) +- [Overview](#overview) + - [Scope](#scope) + - [Testbed](#testbed) + - [Design](#design) +- [Tests](#tests) + +## Revision + +| Rev | Date | Author | Change Description | +| :--: | :------: | :-----------: | ------------------------------------------| +| 0.1 | 13/03/24 | Mor Hen | Initial version | +## Overview +The current test checks the sensors per platform, using a predefined list of sensors called "sku-sensors-data.yml". This means any modification to the PSU on the switch will result in failure of the test and require modification of the predefined yml file. +We want to modify the test to depend on the PSUs sensors of the PSUs installed currently on the device and not those in the yml file. This modification will allow us to use different PSUs on the same platform, swap PSUs on the platform + +### Scope +The test is targeting a running SONIC system with fully functioning configuration. The purpose of the test is to verify sensor validation is done correctly after modification of the test. The modification to the test is only targeted on nvidia platforms at the moment. + +### Testbed +The test could run on any testbed. + +### Design + +The original design of the test is as follows: + +![test_sensors-Original Design drawio](https://github.com/mhen1/sonic-mgmt/assets/155874991/beb04857-56cb-4842-8a3f-2e3f97620f05) + +As can be seen, the sensor data of the platform is loaded from the static sku-sensors-data.yml file and is then sent for validation. +The PSUs sesnors checked are those present in this file. + +The new design of the test will be as follows: + +![test_sensors-New Design drawio](https://github.com/mhen1/sonic-mgmt/assets/155874991/026dbd32-ffab-47b0-a70d-8d8fd7667660) + +As can be seen, there are now two static files in play. The original sku-sensors-data.yml file is still used to fetch the sensors unrelated to the PSUs. +However, there is a second file, called psu-sensors-data.yml which will be used to fetch the sensors relevant to the PSUs installed on the device. That is, we will check dynamically which PSUs are installed on the device (using the command "show platform PSU") and, using that information, fetch the relevant sensor information from the psu-sensors-data.yml file which will act as a mapping between PSUs and their relevant sesnors. + +Example of "show platform PSU" output: +![image](https://github.com/mhen1/sonic-mgmt/assets/155874991/4572a1eb-b583-4d75-81d7-e55271020c36) + + +After fetching the information from both files, we will merge them together to a single source of data for the validation. This data will contain all the non-psu sensors from the original sku-sensors-data.yml, and the PSU sensors as listed in the new psu-sensors-data.yml file. The merged data will be sent for validation and the process will proceed as it did in the old design. + +In the case the PSU is not found in the psu-sensors-data.yml file, we will use the old approach and fetch all sensors from sku-sensors-data.yml file. + +Since some of the information of the psu sensor can rely on aspects like the psu slot and platform (for example, bus number and address), we use an additional file called psu_sensors.json to fetch the necessary data. + +Example of how sensors look in sku-sensors-data.yml vs. how they look in psu-sensors-data.yml: +![image](https://github.com/mhen1/sonic-mgmt/assets/155874991/b8f6f37f-47b4-4d61-8af2-7f81105e85d6) + + +The new design will allow us to use different PSUs on the same platform and swap PSUs without needing to make changes in the static files, except when adding new PSUs, which requires updating the newly added "psu-sensors-data.yml" file. + +## Tests +The flow of the test by itself does not change - we validate the values of sensors that reside on our system. We run the test with diffreent scenarios to verify the modification works as expected. + +We will check the following scenarios: +- New functionality check with a single PSU source - run the test on platforms whose PSUs are all of the same model and are listed in psu-sensors-data.yml. +- New functionality check with multiple PSU sources - run the test on platforms with different PSUs, all listed in psu-sensors-data.yml. +- Regression check with a single PSU source - run the test on platforms whose PSUs are all of the same model and are not listed in psu-sensors-data.yml. +- Regression check with multiple PSU sources - run the test on platforms with different PSUs, some (or all) not listed in psu-sensors-data.yml. +- Regression check with a single psu installed - run the test on platforms that have 1 psu installed. diff --git a/pyproject.toml b/pyproject.toml index 2124601599..d7078487ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ # Local pre-commit hook config [project] name = "sonic-mgmt-pre-commit-hooks" -version = "1.0.0+pre_commit" +version = "1.0.1+pre_commit" description = "Some hooks for pre-commit in sonic-mgmt repo." classifiers = [ "Programming Language :: Python :: 3", diff --git a/sdn_tests/pins_ondatra/infrastructure/binding/BUILD.bazel b/sdn_tests/pins_ondatra/infrastructure/binding/BUILD.bazel index 83ed441df1..9519cd10c5 100644 --- a/sdn_tests/pins_ondatra/infrastructure/binding/BUILD.bazel +++ b/sdn_tests/pins_ondatra/infrastructure/binding/BUILD.bazel @@ -52,5 +52,6 @@ go_library( "@com_github_openconfig_ondatra//proto:go_default_library", "@org_golang_google_grpc//:go_default_library", "@org_golang_google_grpc//credentials", + "@org_golang_google_grpc//credentials/insecure", ], ) diff --git a/sdn_tests/pins_ondatra/infrastructure/binding/pins_backend.go b/sdn_tests/pins_ondatra/infrastructure/binding/pins_backend.go index 9f04452868..54082d2a1f 100644 --- a/sdn_tests/pins_ondatra/infrastructure/binding/pins_backend.go +++ b/sdn_tests/pins_ondatra/infrastructure/binding/pins_backend.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "time" + "flag" log "github.com/golang/glog" gpb "github.com/openconfig/gnmi/proto/gnmi" @@ -16,8 +17,14 @@ import ( "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/binding/bindingbackend" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" ) +var ( + supportedSecurityModes = []string{"insecure", "mtls"} + securityMode = flag.String("security_mode", "insecure", fmt.Sprintf("define the security mode of the conntections to gnmi server, choose from : %v. Uses insecure as default.", supportedSecurityModes)) + ) + // Backend can reserve Ondatra DUTs and provide clients to interact with the DUTs. type Backend struct { configs map[string]*tls.Config @@ -168,17 +175,20 @@ func (b *Backend) Release(ctx context.Context) error { // DialGRPC connects to grpc service and returns the opened grpc client for use. func (b *Backend) DialGRPC(ctx context.Context, addr string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - tlsConfig, ok := b.configs[addr] - if !ok { - return nil, fmt.Errorf("failed to find TLS config for %s", addr) - } - - opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) + if *securityMode == "mtls" { + tlsConfig, ok := b.configs[addr] + if !ok { + return nil, fmt.Errorf("failed to find TLS config for %s", addr) + } + + opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) + } else { + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } conn, err := grpc.DialContext(ctx, addr, opts...) - if err != nil { - return nil, fmt.Errorf("DialContext(%s, %v) : %v", addr, opts, err) + if err != nil { + return nil, fmt.Errorf("DialContext(%s, %v) : %v", addr, opts, err) } - return conn, nil } diff --git a/sdn_tests/pins_ondatra/infrastructure/testhelper/gnmi.go b/sdn_tests/pins_ondatra/infrastructure/testhelper/gnmi.go index be185ced8d..e9cc0e2d98 100644 --- a/sdn_tests/pins_ondatra/infrastructure/testhelper/gnmi.go +++ b/sdn_tests/pins_ondatra/infrastructure/testhelper/gnmi.go @@ -2,8 +2,10 @@ package testhelper import ( "context" + "os" "testing" + closer "github.com/openconfig/gocloser" "github.com/openconfig/ondatra" "github.com/openconfig/ondatra/gnmi" "github.com/openconfig/ondatra/gnmi/oc/system" @@ -88,3 +90,67 @@ func CreateSubscribeRequest(params SubscribeRequestParams) (*gpb.SubscribeReques }, }, nil } + +// GNMIAble returns whether the gNMI server on the specified device is reachable +// or not. +func GNMIAble(t *testing.T, d *ondatra.DUTDevice) error { + // Since the Ondatra Get() API panics in case of failure, we need to use + // raw gNMI client to test reachability with the gNMI server on the switch. + // The gNMI server reachability is checked by fetching the system boot-time + // path from the switch. + params := SubscribeRequestParams{ + Target: testhelperDUTNameGet(d), + Paths: []ygnmi.PathStruct{gnmiSystemBootTimePath()}, + Mode: gpb.SubscriptionList_ONCE, + } + subscribeRequest, err := CreateSubscribeRequest(params) + if err != nil { + return errors.Wrapf(err, "failed to create SubscribeRequest") + } + + subscribeClient, err := gnmiSubscribeClientGet(t, d, context.Background()) + if err != nil { + return errors.Wrapf(err, "unable to get subscribe client") + } + defer closer.CloseAndLog(subscribeClient.CloseSend, "error closing gNMI send stream") + + if err := subscribeClient.Send(subscribeRequest); err != nil { + return errors.Wrapf(err, "failed to send gNMI subscribe request") + } + + if _, err := subscribeClient.Recv(); err != nil { + return errors.Wrapf(err, "subscribe client Recv() failed") + } + + return nil +} + +// ConfigGet returns a full config for the given DUT. +func (d GNMIConfigDUT) ConfigGet() ([]byte, error) { + return os.ReadFile("ondatra/data/config.json") +} + +// ConfigPush pushes the given config onto the DUT. If nil is passed in for config, +// this function will use ConfigGet() to get a full config for the DUT. +func ConfigPush(t *testing.T, dut *ondatra.DUTDevice, config *[]byte) error { + if dut == nil { + return errors.New("nil DUT passed into ConfigPush()") + } + if config == nil { + getConfig, err := GNMIConfigDUT{dut}.ConfigGet() + if err != nil { + return err + } + config = &getConfig + } + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: testhelperDUTNameGet(dut)}, + Replace: []*gpb.Update{{ + Path: &gpb.Path{}, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: *config}}, + }}, + } + t.Logf("Pushing config on %v: %v", testhelperDUTNameGet(dut), setRequest) + _, err := gnmiSet(t, dut, setRequest) + return err +} diff --git a/sdn_tests/pins_ondatra/infrastructure/testhelper/gnoi.go b/sdn_tests/pins_ondatra/infrastructure/testhelper/gnoi.go index de06454f07..a87bfa96b5 100644 --- a/sdn_tests/pins_ondatra/infrastructure/testhelper/gnoi.go +++ b/sdn_tests/pins_ondatra/infrastructure/testhelper/gnoi.go @@ -8,8 +8,10 @@ import ( "testing" "time" + log "github.com/golang/glog" "github.com/openconfig/ondatra" "github.com/openconfig/ondatra/gnmi" + "github.com/pkg/errors" healthzpb "github.com/openconfig/gnoi/healthz" syspb "github.com/openconfig/gnoi/system" @@ -83,6 +85,74 @@ func (p *RebootParams) measureLatency() bool { return p.waitTime > 0 && p.lmTitle != "" } +// Reboot sends a RebootRequest message to the switch. It waits for a specified +// amount of time for the switch reboot to be successful. A switch reboot is +// considered to be successful if the gNOI server is up and the boot time is +// after the reboot request time. +func Reboot(t *testing.T, d *ondatra.DUTDevice, params *RebootParams) error { + if params.waitTime < params.checkInterval { + return errors.Errorf("wait time:%v cannot be less than check interval:%v", params.waitTime, params.checkInterval) + } + + var req *syspb.RebootRequest + switch v := params.request.(type) { + case syspb.RebootMethod: + // User only specified the reboot type. Construct reboot request. + req = &syspb.RebootRequest{ + Method: v, + Message: "Reboot", + } + case *syspb.RebootRequest: + // Use the specified reboot request. + req = v + default: + return errors.New("invalid reboot request (valid parameters are RebootRequest protobuf and RebootMethod)") + } + + log.Infof("Rebooting %v switch", testhelperDUTNameGet(d)) + timeBeforeReboot := time.Now().UnixNano() + systemClient := gnoiSystemClientGet(t, d) + + if _, err := systemClient.Reboot(context.Background(), req); err != nil { + return errors.Wrapf(err, "reboot RPC failed") + } + + if params.waitTime == 0 { + // User did not request a wait time which implies that the API did not verify whether + // the switch has rebooted or not. Therefore, do not return an error in this case. + return nil + } + + log.Infof("Polling gNOI server reachability in %v intervals for max duration of %v", params.checkInterval, params.waitTime) + for timeout := time.Now().Add(params.waitTime); time.Now().Before(timeout); { + // The switch backend might not have processed the request or might take + // sometime to execute the request. So wait for check interval time and + // later verify that the switch rebooted within the specified wait time. + time.Sleep(params.checkInterval) + doneTime := time.Now() + timeElapsed := (doneTime.UnixNano() - timeBeforeReboot) / int64(time.Second) + + if err := GNOIAble(t, d); err != nil { + log.Infof("gNOI server not up after %v seconds", timeElapsed) + continue + } + log.Infof("gNOI server up after %v seconds", timeElapsed) + + // An extra check to ensure that the system has rebooted. + if bootTime := gnmiSystemBootTimeGet(t, d); bootTime < uint64(timeBeforeReboot) { + log.Infof("Switch has not rebooted after %v seconds", timeElapsed) + continue + } + + log.Infof("Switch rebooted after %v seconds", timeElapsed) + return nil + } + + err := errors.Errorf("failed to reboot %v", testhelperDUTNameGet(d)) + + return err +} + // GNOIAble returns whether the gNOI server on the specified device is reachable // or not. func GNOIAble(t *testing.T, d *ondatra.DUTDevice) error { diff --git a/sdn_tests/pins_ondatra/infrastructure/testhelper/lacp.go b/sdn_tests/pins_ondatra/infrastructure/testhelper/lacp.go index 18d50eecd2..e5405ac045 100644 --- a/sdn_tests/pins_ondatra/infrastructure/testhelper/lacp.go +++ b/sdn_tests/pins_ondatra/infrastructure/testhelper/lacp.go @@ -1,7 +1,13 @@ package testhelper import ( + "testing" + "time" + + log "github.com/golang/glog" + "github.com/openconfig/ondatra" "github.com/openconfig/ondatra/gnmi/oc" + "github.com/pkg/errors" ) // PeerPorts holds the name of 2 Ethernet interfaces. These interfaces will be on separate machines, @@ -11,6 +17,59 @@ type PeerPorts struct { Peer string } +// PeerPortsBySpeed iterates through all the available Ethernet ports on a host device, and +// determines if they have a matching port on the peer device. All host ports with a valid peer will +// be grouped together based on their speed. +func PeerPortsBySpeed(t *testing.T, host *ondatra.DUTDevice, peer *ondatra.DUTDevice) map[oc.E_IfEthernet_ETHERNET_SPEED][]PeerPorts { + peerPortsBySpeed := make(map[oc.E_IfEthernet_ETHERNET_SPEED][]PeerPorts) + + for _, hostPort := range testhelperDUTPortsGet(host) { + peerPort := testhelperDUTPortGet(t, peer, testhelperOndatraPortIDGet(hostPort)) + + // Verify the host port is UP. Otherwise, LACPDU packets will never be transmitted. + if got, want := testhelperIntfOperStatusGet(t, host, testhelperOndatraPortNameGet(hostPort)), oc.Interface_OperStatus_UP; got != want { + log.Warningf("Port %v:%v oper state will not work for LACP testing: want=%v got=%v", testhelperDUTNameGet(host), testhelperOndatraPortNameGet(hostPort), want, got) + continue + } + + // Verify we are not already part of an existing PortChannel since one port cannot belong to + // multiple PortChannels. + if got, want := testhelperConfigIntfAggregateIDGet(t, host, testhelperOndatraPortNameGet(hostPort)), ""; got != want { + log.Warningf("Port %v:%v cannot be used since it is already assigned to a PortChannel: want=%v got=%v", testhelperDUTNameGet(host), testhelperOndatraPortNameGet(hostPort), want, got) + continue + } + + // Check that the host port has a valid peer port. + if peerPort == nil { + log.Warningf("Port %v:%v does not have a peer on %v.", testhelperDUTNameGet(host), testhelperOndatraPortNameGet(hostPort), testhelperDUTNameGet(peer)) + continue + } + + log.Infof("Found peer ports: %v:%v and %v:%v", testhelperDUTNameGet(host), testhelperOndatraPortNameGet(hostPort), testhelperDUTNameGet(peer), testhelperOndatraPortNameGet(peerPort)) + portSpeed := testhelperConfigPortSpeedGet(t, host, testhelperOndatraPortNameGet(hostPort)) + peerPortsBySpeed[portSpeed] = append(peerPortsBySpeed[portSpeed], PeerPorts{testhelperOndatraPortNameGet(hostPort), testhelperOndatraPortNameGet(peerPort)}) + } + + return peerPortsBySpeed +} + +// PeerPortGroupWithNumMembers returns a list of PeerPorts of size `numMembers`. +func PeerPortGroupWithNumMembers(t *testing.T, host *ondatra.DUTDevice, peer *ondatra.DUTDevice, numMembers int) ([]PeerPorts, error) { + // GPINs requires that all members of a LACP LAG have the same speed. So we first group all the + // ports based on their configured speed. + peerPortsBySpeed := PeerPortsBySpeed(t, host, peer) + + // Then we search through the port speed gropus for one that has enough members to match the users + // requested amount. + for _, ports := range peerPortsBySpeed { + if len(ports) >= numMembers { + // Only return enough members to match the users request. + return ports[0:numMembers], nil + } + } + return nil, errors.Errorf("cannot make group of %v member ports with the same speed from %v.", numMembers, peerPortsBySpeed) +} + // GeneratePortChannelInterface will return a minimal PortChannel interface that tests can extend as needed. func GeneratePortChannelInterface(portChannelName string) oc.Interface { enabled := true @@ -40,3 +99,31 @@ func GenerateLACPInterface(pcName string) oc.Lacp_Interface { LacpMode: oc.Lacp_LacpActivityType_ACTIVE, } } + +// RemovePortChannelFromDevice will cleanup all configs relating to a PortChannel on a given switch. +// It will also verify that the state has been updated before returning. If the state fails to +// converge then an error will be returned. +func RemovePortChannelFromDevice(t *testing.T, timeout time.Duration, dut *ondatra.DUTDevice, portChannelName string) error { + // We only need to delete the PortChannel interface and gNMI should take care of all the other + // configs relating to the PortChannel. + testhelperIntfDelete(t, dut, portChannelName) + + stopTime := time.Now().Add(timeout) + for time.Now().Before(stopTime) { + if !testhelperIntfLookup(t, dut, portChannelName).IsPresent() { + return nil + } + time.Sleep(time.Second) + } + + return errors.Errorf("interface still exists after %v", timeout) +} + +// AssignPortsToAggregateID will assign the list of ports to the given aggregate ID on a device. The +// aggregate ID should correspond to an existing PortChannel interface or this call will fail. +func AssignPortsToAggregateID(t *testing.T, dut *ondatra.DUTDevice, portChannelName string, portNames ...string) { + for _, portName := range portNames { + log.Infof("Assigning %v:%v to %v", testhelperDUTNameGet(dut), portName, portChannelName) + testhelperIntfAggregateIDReplace(t, dut, portName, portChannelName) + } +} diff --git a/sdn_tests/pins_ondatra/infrastructure/testhelper/p4rt.go b/sdn_tests/pins_ondatra/infrastructure/testhelper/p4rt.go index e9f5092008..fc723d86f4 100644 --- a/sdn_tests/pins_ondatra/infrastructure/testhelper/p4rt.go +++ b/sdn_tests/pins_ondatra/infrastructure/testhelper/p4rt.go @@ -141,7 +141,7 @@ func (p *P4RTClient) P4Info() (*p4infopb.P4Info, error) { // Read P4Info from file. p4Info = &p4infopb.P4Info{} - data, err := os.ReadFile("infrastructure/data/p4rtconfig.prototext") + data, err := os.ReadFile("ondatra/data/p4rtconfig.prototext") if err != nil { return nil, err } diff --git a/sdn_tests/pins_ondatra/tests/BUILD.bazel b/sdn_tests/pins_ondatra/tests/BUILD.bazel index 7a633e7ac2..20dc52c6c9 100644 --- a/sdn_tests/pins_ondatra/tests/BUILD.bazel +++ b/sdn_tests/pins_ondatra/tests/BUILD.bazel @@ -36,6 +36,22 @@ ondatra_test( ], ) +go_library( + name = "gnmi_stress_helper", + testonly = 1, + srcs = ["gnmi_helper.go"], + importpath = "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/tests/gnmi_stress_helper", + deps = [ + "//infrastructure/testhelper", + "@com_github_openconfig_gnmi//proto/gnmi:gnmi_go_proto", + "@com_github_openconfig_gnmi//value", + "@com_github_openconfig_ondatra//:go_default_library", + "@com_github_openconfig_ygot//ygot", + "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_protobuf//encoding/prototext", + ], +) + # Gnoi File tests ondatra_test( name = "gnoi_file_test", @@ -308,3 +324,88 @@ ondatra_test( "@com_github_pkg_errors//:errors", ], ) + +# gNMI Features: Stress Test +ondatra_test( + name = "z_gnmi_stress_test", + srcs = ["gnmi_stress_test.go"], + run_timeout = "120m", + deps = [ + ":gnmi_stress_helper", + "//infrastructure/binding:pinsbind", + "//infrastructure/testhelper", + "@com_github_openconfig_gnmi//proto/gnmi:gnmi_go_proto", + "@com_github_openconfig_ondatra//:go_default_library", + "@com_github_openconfig_ondatra//gnmi", + "@com_github_openconfig_ygot//ygot", + "@org_golang_google_grpc//:go_default_library", + ], +) + +# System paths tests +ondatra_test( + name = "system_paths_test", + srcs = ["system_paths_test.go"], + deps = [ + "//infrastructure/binding:pinsbind", + "//infrastructure/testhelper", + "@com_github_golang_glog//:glog", + "@com_github_google_go_cmp//cmp", + "@com_github_openconfig_gnoi//system:system_go_proto", + "@com_github_openconfig_ondatra//:go_default_library", + "@com_github_openconfig_ondatra//gnmi", + "@com_github_openconfig_ondatra//gnmi/oc", + "@com_github_pkg_errors//:errors", + ], +) + +# gNMI Features: Wildcard Subscription +ondatra_test( + name = "gnmi_wildcard_subscription_test", + srcs = ["gnmi_wildcard_subscription_test.go"], + deps = [ + "//infrastructure/binding:pinsbind", + "//infrastructure/testhelper", + "@com_github_google_go_cmp//cmp", + "@com_github_openconfig_gnmi//proto/gnmi:gnmi_go_proto", + "@com_github_openconfig_gnoi//system:system_go_proto", + "@com_github_openconfig_ondatra//:go_default_library", + "@com_github_openconfig_ondatra//gnmi", + "@com_github_openconfig_ondatra//gnmi/oc", + "@com_github_openconfig_ygnmi//ygnmi", + "@com_github_openconfig_ygot//ygot", + "@com_github_pkg_errors//:errors", + ], +) + +# Management interface tests +ondatra_test( + name = "mgmt_interface_test", + srcs = ["mgmt_interface_test.go"], + deps = [ + "//infrastructure/binding:pinsbind", + "//infrastructure/testhelper", + "@com_github_openconfig_ondatra//:go_default_library", + "@com_github_openconfig_ondatra//gnmi", + "@com_github_openconfig_ondatra//gnmi/oc", + "@com_github_openconfig_testt//:testt", + ], +) + +# gNMI Features: SUBSCRIBE Modes +ondatra_test( + name = "gnmi_subscribe_modes_test", + srcs = ["gnmi_subscribe_modes_test.go"], + deps = [ + "//infrastructure/binding:pinsbind", + "//infrastructure/testhelper", + "@com_github_google_go_cmp//cmp", + "@com_github_google_go_cmp//cmp/cmpopts", + "@com_github_openconfig_gnmi//proto/gnmi:gnmi_go_proto", + "@com_github_openconfig_ondatra//:go_default_library", + "@com_github_openconfig_ondatra//gnmi", + "@com_github_openconfig_ygot//ygot", + "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_protobuf//encoding/prototext", + ], +) diff --git a/sdn_tests/pins_ondatra/tests/ethcounter_sw_single_switch_test.go b/sdn_tests/pins_ondatra/tests/ethcounter_sw_single_switch_test.go index f8cdb6bec0..a110a2c56c 100644 --- a/sdn_tests/pins_ondatra/tests/ethcounter_sw_single_switch_test.go +++ b/sdn_tests/pins_ondatra/tests/ethcounter_sw_single_switch_test.go @@ -18,6 +18,7 @@ import ( "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/binding/pinsbind" "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/testhelper/testhelper" ) + // These are the counters we track in these tests. type Counters struct { inPkts uint64 @@ -775,3 +776,675 @@ func TestGNMIEthernetIn(t *testing.T) { t.Logf("\n\n----- TestGNMIEthernetIn: SUCCESS after %v Iteration(s) -----\n\n", i) } + +// ---------------------------------------------------------------------------- +// TestGNMIEthernetInMulticast - Check EthernetX In-Multicast-Pkts +func TestGNMIEthernetInMulticast(t *testing.T) { + // Report results to TestTracker at the end. + defer testhelper.NewTearDownOptions(t).WithID("0b34a2a3-4b30-41cf-a642-634334357cee").Teardown(t) + + // Select the dut, or device under test. + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + CheckInitial(t, dut, intf) + defer RestoreInitial(t, dut, intf) + + // To get ingress traffic in Ondatra, turn on loopback mode on + // the selected port just for this test. + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).LoopbackMode().Config(), oc.Interfaces_LoopbackModeType_FACILITY) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).LoopbackMode().State(), loopbackStateTimeout, oc.Interfaces_LoopbackModeType_FACILITY) + + var bad bool + var i int + + // Iterate up to 10 times to get a successful test. + for i = 1; i <= 10; i++ { + t.Logf("\n----- TestGNMIEthernetInMulticast: Iteration %v -----\n", i) + bad = false + + // Read all the relevant counters initial values. + before := ReadCounters(t, dut, intf) + + // Compute the expected counters after the test. + expect := before + expect.outPkts += pktsPer + expect.outOctets += 64 * pktsPer + expect.outMulticastPkts += pktsPer + expect.inPkts += pktsPer + expect.inOctets += 64 * pktsPer + expect.inMulticastPkts += pktsPer + + // Construct a simple multicast Ethernet L2 packet. + eth := &layers.Ethernet{ + SrcMAC: net.HardwareAddr{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}, + DstMAC: net.HardwareAddr{0x01, 0x00, 0x5E, 0xFF, 0xFF, 0xFF}, + EthernetType: layers.EthernetTypeEthernetCTP, + } + + buf := gopacket.NewSerializeBuffer() + + // Enable reconstruction of length and checksum fields. + opts := gopacket.SerializeOptions{ + FixLengths: true, + ComputeChecksums: true, + } + + if err := gopacket.SerializeLayers(buf, opts, eth); err != nil { + t.Fatalf("Failed to serialize packet (%v)", err) + } + + packetOut := &testhelper.PacketOut{ + EgressPort: intf, + Count: uint(pktsPer), + Interval: 1 * time.Millisecond, + Packet: buf.Bytes(), + } + + p4rtClient, err := testhelper.FetchP4RTClient(t, dut, dut.RawAPIs().P4RT(t), nil) + if err != nil { + t.Fatalf("Failed to create P4RT client: %v", err) + } + if err := p4rtClient.SendPacketOut(t, packetOut); err != nil { + t.Fatalf("SendPacketOut operation failed for %+v (%v)", packetOut, err) + } + + // Sleep for enough time that the counters are polled after the + // transmit completes sending bytes. + time.Sleep(counterUpdateDelay) + + // Read all the relevant counters again. + after := ReadCounters(t, dut, intf) + + // We're seeing some random discards during testing due to + // existing traffic being discarded in loopback mode so simply + // set up to ignore them. + expect.inDiscards = after.inDiscards + + if after != expect { + ShowCountersDelta(t, before, after, expect) + bad = true + } + + if !bad { + break + } + } + + if bad { + t.Fatalf("\n\n----- TestGNMIEthernetInMulticast: FAILED after %v Iterations -----\n\n", i-1) + } + + t.Logf("\n\n----- TestGNMIEthernetInMulticast: SUCCESS after %v Iteration(s) -----\n\n", i) +} + +// ---------------------------------------------------------------------------- +// TestGNMIEthernetInBroadcast - Check EthernetX In-Broadcast-Pkts +func TestGNMIEthernetInBroadcast(t *testing.T) { + // Report results to TestTracker at the end. + defer testhelper.NewTearDownOptions(t).WithID("334c1369-b12f-4f73-aec1-effbb0a3fd4b").Teardown(t) + + // Select the dut, or device under test. + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + CheckInitial(t, dut, intf) + defer RestoreInitial(t, dut, intf) + + // To get ingress traffic in Ondatra, turn on loopback mode on + // the selected port just for this test. + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).LoopbackMode().Config(), oc.Interfaces_LoopbackModeType_FACILITY) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).LoopbackMode().State(), loopbackStateTimeout, oc.Interfaces_LoopbackModeType_FACILITY) + + var bad bool + var i int + + // Iterate up to 10 times to get a successful test. + for i = 1; i <= 10; i++ { + t.Logf("\n----- TestGNMIEthernetInBroadcast: Iteration %v -----\n", i) + bad = false + + // Read all the relevant counters initial values. + before := ReadCounters(t, dut, intf) + + // Compute the expected counters after the test. + expect := before + expect.outPkts += pktsPer + expect.outOctets += 64 * pktsPer + expect.outBroadcastPkts += pktsPer + expect.inPkts += pktsPer + expect.inOctets += 64 * pktsPer + expect.inBroadcastPkts += pktsPer + + // Construct a simple multicast Ethernet L2 packet. + eth := &layers.Ethernet{ + SrcMAC: net.HardwareAddr{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}, + DstMAC: net.HardwareAddr{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + EthernetType: layers.EthernetTypeEthernetCTP, + } + + buf := gopacket.NewSerializeBuffer() + + // Enable reconstruction of length and checksum fields. + opts := gopacket.SerializeOptions{ + FixLengths: true, + ComputeChecksums: true, + } + + if err := gopacket.SerializeLayers(buf, opts, eth); err != nil { + t.Fatalf("Failed to serialize packet (%v)", err) + } + + packetOut := &testhelper.PacketOut{ + EgressPort: intf, + Count: uint(pktsPer), + Interval: 1 * time.Millisecond, + Packet: buf.Bytes(), + } + + p4rtClient, err := testhelper.FetchP4RTClient(t, dut, dut.RawAPIs().P4RT(t), nil) + if err != nil { + t.Fatalf("Failed to create P4RT client: %v", err) + } + if err := p4rtClient.SendPacketOut(t, packetOut); err != nil { + t.Fatalf("SendPacketOut operation failed for %+v (%v)", packetOut, err) + } + + // Sleep for enough time that the counters are polled after the + // transmit completes sending bytes. + time.Sleep(counterUpdateDelay) + + // Read all the relevant counters again. + after := ReadCounters(t, dut, intf) + + // We're seeing some random discards during testing due to + // existing traffic being discarded in loopback mode so simply + // set up to ignore them. + expect.inDiscards = after.inDiscards + + if after != expect { + ShowCountersDelta(t, before, after, expect) + bad = true + } + + if !bad { + break + } + } + + if bad { + t.Fatalf("\n\n----- TestGNMIEthernetInBroadcast: FAILED after %v Iterations -----\n\n", i-1) + } + + t.Logf("\n\n----- TestGNMIEthernetInBroadcast: SUCCESS after %v Iteration(s) -----\n\n", i) +} + +// ---------------------------------------------------------------------------- +// TestGNMIEthernetInIPv4Pkts - Check EthernetX Subinterface IPv4 in-pkts +func TestGNMIEthernetInIPv4(t *testing.T) { + // Report results to TestTracker at the end. + defer testhelper.NewTearDownOptions(t).WithID("8e134557-a159-44ba-9005-e67c7bf8744c").Teardown(t) + + // Select the dut, or device under test. + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + CheckInitial(t, dut, intf) + defer RestoreInitial(t, dut, intf) + + // To get ingress traffic in Ondatra, turn on loopback mode on + // the selected port just for this test. + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).LoopbackMode().Config(), oc.Interfaces_LoopbackModeType_FACILITY) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).LoopbackMode().State(), loopbackStateTimeout, oc.Interfaces_LoopbackModeType_FACILITY) + + var bad bool + var i int + + // Iterate up to 10 times to get a successful test. + for i = 1; i <= 10; i++ { + t.Logf("\n----- TestGNMIEthernetInIPv4: Iteration %v -----\n", i) + bad = false + + // Read all the relevant counters initial values. + before := ReadCounters(t, dut, intf) + + // Compute the expected counters after the test. + expect := before + expect.outPkts += pktsPer + expect.outOctets += 64 * pktsPer + expect.outUnicastPkts += pktsPer + expect.inPkts += pktsPer + expect.inOctets += 64 * pktsPer + expect.inUnicastPkts += pktsPer + + // Construct a simple IPv4 packet. + eth := &layers.Ethernet{ + SrcMAC: net.HardwareAddr{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}, + DstMAC: net.HardwareAddr{0x00, 0x1A, 0x11, 0x17, 0x5F, 0x80}, + EthernetType: layers.EthernetTypeIPv4, + } + ip := &layers.IPv4{ + Version: 4, + TTL: 64, + Protocol: layers.IPProtocolTCP, + SrcIP: net.ParseIP("100.0.0.1").To4(), + DstIP: net.ParseIP("200.0.0.1").To4(), + } + tcp := &layers.TCP{ + SrcPort: 10000, + DstPort: 20000, + Seq: 11050, + } + // Required for checksum computation. + tcp.SetNetworkLayerForChecksum(ip) + payload := gopacket.Payload([]byte{'t', 'e', 's', 't'}) + buf := gopacket.NewSerializeBuffer() + + // Enable reconstruction of length and checksum fields based on packet headers. + opts := gopacket.SerializeOptions{ + FixLengths: true, + ComputeChecksums: true, + } + if err := gopacket.SerializeLayers(buf, opts, eth, ip, tcp, payload); err != nil { + t.Fatalf("Failed to serialize packet (%v)", err) + } + + packetOut := &testhelper.PacketOut{ + EgressPort: intf, + Count: uint(pktsPer), + Interval: 1 * time.Millisecond, + Packet: buf.Bytes(), + } + + p4rtClient, err := testhelper.FetchP4RTClient(t, dut, dut.RawAPIs().P4RT(t), nil) + if err != nil { + t.Fatalf("Failed to create P4RT client: %v", err) + } + if err := p4rtClient.SendPacketOut(t, packetOut); err != nil { + t.Fatalf("SendPacketOut operation failed for %+v (%v)", packetOut, err) + } + + // Sleep for enough time that the counters are polled after the + // transmit completes sending bytes. + time.Sleep(counterUpdateDelay) + + // Read all the relevant counters again. + after := ReadCounters(t, dut, intf) + + // We're seeing some random discards during testing due to + // existing traffic being discarded in loopback mode so simply + // set up to ignore them. + expect.inDiscards = after.inDiscards + + if after != expect { + ShowCountersDelta(t, before, after, expect) + bad = true + } + + if !bad { + break + } + } + + if bad { + t.Fatalf("\n\n----- TestGNMIEthernetInIPv4: FAILED after %v Iterations -----\n\n", i-1) + } + + t.Logf("\n\n----- TestGNMIEthernetInIPv4: SUCCESS after %v Iteration(s) -----\n\n", i) +} + +// ---------------------------------------------------------------------------- +// TestGNMIEthernetInIPv6Pkts - Check EthernetX Subinterface IPv6 in-pkts +func TestGNMIEthernetInIPv6(t *testing.T) { + // Report results to TestTracker at the end. + defer testhelper.NewTearDownOptions(t).WithID("bb5e6b9f-404d-441d-9a0b-a2ecb9785e1a").Teardown(t) + + // Select the dut, or device under test. + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + CheckInitial(t, dut, intf) + defer RestoreInitial(t, dut, intf) + + // To get ingress traffic in Ondatra, turn on loopback mode on + // the selected port just for this test. + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).LoopbackMode().Config(), oc.Interfaces_LoopbackModeType_FACILITY) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).LoopbackMode().State(), loopbackStateTimeout, oc.Interfaces_LoopbackModeType_FACILITY) + + var bad bool + var i int + + // Iterate up to 10 times to get a successful test. + for i = 1; i <= 10; i++ { + t.Logf("\n----- TestGNMIEthernetInIPv6: Iteration %v -----\n", i) + bad = false + // Read all the relevant counters initial values. + before := ReadCounters(t, dut, intf) + + // Compute the expected counters after the test. + expect := before + expect.outPkts += pktsPer + expect.outOctets += 64 * pktsPer + expect.outUnicastPkts += pktsPer + expect.inPkts += pktsPer + expect.inOctets += 64 * pktsPer + expect.inUnicastPkts += pktsPer + + // Construct a simple IPv6 packet. + eth := &layers.Ethernet{ + SrcMAC: net.HardwareAddr{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}, + DstMAC: net.HardwareAddr{0x00, 0x1A, 0x11, 0x17, 0x5F, 0x80}, + EthernetType: layers.EthernetTypeIPv6, + } + ip := &layers.IPv6{ + Version: 6, + HopLimit: 64, + SrcIP: net.ParseIP("2001:db8::1"), + DstIP: net.ParseIP("2001:db8::2"), + NextHeader: layers.IPProtocolICMPv6, + } + icmp := &layers.ICMPv6{ + TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypePacketTooBig, 0), + } + + icmp.SetNetworkLayerForChecksum(ip) + buf := gopacket.NewSerializeBuffer() + + // Enable reconstruction of length and checksum fields based on packet headers. + opts := gopacket.SerializeOptions{ + FixLengths: true, + ComputeChecksums: true, + } + if err := gopacket.SerializeLayers(buf, opts, eth, ip, icmp); err != nil { + t.Fatalf("Failed to serialize packet (%v)", err) + } + + packetOut := &testhelper.PacketOut{ + EgressPort: intf, + Count: uint(pktsPer), + Interval: 1 * time.Millisecond, + Packet: buf.Bytes(), + } + + p4rtClient, err := testhelper.FetchP4RTClient(t, dut, dut.RawAPIs().P4RT(t), nil) + if err != nil { + t.Fatalf("Failed to create P4RT client: %v", err) + } + if err := p4rtClient.SendPacketOut(t, packetOut); err != nil { + t.Fatalf("SendPacketOut operation failed for %+v (%v)", packetOut, err) + } + + // Sleep for enough time that the counters are polled after the + // transmit completes sending bytes. + time.Sleep(counterUpdateDelay) + + // Read all the relevant counters again. + after := ReadCounters(t, dut, intf) + + // We're seeing some random discards during testing due to + // existing traffic being discarded in loopback mode so simply + // set up to ignore them. + expect.inDiscards = after.inDiscards + + if after != expect { + ShowCountersDelta(t, before, after, expect) + bad = true + } + + if !bad { + break + } + } + + if bad { + t.Fatalf("\n\n----- TestGNMIEthernetInIPv6: FAILED after %v Iterations -----\n\n", i-1) + } + + t.Logf("\n\n----- TestGNMIEthernetInIPv6: SUCCESS after %v Iteration(s) -----\n\n", i) +} + +// ---------------------------------------------------------------------------- +// TestGNMIEthernetInDiscards - Check EthernetX in-discards +func TestGNMIEthernetInDiscards(t *testing.T) { + // Report results to TestTracker at the end. + defer testhelper.NewTearDownOptions(t).WithID("dde5578a-33f2-40b2-a7fa-a978b9ee0a51").Teardown(t) + + // Select the dut, or device under test. + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + CheckInitial(t, dut, intf) + defer RestoreInitial(t, dut, intf) + + // To get ingress traffic in Ondatra, turn on loopback mode on + // the selected port just for this test. + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).LoopbackMode().Config(), oc.Interfaces_LoopbackModeType_FACILITY) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).LoopbackMode().State(), loopbackStateTimeout, oc.Interfaces_LoopbackModeType_FACILITY) + + var bad bool + var i int + + // Iterate up to 10 times to get a successful test. + for i = 1; i <= 10; i++ { + t.Logf("\n----- TestGNMIEthernetInDiscards: Iteration %v -----\n", i) + bad = false + + // Read all the relevant counters initial values. + before := ReadCounters(t, dut, intf) + + // Compute the expected counters after the test. Since + // we're seeing some discard traffic (1 or 2 per second) during + // normal operation on the Ondatra testbeds with loopback + // turned on, setting the number of packets to be sent larger + // so we can actually verify its those packets that we got. + expect := before + expect.outPkts += pktsPer + expect.outOctets += pktsPer * 64 + expect.outUnicastPkts += pktsPer + expect.inPkts += pktsPer + expect.inOctets += pktsPer * 64 + expect.inUnicastPkts += pktsPer + expect.inDiscards += pktsPer + + // Construct a simple IPv4 packet that will get discarded. In + // offline testing, setting the IP protocol field to zero + // worked to cause a discard on ingest. + eth := &layers.Ethernet{ + SrcMAC: net.HardwareAddr{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}, + DstMAC: net.HardwareAddr{0x00, 0x1A, 0x11, 0x17, 0x5F, 0x80}, + EthernetType: layers.EthernetTypeIPv4, + } + + ip := &layers.IPv4{ + Version: 4, + TTL: 0, + Protocol: layers.IPProtocol(0), + SrcIP: net.ParseIP("100.0.0.1").To4(), + DstIP: net.ParseIP("200.0.0.1").To4(), + } + + buf := gopacket.NewSerializeBuffer() + + // Enable reconstruction of length and checksum fields based on packet headers. + opts := gopacket.SerializeOptions{ + FixLengths: true, + ComputeChecksums: true, + } + + if err := gopacket.SerializeLayers(buf, opts, eth, ip); err != nil { + t.Fatalf("Failed to serialize packet (%v)", err) + } + + packetOut := &testhelper.PacketOut{ + EgressPort: intf, + Count: uint(pktsPer), + Interval: 1 * time.Millisecond, + Packet: buf.Bytes(), + } + + p4rtClient, err := testhelper.FetchP4RTClient(t, dut, dut.RawAPIs().P4RT(t), nil) + if err != nil { + t.Fatalf("Failed to create P4RT client: %v", err) + } + if err := p4rtClient.SendPacketOut(t, packetOut); err != nil { + t.Fatalf("SendPacketOut operation failed for %+v (%v)", packetOut, err) + } + + // Sleep for enough time that the counters are polled after the + // transmit completes sending bytes. + time.Sleep(counterUpdateDelay) + + // Read all the relevant counters again. + after := ReadCounters(t, dut, intf) + + if after != expect { + ShowCountersDelta(t, before, after, expect) + bad = true + } + + if !bad { + break + } + } + + if bad { + t.Fatalf("\n\n----- TestGNMIEthernetInDiscards: FAILED after %v Iterations -----\n\n", i-1) + } + + t.Logf("\n\n----- TestGNMIEthernetInDiscards: SUCCESS after %v Iteration(s) -----\n\n", i) +} + +// ---------------------------------------------------------------------------- +// TestGNMIEthernetInIPv6Discards - Check EthernetX Subinterface in-ipv6-discards +func TestGNMIEthernetInIPv6Discards(t *testing.T) { + // Report results to TestTracker at the end. + defer testhelper.NewTearDownOptions(t).WithID("2b04e2cb-cce4-43ef-ad42-5cef4dc8f55c").Teardown(t) + + // Select the dut, or device under test. + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + CheckInitial(t, dut, intf) + defer RestoreInitial(t, dut, intf) + + // To get ingress traffic in Ondatra, turn on loopback mode on + // the selected port just for this test. + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).LoopbackMode().Config(), oc.Interfaces_LoopbackModeType_FACILITY) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).LoopbackMode().State(), loopbackStateTimeout, oc.Interfaces_LoopbackModeType_FACILITY) + + var bad bool + var i int + + // Iterate up to 10 times to get a successful test. + for i = 1; i <= 10; i++ { + t.Logf("\n----- TestGNMIEthernetInIPv6Discards: Iteration %v -----\n", i) + bad = false + + // Read all the relevant counters initial values. + before := ReadCounters(t, dut, intf) + + // Compute the expected counters after the test.. Since + // we're seeing some discard traffic (1 or 2 per second) during + // normal operation on the Ondatra testbeds with loopback + // turned on, setting the number of packets to be sent larger + // so we can actually verify its those packets that we got. + expect := before + expect.outPkts += pktsPer + expect.outOctets += pktsPer * 64 + expect.outUnicastPkts += pktsPer + expect.inPkts += pktsPer + expect.inOctets += pktsPer * 64 + expect.inUnicastPkts += pktsPer + expect.inDiscards += pktsPer + expect.inIPv6Discards += pktsPer + + // Construct a simple IPv6 packet that will get discarded. + // Construct a simple IPv6 packet. + eth := &layers.Ethernet{ + SrcMAC: net.HardwareAddr{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}, + DstMAC: net.HardwareAddr{0x00, 0x1A, 0x11, 0x17, 0x5F, 0x80}, + EthernetType: layers.EthernetTypeIPv6, + } + + ip := &layers.IPv6{ + Version: 6, + HopLimit: 0, + SrcIP: net.ParseIP("2001:db8::1"), + DstIP: net.ParseIP("2001:db8::2"), + NextHeader: layers.IPProtocol(0), + } + + buf := gopacket.NewSerializeBuffer() + + // Enable reconstruction of length and checksum fields based on packet headers. + opts := gopacket.SerializeOptions{ + FixLengths: true, + ComputeChecksums: true, + } + + if err := gopacket.SerializeLayers(buf, opts, eth, ip); err != nil { + t.Fatalf("Failed to serialize packet (%v)", err) + } + + packetOut := &testhelper.PacketOut{ + EgressPort: intf, + Count: uint(pktsPer), + Interval: 1 * time.Millisecond, + Packet: buf.Bytes(), + } + + p4rtClient, err := testhelper.FetchP4RTClient(t, dut, dut.RawAPIs().P4RT(t), nil) + if err != nil { + t.Fatalf("Failed to create P4RT client: %v", err) + } + if err := p4rtClient.SendPacketOut(t, packetOut); err != nil { + t.Fatalf("SendPacketOut operation failed for %+v (%v)", packetOut, err) + } + + // Sleep for enough time that the counters are polled after the + // transmit completes sending bytes. + time.Sleep(counterUpdateDelay) + + // Read all the relevant counters again. + after := ReadCounters(t, dut, intf) + + if after != expect { + ShowCountersDelta(t, before, after, expect) + bad = true + } + + if !bad { + break + } + } + + if bad { + t.Fatalf("\n\n----- TestGNMIEthernetInIPv6Discards: FAILED after %v Iterations -----\n\n", i-1) + } + + t.Logf("\n\n----- TestGNMIEthernetInIPv6Discards: SUCCESS after %v Iteration(s) -----\n\n", i) +} diff --git a/sdn_tests/pins_ondatra/tests/gnmi_get_modes_test.go b/sdn_tests/pins_ondatra/tests/gnmi_get_modes_test.go index 66144de4b6..d557672135 100644 --- a/sdn_tests/pins_ondatra/tests/gnmi_get_modes_test.go +++ b/sdn_tests/pins_ondatra/tests/gnmi_get_modes_test.go @@ -1,18 +1,23 @@ package gnmi_get_modes_test import ( + "fmt" "context" "encoding/json" "strings" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/openconfig/gnmi/value" "github.com/openconfig/ondatra" "github.com/openconfig/ygot/ygot" + "github.com/openconfig/ondatra/gnmi" "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/binding/pinsbind" "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/testhelper/testhelper" "google.golang.org/grpc" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/encoding/prototext" gpb "github.com/openconfig/gnmi/proto/gnmi" ) @@ -761,7 +766,7 @@ func createAndValidateLeafRequest(t *testing.T, dut *ondatra.DUTDevice, paths [] // Test for gNMI GET consistency for specified data type with ALL type at leaf level. func (c getDataTypeTest) consistencyCheckLeafLevel(t *testing.T) { t.Helper() - defer esthelper.NewTearDownOptions(t).WithID(c.uuid).Teardown(t) + defer testhelper.NewTearDownOptions(t).WithID(c.uuid).Teardown(t) dut := ondatra.DUT(t, "DUT") sPath, err := ygot.StringToStructuredPath(c.reqPath) @@ -829,6 +834,21 @@ func (c getDataTypeTest) consistencyCheckSubtreeLevel(t *testing.T) { } } +// This test exposes an issue with /system/mount-points paths +func TestGetAllEqualsConfigStateOperationalWithRoot(t *testing.T) { + t.Skip("This isn't a tracked test, but it reveals behavior that requires additional investigation") + var paths []*gpb.Path + verifyGetAllEqualsConfigStateOperational(t, "--Not currently a tracked test--", paths) +} + +func TestGetAllEqualsConfigStateOperational(t *testing.T) { + sPath, err := ygot.StringToStructuredPath("/interfaces/") + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + verifyGetAllEqualsConfigStateOperational(t, "f49b3091-97d9-4bf0-b82d-712acf7ffba8", []*gpb.Path{sPath}) +} + func verifyGetAllEqualsConfigStateOperational(t *testing.T, tid string, paths []*gpb.Path) { defer testhelper.NewTearDownOptions(t).WithID(tid).Teardown(t) dut := ondatra.DUT(t, "DUT") @@ -902,3 +922,194 @@ func verifyGetAllEqualsConfigStateOperational(t *testing.T, tid string, paths [] t.Fatalf("(%v): Found %v ALL updates missing from CSO updates set:\n%v\n\nFound %v CSO updates missing from ALL updates set:\n%v", t.Name(), len(missesFromCSO), missesFromCSO, len(missesFromAll), missesFromAll) } } + +func TestGetConsistencyOperationalSubtree(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("b3bc19aa-defe-41be-8344-9ad30460136f").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + sPath, err := ygot.StringToStructuredPath(fmt.Sprintf(compStatePath, "os0")) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + paths := []*gpb.Path{sPath} + + stateNotifs, err := notificationsFromGetRequest(t, dut, createGetRequest(dut, paths, gpb.GetRequest_STATE)) + if err != nil { + t.Fatalf(err.Error()) + } + operNotifs, err := notificationsFromGetRequest(t, dut, createGetRequest(dut, paths, gpb.GetRequest_OPERATIONAL)) + if err != nil { + t.Fatalf(err.Error()) + } + allNotifs, err := notificationsFromGetRequest(t, dut, createGetRequest(dut, paths, gpb.GetRequest_ALL)) + if err != nil { + t.Fatalf(err.Error()) + } + + // Build sets from both the STATE and ALL notifications + updateSetSlice := make([]map[string]bool, 2) + for i, notifs := range [][]*gpb.Notification{stateNotifs, allNotifs} { + updateSetSlice[i] = make(map[string]bool) + for _, notif := range notifs { + updates := notif.GetUpdate() + if len(updates) == 0 { + continue + } + for _, update := range updates { + updateSetSlice[i][update.String()] = true + } + } + } + // Confirm that every OPERATIONAL update is present in both STATE/ALL updates + var misses []string + for i := range updateSetSlice { + for _, notif := range operNotifs { + updates := notif.GetUpdate() + if len(updates) == 0 { + continue + } + for _, update := range updates { + if _, ok := updateSetSlice[i][update.String()]; !ok { + misses = append(misses, update.String()) + } + } + } + } + if len(misses) > 0 { + t.Fatalf("(%v): Found %v OPER updates missing:\n%v", t.Name(), len(misses), misses) + } +} + +func TestGetInvalidLeaves(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("7e81cbdf-a113-47a4-851c-1df917646c01").Teardown(t) + dut := ondatra.DUT(t, "DUT") + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + types := []gpb.GetRequest_DataType{gpb.GetRequest_STATE, gpb.GetRequest_CONFIG, gpb.GetRequest_OPERATIONAL, gpb.GetRequest_ALL} + invalidPaths := []string{ + "/interfaces/interface[name=%s]/config/fake-leaf", + "/interfaces/interface[name=%s]/state/fake-leaf", + "/interfaces/interface[name=%s]/state/counters/fake-counter", + "/interfaces/interface[name=%s]/state/fake-leaf"} + if len(types) != len(invalidPaths) { + t.Fatalf("types and invalidPaths should be the same size") + } + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + for i := range invalidPaths { + sPath, err := ygot.StringToStructuredPath(fmt.Sprintf(invalidPaths[i], intf)) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + paths := []*gpb.Path{sPath} + if _, err := gnmiClient.Get(ctx, createGetRequest(dut, paths, types[i])); err == nil { + t.Fatalf("Expected an error with this invalid path(%v)", invalidPaths[i]) + } + } +} + +func TestGetInvalidTypesReturnError(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("5a46b9d1-9f9a-4567-a852-93eb2548f3f6").Teardown(t) + dut := ondatra.DUT(t, "DUT") + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + types := []gpb.GetRequest_DataType{4, 5, 6, 7} + validPaths := []string{ + "/interfaces/interface[name=%s]/config", + "/interfaces/interface[name=%s]/state", + "/interfaces/interface[name=%s]/state/counters", + "/interfaces/interface[name=%s]/state"} + if len(types) != len(validPaths) { + t.Fatalf("types and invalidPaths should be the same size") + } + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + for i := range validPaths { + path := fmt.Sprintf(validPaths[i], intf) + sPath, err := ygot.StringToStructuredPath(path) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + paths := []*gpb.Path{sPath} + + if _, err := gnmiClient.Get(ctx, createGetRequest(dut, paths, types[i])); err == nil { + t.Fatalf("No error received for Get with invalid type. Expected an error with invalid type (%v) for path (%v)", types[i], path) + } + } +} + +func TestMissingTypeAssumesAll(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("1f3f5692-47a6-4c47-ac05-96d705752883").Teardown(t) + dut := ondatra.DUT(t, "DUT") + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + types := []gpb.GetRequest_DataType{4, 5, 6, 7} + validPaths := []string{ + "/interfaces/interface[name=%s]/config", + "/interfaces/interface[name=%s]/state", + "/interfaces/interface[name=%s]/state/counters", + "/interfaces/interface[name=%s]/state"} + if len(types) != len(validPaths) { + t.Fatalf("types and invalidPaths should be the same size") + } + for i := range validPaths { + path := fmt.Sprintf(validPaths[i], intf) + sPath, err := ygot.StringToStructuredPath(path) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + paths := []*gpb.Path{sPath} + + // Get notifications from a Get request without an explicit type specified + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + notifs, err := notificationsFromGetRequest(t, dut, + &gpb.GetRequest{ + Prefix: prefix, + Path: paths, + // Type: OMITED + Encoding: gpb.Encoding_PROTO, + }) + if err != nil { + t.Fatalf(err.Error()) + } + + // Verify response completeness + i := 0 + for _, notif := range notifs { + pathRootStr, err := ygot.PathToString(&gpb.Path{Elem: notif.GetPrefix().GetElem()}) + if err != nil { + t.Fatalf("failed to convert elems (%v) to string: %v", notif.GetPrefix().GetElem(), err) + } + updates := notif.GetUpdate() + if len(updates) == 0 { + continue + } + for _, update := range updates { + updatePath, err := ygot.PathToString(update.GetPath()) + if err != nil { + t.Fatalf("(%v): failed to convert path (%v) to string (%v): %v", t.Name(), updatePath, prototext.Format(update), err) + } + fullPath := pathRootStr + updatePath + if !strings.HasPrefix(fullPath, path) { + t.Fatalf("(%v): Expected path (%v) to have prefix(%v)", t.Name(), fullPath, path) + } + i++ + } + if i == 0 { + t.Fatalf("(%v): No updates returned for path (%v)", t.Name(), path) + } + } + } +} diff --git a/sdn_tests/pins_ondatra/tests/gnmi_helper.go b/sdn_tests/pins_ondatra/tests/gnmi_helper.go new file mode 100644 index 0000000000..6769b8c6d1 --- /dev/null +++ b/sdn_tests/pins_ondatra/tests/gnmi_helper.go @@ -0,0 +1,987 @@ +package gnmi_stress_helper + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "strconv" + "sync" + "testing" + "time" + + "github.com/openconfig/ondatra" + "github.com/openconfig/ygot/ygot" + "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/testhelper/testhelper" + "google.golang.org/grpc" + "google.golang.org/protobuf/encoding/prototext" + + gpb "github.com/openconfig/gnmi/proto/gnmi" + "github.com/openconfig/gnmi/value" +) + +// PathInfo structure defines the path info. +type PathInfo struct { + path string + payload string + expectedResult bool + expectedResponse any + isUsingRandomIntf bool +} + +// Paths are used in get and set tests randomly. +var Paths = []PathInfo{ + PathInfo{ + path: "/interfaces/interface[name=%s]/config/mtu", + payload: strconv.FormatUint(uint64(9216), 10), + expectedResult: true, + expectedResponse: uint64(9216), + isUsingRandomIntf: true, + }, + PathInfo{ + path: "/interfaces/interface[name=%s]/config/description", + payload: "\"test\"", + expectedResult: true, + expectedResponse: "\"test\"", + isUsingRandomIntf: true, + }, + PathInfo{ + path: "/interfaces/interface[name=%s]/config/enabled", + payload: strconv.FormatBool(true), + expectedResult: true, + expectedResponse: true, + isUsingRandomIntf: true, + }, + PathInfo{ + path: "/interfaces/interface[name=%s]/config/xyz", + payload: strconv.FormatBool(true), + expectedResult: false, + expectedResponse: `{}`, + isUsingRandomIntf: true, + }, + PathInfo{ + path: "/interfaces/interface[name=%s]/config/description", + payload: "\"This is a description from gnmi helper.\"", + expectedResult: true, + expectedResponse: "\"This is a description from gnmi helper.\"", + isUsingRandomIntf: true, + }, + PathInfo{ + path: "/interfaces/interface[name=%s]/config/health-indicator", + payload: "\"GOOD\"", + expectedResult: true, + expectedResponse: "\"GOOD\"", + isUsingRandomIntf: true, + }, + PathInfo{ + path: "/interfaces/interface[name=%s]/config/fully-qualified-interface-name", + payload: "\"test_interface\"", + expectedResult: false, + expectedResponse: "\"test_interface\"", + isUsingRandomIntf: true, + }, + PathInfo{ + path: "/openconfig-platform:components/abc", + payload: "{name: chassis}", + expectedResult: false, + expectedResponse: `{}`, + isUsingRandomIntf: false, + }, +} + +// Path list is a set of random interface paths. +var Path = []string{ + "/interfaces/interface[name=%s]/config/mtu", + "/interfaces/interface[name=%s]/config/enabled", + "/interfaces/interface[name=%s]/state/type", + "/interfaces/interface[name=%s]/state/cpu", +} + +// DeletePaths is a set of random interface paths for delete operations. +var DeletePaths = []PathInfo{ + PathInfo{ + path: "/interfaces/interface[name=%s]/config/description", + payload: "\"test_interface\"", + }, +} + +// DelSubtree list is the possible combination of gNMI path subtrees. +var DelSubtree = []string{ + "qos/forwarding-groups/", + "qos/queues/", +} + +// Subtree list is the possible combination of gNMI path subtrees. +var Subtree = []string{ + "interfaces/", + "qos/", + "system/", +} + +// list of gNMI operations +var ops = []string{ + "get", + "set", + "subscribe", +} + +// The following payload used as config push payload during set stress tests. +const ( + ShortStressTestInterval = 600000000000 // 10 minute interval in ns + LongStressTestInterval = 28800000000000 // 8 hour interval in ns + IdleTime = 10 // 10 seconds for the DUT to cool down + MinIteration = 6 + AvgIteration = 20 + MinMtuStepInc = 100 + MaxMtuStepInc = 200 + SampleInterval = 2000000000 + Timeout = 3 * time.Second + UpdatesOnly = true +) + +// ConfigPush function to push config via gNMI raw Set. +func ConfigPush(t *testing.T, dut *ondatra.DUTDevice) { + // Create setRequest message. + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Replace: []*gpb.Update{{ + Path: &gpb.Path{Elem: []*gpb.PathElem{{Name: "/"}}}, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: []byte("")}}, + }}, + } + + // Fetch set client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + setResp, err := gnmiClient.Set(ctx, setRequest) + if err != nil { + t.Fatalf("Error while calling Set API during config push: (%v)", err) + } + t.Logf("SetResponse:\n%v", setResp) +} + +// SanityCheck function validates the sanity of the DUT +func SanityCheck(t *testing.T, dut *ondatra.DUTDevice, ports ...string) { + t.Helper() + if err := testhelper.GNOIAble(t, dut); err != nil { + t.Fatalf("gNOI server is not running in the DUT") + } + if err := testhelper.GNMIAble(t, dut); err != nil { + t.Fatalf("gNMI server is not running in the DUT") + } + if ports != nil { + if err := testhelper.VerifyPortsOperStatus(t, dut, ports...); err != nil { + t.Logf("Ports %v oper status is not up", ports) + t.Fatalf("Ports are not oper upT") + } + } +} + +// CollectPerformanceMetrics collect the system performance metrics via gNMI get +func CollectPerformanceMetrics(t *testing.T, dut *ondatra.DUTDevice) { + t.Helper() + // TODO: Receiving DB connection error for both process and memory path, + // backend is not implemented yet. The following code block can be + // uncommented out once the implementation is complete + /* memory := dut.Telemetry().System().Memory().Get(t) + t.Logf("System memory details:", memory.Physical, memory.Reserved) + + // Create getRequest message with ASCII encoding. + getRequest := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: []*gpb.Path{{ + Elem: []*gpb.PathElem{{ + Name: "system", + }, { + Name: "processes", + }}, + }}, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_PROTO, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + getResp, err := gnmiClient.Get(ctx, getRequest) + if err != nil { + t.Fatalf("Unable to fetch get client (%v)", err) + } + if getResp == nil { + t.Fatalf("Unable to fetch get client, get response is nil") + } + t.Logf("System Processes Info: %v", getResp) + */ +} + +// StressTestHelper function to invoke various gNMI set and get operations +func StressTestHelper(t *testing.T, dut *ondatra.DUTDevice, interval time.Duration) { + SanityCheck(t, dut) + rand.Seed(time.Now().Unix()) + CollectPerformanceMetrics(t, dut) + t.Logf("Interval : %v", interval) + + // Simple gNMI get request followed by a gNMI set replace to stress the DUT. + for timeout := time.Now().Add(interval); time.Now().Before(timeout); { + port, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + pathInfo := Paths[rand.Intn(len(Paths))] + path := pathInfo.path + if pathInfo.isUsingRandomIntf == true { + path = fmt.Sprintf(pathInfo.path, port) + } + t.Logf("path : %v", path) + // Create set the Request. + sPath, err := ygot.StringToStructuredPath(path) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + pathList := []*gpb.Path{sPath} + + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Update: []*gpb.Update{{ + Path: sPath, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: []byte(pathInfo.payload)}}, + }}, + } + t.Logf("SetRequest:\n%v", setRequest) + // Fetch set client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + setResp, err := gnmiClient.Set(ctx, setRequest) + if pathInfo.expectedResult == true && err != nil { + t.Fatalf("Unable to fetch set client (%v)", err) + } + t.Logf("SetResponse:\n%v", setResp) + + // Create getRequest message with data type. + getRequest := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: pathList, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_PROTO, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + getResp, err := gnmiClient.Get(ctx, getRequest) + if pathInfo.expectedResult == true && err != nil { + t.Fatalf("Error while calling Get Raw API: (%v)", err) + } + + if pathInfo.expectedResult == true && getResp == nil { + t.Fatalf("Get response is nil") + } + t.Logf("GetResponse:\n%v", getResp) + CollectPerformanceMetrics(t, dut) + } + t.Logf("After 10 seconds of idle time, the performance metrics are:") + time.Sleep(IdleTime * time.Second) + CollectPerformanceMetrics(t, dut) + SanityCheck(t, dut) + +} + +// StressSetTestHelper function to invoke various gNMI set and get operations +func StressSetTestHelper(t *testing.T, dut *ondatra.DUTDevice, interval int, replace bool) { + SanityCheck(t, dut) + rand.Seed(time.Now().Unix()) + CollectPerformanceMetrics(t, dut) + t.Logf("Interval : %v", interval) + + // Simple gNMI get request followed by a gNMI set replace to stress the DUT. + for i := 0; i < interval; i++ { + port, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + pathInfo := Paths[rand.Intn(len(Paths))] + path := pathInfo.path + if pathInfo.isUsingRandomIntf == true { + path = fmt.Sprintf(pathInfo.path, port) + } + t.Logf("path : %v", path) + // Create set the Request. + sPath, err := ygot.StringToStructuredPath(path) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + paths := []*gpb.Path{sPath} + getResp := &gpb.GetResponse{} + + // Create getRequest message with data type. + getRequest := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: paths, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_JSON_IETF, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + if pathInfo.expectedResult == true { + getResp, err = gnmiClient.Get(context.Background(), getRequest) + if err == nil { + t.Logf("The path is not populated") + } + } + + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Update: []*gpb.Update{{ + Path: sPath, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: []byte(pathInfo.payload)}}, + }}, + } + if replace == true { + setRequest = &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Replace: []*gpb.Update{{ + Path: sPath, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: []byte(pathInfo.payload)}}, + }}, + } + } + t.Logf("SetRequest:\n%v", setRequest) + // Fetch set client using the raw gNMI client. + setResp, err := gnmiClient.Set(context.Background(), setRequest) + if pathInfo.expectedResult == true && err != nil { + t.Fatalf("Unable to fetch set client (%v)", err) + } + t.Logf("SetResponse:\n%v", setResp) + CollectPerformanceMetrics(t, dut) + + // Restore the old values for the path if the above set resulted in changing the values + if getResp != nil && pathInfo.expectedResult == true { + updates, err := UpdatesWithJSONIETF(getResp) + if err != nil { + t.Fatalf("Unable to get updates with JSON IETF: (%v)", err) + } + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Replace: updates, + } + setResp, err := gnmiClient.Set(context.Background(), setRequest) + if err != nil { + t.Fatalf("Unable to restore the original value using set client (%v)", err) + } + t.Logf("SetResponse:\n%v", setResp) + } + + } + t.Logf("After 10 seconds of idle time, the performance metrics are:") + time.Sleep(IdleTime * time.Second) + CollectPerformanceMetrics(t, dut) + SanityCheck(t, dut) + +} + +// ParseGetResponseHelper function will parse the gNMI get response to get the value +func ParseGetResponseHelper(t *testing.T, getResp *gpb.GetResponse) string { + // Validate GET response. + notifs := getResp.GetNotification() + if len(notifs) != 1 { + t.Fatalf("got %d notifications, want 1", len(notifs)) + } + notif, updates := notifs[0], notifs[0].GetUpdate() + if len(updates) < 1 { + t.Fatalf("got %d updates in the notification, want 1", len(updates)) + } + pathStr, err := ygot.PathToString(&gpb.Path{Elem: notif.GetPrefix().GetElem()}) + if err != nil { + t.Fatalf("failed to convert elems (%v) to string: %v", notif.GetPrefix().GetElem(), err) + } + + updatePath, err := ygot.PathToString(updates[0].GetPath()) + if err != nil { + t.Fatalf("failed to convert path to string (%v): %v", updatePath, err) + } + gotPath := updatePath + if pathStr != "/" { + gotPath = pathStr + updatePath + } + t.Logf("The path is in the response:%v", gotPath) + val := updates[0].GetVal() + + var gotVal any + if val.GetJsonIetfVal() == nil { + // Get Scalar value. + gotVal, err = value.ToScalar(val) + if err != nil { + t.Logf("got %v, want scalar value", gotVal) + } + gotVal = fmt.Sprintf("%v", gotVal) + } else { + // Unmarshal json data to container. + if err := json.Unmarshal(val.GetJsonIetfVal(), &gotVal); err != nil { + t.Logf("could not unmarshal json data to container: %v", err) + } + } + return gotVal.(string) +} + +// SetDifferentClientTest function to invoke set operation from different clients +func SetDifferentClientTest(t *testing.T, dut *ondatra.DUTDevice, replace bool) { + SanityCheck(t, dut) + ctx := context.Background() + newGNMIClient := func() gpb.GNMIClient { + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + return gnmiClient + } + clients := map[string]gpb.GNMIClient{ + "c1": newGNMIClient(), + "c2": newGNMIClient(), + "c3": newGNMIClient(), + "c4": newGNMIClient(), + "c5": newGNMIClient(), + "c6": newGNMIClient(), + "c7": newGNMIClient(), + "c8": newGNMIClient(), + "c9": newGNMIClient(), + "c10": newGNMIClient(), + } + + var wg sync.WaitGroup + for k := range clients { + v := k + wg.Add(1) + rand.Seed(time.Now().Unix()) + CollectPerformanceMetrics(t, dut) + port, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + // Create Set Request. + pathInfo := Paths[rand.Intn(len(Paths))] + path := pathInfo.path + if pathInfo.isUsingRandomIntf == true { + path = fmt.Sprintf(pathInfo.path, port) + } + t.Logf("path : %v", path) + // Create set the Request. + sPath, err := ygot.StringToStructuredPath(path) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Update: []*gpb.Update{{ + Path: sPath, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: []byte(pathInfo.payload)}}, + }}, + } + if replace == true { + setRequest = &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Replace: []*gpb.Update{{ + Path: sPath, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: []byte(pathInfo.payload)}}, + }}, + } + } + t.Logf("SetRequest:\n%v", setRequest) + + // Fetch get client using the raw gNMI client. + go func() { + setResp, err := clients[v].Set(context.Background(), setRequest) + if err != nil { + t.Log("Error while calling Get Raw API") + } + if setResp == nil { + t.Log("Get response is nil") + } + t.Logf("GetResponse:\n%v", setResp) + wg.Done() + }() + + CollectPerformanceMetrics(t, dut) + } + wg.Wait() + t.Logf("After 10 seconds of idle time, the performance metrics are:") + time.Sleep(IdleTime * time.Second) + CollectPerformanceMetrics(t, dut) + SanityCheck(t, dut) +} + +// SetDefaultValuesHelper function will set the default values for the paths if it doesn't exist already. +func SetDefaultValuesHelper(t *testing.T, dut *ondatra.DUTDevice, port string) { + for i := 0; i < len(DeletePaths); i++ { + pathInfo := DeletePaths[i] + reqPath := fmt.Sprintf(pathInfo.path, port) + // Create Set Request. + sPath, err := ygot.StringToStructuredPath(reqPath) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + // Set the default value for the path if there is none + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Update: []*gpb.Update{{ + Path: sPath, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: []byte(pathInfo.payload)}}, + }}, + } + t.Logf("SetRequest:\n%v", setRequest) + // Fetch set client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + setResp, err := gnmiClient.Set(ctx, setRequest) + if err != nil { + t.Fatalf("Unable to fetch set client (%v)", err) + } + t.Logf("SetResponse:\n%v", setResp) + } +} + +// RandomDeletePath function will choose a ramdom path from list of paths +func RandomDeletePath(port string) string { + pathInfo := DeletePaths[rand.Intn(len(DeletePaths))] + reqPath := fmt.Sprintf(pathInfo.path, port) + return reqPath +} + +// VerifyFullResponse function will verify the subscription response message +func VerifyFullResponse(t *testing.T, subClient gpb.GNMI_SubscribeClient, timeout time.Duration) { + t.Helper() + // Process response from DUT. + for { + // Wait for response from DUT. + res, err := subClient.Recv() + if err != nil { + t.Fatalf("Response error received from DUT (%v)", err) + } + + switch v := res.Response.(type) { + case *gpb.SubscribeResponse_Update: + // Process Update message received in SubscribeResponse. + updates := v.Update + // Perform basic sanity on the Update message. + for _, update := range updates.GetUpdate() { + if update.Path == nil { + t.Errorf("Invalid nil Path in update: %v", prototext.Format(update)) + continue + } + if update.Val == nil { + t.Errorf("Invalid nil Val in update: %v", prototext.Format(update)) + continue + } + + // Path is partially present in Prefix and partially in Update in the response. + prefixStr, err := ygot.PathToString(updates.GetPrefix()) + if err != nil { + t.Errorf("Failed to convert path to string (%v) %v", err, updates.GetPrefix()) + continue + } + elemStr, err := ygot.PathToString(update.Path) + if err != nil { + t.Errorf("Failed to convert path to string (%v) %v", err, update.Path) + continue + } + pathStr := prefixStr + elemStr + t.Logf("Path in the response:%v", pathStr) + } + return + } + } +} + +// StressTestSubsHelper function to invoke various subscription operations +func StressTestSubsHelper(t *testing.T, dut *ondatra.DUTDevice, subtree bool, poll bool) { + SanityCheck(t, dut) + rand.Seed(time.Now().Unix()) + CollectPerformanceMetrics(t, dut) + for i := 0; i < MinIteration; i++ { + port, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + reqPath := fmt.Sprintf(Path[rand.Intn(len(Path))], port) + if subtree == true { + reqPath = fmt.Sprintf(Subtree[rand.Intn(len(Subtree))]) + } + // Create Subscribe Request. + sPath, err := ygot.StringToStructuredPath(reqPath) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + req := &gpb.SubscribeRequest{ + Request: &gpb.SubscribeRequest_Subscribe{ + Subscribe: &gpb.SubscriptionList{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Subscription: []*gpb.Subscription{ + &gpb.Subscription{ + Path: &gpb.Path{Elem: sPath.Elem}, + Mode: gpb.SubscriptionMode_SAMPLE, + SampleInterval: SampleInterval, + }}, + Mode: gpb.SubscriptionList_STREAM, + Encoding: gpb.Encoding_PROTO, + UpdatesOnly: UpdatesOnly, + }, + }, + } + if poll == true { + req = &gpb.SubscribeRequest{ + Request: &gpb.SubscribeRequest_Subscribe{ + Subscribe: &gpb.SubscriptionList{ + Subscription: []*gpb.Subscription{ + &gpb.Subscription{ + Path: &gpb.Path{Elem: sPath.Elem}, + }}, + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Mode: gpb.SubscriptionList_POLL, + Encoding: gpb.Encoding_PROTO, + }, + }, + } + } + t.Logf("Subscribe request:%v", req) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + subscribeClient, err := gnmiClient.Subscribe(ctx) + if err != nil { + t.Fatalf("Unable to get subscribe client (%v)", err) + } + + if err := subscribeClient.Send(req); err != nil { + t.Fatalf("Failed to send gNMI subscribe request (%v)", err) + } + if _, err := subscribeClient.Recv(); err != nil { + t.Fatalf("Failed to receive gNMI sample subscribe request (%v)", err) + } + CollectPerformanceMetrics(t, dut) + } + t.Logf("After %v seconds of idle time, the performance metrics are collected", IdleTime) + time.Sleep(IdleTime * time.Second) + CollectPerformanceMetrics(t, dut) + SanityCheck(t, dut) +} + +// SubscribeDifferentClientTest function to invoke set operation from different clients +func SubscribeDifferentClientTest(t *testing.T, dut *ondatra.DUTDevice, poll bool) { + SanityCheck(t, dut) + clients := map[string]gpb.GNMIClient{} + ctx := context.Background() + for i := 1; i <= 10; i++ { + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + clients[fmt.Sprintf("c%d", i)] = gnmiClient + } + + var wg sync.WaitGroup + for k := range clients { + v := k + wg.Add(1) + SanityCheck(t, dut) + rand.Seed(time.Now().Unix()) + CollectPerformanceMetrics(t, dut) + port, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + reqPath := fmt.Sprintf(Path[rand.Intn(len(Path))], port) + // Create Subscribe Request. + sPath, err := ygot.StringToStructuredPath(reqPath) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + req := &gpb.SubscribeRequest{ + Request: &gpb.SubscribeRequest_Subscribe{ + Subscribe: &gpb.SubscriptionList{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Subscription: []*gpb.Subscription{ + &gpb.Subscription{ + Path: &gpb.Path{Elem: sPath.Elem}, + Mode: gpb.SubscriptionMode_SAMPLE, + SampleInterval: SampleInterval, + }}, + Mode: gpb.SubscriptionList_STREAM, + Encoding: gpb.Encoding_PROTO, + UpdatesOnly: UpdatesOnly, + }, + }, + } + if poll == true { + req = &gpb.SubscribeRequest{ + Request: &gpb.SubscribeRequest_Subscribe{ + Subscribe: &gpb.SubscriptionList{ + Subscription: []*gpb.Subscription{ + &gpb.Subscription{ + Path: &gpb.Path{Elem: sPath.Elem}, + }}, + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Mode: gpb.SubscriptionList_POLL, + Encoding: gpb.Encoding_PROTO, + }, + }, + } + } + t.Logf("Subscribe request:%v", req) + // Fetch get client using the raw gNMI client. + go func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + client, err := clients[v].Subscribe(ctx) + if err != nil { + t.Logf("Unable to get subscribe client (%v)", err) + } + if err := client.Send(req); err != nil { + t.Logf("Failed to send gNMI subscribe request (%v)", err) + } + _, error := client.Recv() + if error != nil { + t.Logf("Response error received from DUT (%v)", error) + } + CollectPerformanceMetrics(t, dut) + wg.Done() + }() + } + wg.Wait() + t.Logf("After %v seconds of idle time, the performance metrics are collected", IdleTime) + time.Sleep(IdleTime * time.Second) + CollectPerformanceMetrics(t, dut) + SanityCheck(t, dut) +} + +func selectgNMIPathHelper(t *testing.T, ops string, port string) (string, string, bool) { + t.Helper() + var path string + if ops == "set" { + pathInfo := Paths[rand.Intn(len(Paths))] + path = pathInfo.path + if pathInfo.isUsingRandomIntf == true { + path = fmt.Sprintf(pathInfo.path, port) + } + return path, pathInfo.payload, pathInfo.expectedResult + } + path = fmt.Sprintf(Path[rand.Intn(len(Path))], port) + return path, "", false +} + +// RandomDifferentClientTestHelper function to invoke various gNMI operation from different clients +func RandomDifferentClientTestHelper(t *testing.T, dut *ondatra.DUTDevice, interval time.Duration) { + SanityCheck(t, dut) + clients := map[string]gpb.GNMIClient{} + ctx := context.Background() + newGNMIClient := func() gpb.GNMIClient { + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + return gnmiClient + } + for i := 1; i <= MinIteration; i++ { + clients[fmt.Sprintf("c%d", i)] = newGNMIClient() + } + + var wg sync.WaitGroup + t.Logf("Interval : %v", interval) + for timeout := time.Now().Add(interval); time.Now().Before(timeout); { + // Create a gNMI Request. + rand.Seed(time.Now().Unix()) + ops := ops[rand.Intn(len(ops))] + port, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + path, payload, expectedResult := selectgNMIPathHelper(t, ops, port) + t.Logf("The path expect the valid result: %v", expectedResult) + sPath, err := ygot.StringToStructuredPath(path) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + for k := range clients { + v := k + wg.Add(1) + CollectPerformanceMetrics(t, dut) + if ops == "get" { + // Create getRequest message with data type. + req := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: []*gpb.Path{sPath}, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_PROTO, + } + t.Logf("GetRequest:\n%v", req) + // Fetch get client using the raw gNMI client. + go func() { + setResp, err := clients[v].Get(ctx, req) + if err != nil { + t.Log("Error while calling Get Raw API") + } + if setResp == nil { + t.Log("Get response is nil") + } + t.Logf("GetResponse:\n%v", setResp) + wg.Done() + }() + } else if ops == "set" { + paths := []*gpb.Path{sPath} + getResp := &gpb.GetResponse{} + + // Create getRequest message with data type. + getRequest := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: paths, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_JSON_IETF, + } + t.Logf("GetRequest:\n%v", getRequest) + + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + + // Fetch get client using the raw gNMI client. + if expectedResult == true { + getResp, err = gnmiClient.Get(ctx, getRequest) + if err == nil { + t.Logf("The path is not populated") + } + } + req := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Update: []*gpb.Update{{ + Path: sPath, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: []byte(payload)}}, + }}, + } + t.Logf("SetRequest:\n%v", req) + // Fetch set client using the raw gNMI client. + go func() { + setResp, err := clients[v].Set(ctx, req) + if err != nil { + t.Logf("Error while calling Set Raw API: %v\n", err) + } + if setResp == nil { + t.Log("Set response is nil") + } + t.Logf("SetResponse:\n%v", setResp) + wg.Done() + }() + // Restore the old values for the path if the above set resulted in changing the values + if getResp != nil && expectedResult == true { + updates, err := UpdatesWithJSONIETF(getResp) + if err != nil { + t.Fatalf("Unable to get updates with JSON IETF: (%v)", err) + } + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Replace: updates, + } + setResp, err := gnmiClient.Set(ctx, setRequest) + if err != nil { + t.Fatalf("Unable to restore the original value using set client (%v)", err) + } + t.Logf("SetResponse:\n%v", setResp) + } + } else { + req := &gpb.SubscribeRequest{ + Request: &gpb.SubscribeRequest_Subscribe{ + Subscribe: &gpb.SubscriptionList{ + Subscription: []*gpb.Subscription{ + &gpb.Subscription{ + Path: &gpb.Path{Elem: sPath.Elem}, + }}, + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Mode: gpb.SubscriptionList_ONCE, + Encoding: gpb.Encoding_PROTO, + }, + }, + } + t.Logf("Subscribe request:%v", req) + // Fetch get client using the raw gNMI client. + go func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + client, err := clients[v].Subscribe(ctx) + if err != nil { + t.Logf("Unable to get subscribe client (%v)", err) + } + if err := client.Send(req); err != nil { + t.Logf("Failed to send gNMI subscribe request (%v)", err) + } + _, error := client.Recv() + if error != nil { + t.Logf("Response error received from DUT (%v)", error) + } + wg.Done() + }() + } + CollectPerformanceMetrics(t, dut) + } + wg.Wait() + } + t.Logf("After %v seconds of idle time, the performance metrics are collected", IdleTime) + time.Sleep(IdleTime * time.Second) + CollectPerformanceMetrics(t, dut) + SanityCheck(t, dut) +} + +// UpdatesWithJSONIETF parses a Get Response and returns the Updates in the correct format +// to be used in a Set Request. This is useful for restoring the contents of a Get Response. The +// Get Response must be encoded in JSON IETF, specified by the Get Request. +func UpdatesWithJSONIETF(getResp *gpb.GetResponse) ([]*gpb.Update, error) { + updates := []*gpb.Update{} + for _, notification := range getResp.GetNotification() { + if notification == nil { + return nil, fmt.Errorf("Notification in GetResponse is empty") + } + for _, update := range notification.GetUpdate() { + if update == nil { + return nil, fmt.Errorf("Update in Notification is empty") + } + jsonVal := update.GetVal().GetJsonIetfVal() + jsonMap := make(map[string]json.RawMessage) + err := json.Unmarshal(jsonVal, &jsonMap) + if err != nil { + return nil, err + } + if len(jsonMap) == 1 { + for _, v := range jsonMap { + jsonVal, err = v.MarshalJSON() + if err != nil { + return nil, err + } + } + } + updates = append(updates, &gpb.Update{ + Path: update.GetPath(), + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: jsonVal}}, + }) + } + } + return updates, nil +} diff --git a/sdn_tests/pins_ondatra/tests/gnmi_set_get_test.go b/sdn_tests/pins_ondatra/tests/gnmi_set_get_test.go index 3909d13dbc..ad35fd3f38 100644 --- a/sdn_tests/pins_ondatra/tests/gnmi_set_get_test.go +++ b/sdn_tests/pins_ondatra/tests/gnmi_set_get_test.go @@ -7,19 +7,19 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/openconfig/ondatra" "github.com/openconfig/ondatra/gnmi" "github.com/openconfig/ondatra/gnmi/oc" "github.com/openconfig/testt" "github.com/openconfig/ygnmi/ygnmi" - "github.com/openconfig/ygot/ygot" + "github.com/openconfig/ygot/ygot" "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/binding/pinsbind" "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/testhelper/testhelper" - "golang.org/x/sync/errgroup" + "golang.org/x/sync/errgroup" "google.golang.org/grpc" - "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/encoding/prototext" gpb "github.com/openconfig/gnmi/proto/gnmi" ) @@ -558,3 +558,1212 @@ func TestGNMISetReplaceInvalidLeaf(t *testing.T) { } } + +func TestGNMISetReplaceMultipleLeafsValid(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("c1f9dd81-d509-405f-bed2-e108e619b5f6").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + // Get fields from interface subtree. + res := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + descPath := gnmi.OC().Interface(intf).Description() + resolvedDescPath, _, errs := ygnmi.ResolvePath(descPath.Config().PathStruct()) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", descPath, err) + } + var desc, wantDesc string + descExist := false + if res.Description != nil { + desc = *res.Description + descExist = true + } + mtuPath := gnmi.OC().Interface(intf).Mtu() + resolvedMtuPath, _, errs := ygnmi.ResolvePath(mtuPath.Config().PathStruct()) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", mtuPath, err) + } + mtu := res.GetMtu() + // Adding 22 bytes to the existing MTU value for the test. + wantMtu := mtu + 22 + wantDesc = "wanted description" + enabled := res.GetEnabled() + fqin := testhelper.FullyQualifiedInterfaceName(t, dut, intf) + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + + defer func() { + // Replace the old values for test cleanup. + gnmi.Replace(t, dut, mtuPath.Config(), mtu) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).Mtu().State(), 5*time.Second, mtu) + if descExist { + gnmi.Replace(t, dut, descPath.Config(), desc) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).Description().State(), 20*time.Second, desc) + } else { + delRequest := &gpb.SetRequest{ + Prefix: prefix, + Delete: []*gpb.Path{resolvedDescPath}, + } + // Fetch set client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + if _, err := gnmiClient.Set(ctx, delRequest); err != nil { + t.Fatalf("Unable to fetch set delete client (%v)", err) + } + } + }() + + setRequest := &gpb.SetRequest{ + Prefix: prefix, + Replace: []*gpb.Update{ + { + Path: resolvedMtuPath, + Val: &gpb.TypedValue{ + Value: &gpb.TypedValue_JsonIetfVal{ + JsonIetfVal: []byte(strconv.FormatUint(uint64(wantMtu), 10)), + }, + }, + }, + { + Path: resolvedDescPath, + Val: &gpb.TypedValue{ + Value: &gpb.TypedValue_JsonIetfVal{ + JsonIetfVal: []byte(fmt.Sprintf("\"%s\"", wantDesc)), + }, + }, + }, + }, + } + + // Fetch raw gNMI client and call Set API to send Set Request. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + setResp, err := gnmiClient.Set(ctx, setRequest) + if err != nil { + t.Fatalf("Error while calling Set Raw API: (%v)", err) + } + t.Logf("SetResponse:\n%v", setResp) + + // Get fields from interface subtree. + intfAfterSet := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + // Verify the values are set properly using get. + if got := intfAfterSet.GetMtu(); got != wantMtu { + t.Errorf("MTU match failed! got: %v, want: %v", got, wantMtu) + } + if got := intfAfterSet.GetDescription(); got != wantDesc { + t.Errorf("Description match failed! got %v, want %v", got, wantDesc) + } + + // Verify that other leaf nodes are not changed. + if got := intfAfterSet.GetEnabled(); got != enabled { + t.Errorf("Enabled match failed! got %v, want %v", got, enabled) + } + + if got := testhelper.FullyQualifiedInterfaceName(t, dut, intf); got != fqin { + t.Errorf("FullyQualifiedInterfaceName match failed! got %v, want %v", got, fqin) + } +} + +func TestGNMISetReplaceMultipleLeafsInvalid(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("c485b29b-7e1e-4b5a-bccd-797ced277008").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + // Get fields from interface subtree. + mtuPath := gnmi.OC().Interface(intf).Mtu() + resolvedPath, _, errs := ygnmi.ResolvePath(mtuPath.Config().PathStruct()) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", mtuPath, err) + } + mtu := gnmi.Get(t, dut, mtuPath.Config()) + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + setRequest := &gpb.SetRequest{ + Prefix: prefix, + Replace: []*gpb.Update{ + { + Path: resolvedPath, + Val: &gpb.TypedValue{ + Value: &gpb.TypedValue_JsonIetfVal{ + JsonIetfVal: []byte(strconv.FormatUint(uint64(mtu+22), 10)), + }, + }, + }, + { + Path: &gpb.Path{ + Elem: []*gpb.PathElem{ + { + Name: "interfaces", + }, + { + Name: "interface", + Key: map[string]string{"name": intf}, + }, + { + Name: "config", + }, + { + Name: "openconfig-abc:xyz", + }, + }, + }, + Val: &gpb.TypedValue{ + Value: &gpb.TypedValue_JsonIetfVal{ + JsonIetfVal: []byte("987"), + }, + }, + }, + }, + } + + // Fetch raw gNMI client and call Set API to send Set Request. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + if _, err = gnmiClient.Set(ctx, setRequest); err == nil { + t.Fatalf("Set request is expected to fail but it didn't") + } + + // Verify that the MTU value did not get changed. + if got := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Mtu().Config()); got != mtu { + t.Errorf("MTU match failed! gotMtu %v, want %v", got, mtu) + } +} + +/********************************************************** +* gNMI SET Delete operations +**********************************************************/ +func TestGNMISetDeleteSingleLeaf(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("01850837-93b3-44a6-9d8f-84a0bd6c8725").Teardown(t) + dut := ondatra.DUT(t, "DUT") + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + res := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + desPath := gnmi.OC().Interface(intf).Description() + resolvedPath, _, errs := ygnmi.ResolvePath(desPath.Config().PathStruct()) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", desPath, err) + } + ctx := context.Background() + var des string = "" + var desExist bool = false + mtu := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Mtu().Config()) + if res.Description != nil { + des = *res.Description + desExist = true + } else { + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Description().Config(), "Test description from ondatra test") + } + paths := []*gpb.Path{resolvedPath} + + defer func() { + // Replace the id to original value if it exists before the test. + if desExist { + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Description().Config(), des) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).Description().State(), 5*time.Second, des) + } + }() + + delRequest := &gpb.SetRequest{ + Prefix: prefix, + Delete: paths, + } + // Fetch raw gNMI client and call Set API to send Set Request. + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + delResp, err := gnmiClient.Set(ctx, delRequest) + if err != nil { + t.Fatalf("Unable to fetch set delete client (%v)", err) + } + t.Logf("SetResponse:\n%v", delResp) + + // Verify that other leaf nodes are not changed. + if got := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Mtu().Config()); got != mtu { + t.Errorf("MTU matched failed! got:%v, want:%v", got, mtu) + } +} + +func TestGNMISetDeleteMultipleLeafs(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("66addddd-df5d-4ff0-91ca-ff2a3582be69").Teardown(t) + dut := ondatra.DUT(t, "DUT") + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + res := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + ctx := context.Background() + + descPath := gnmi.OC().Interface(intf).Description() + resolvedDescPath, _, errs := ygnmi.ResolvePath(descPath.Config().PathStruct()) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", descPath, err) + } + var desc string + descExist := false + if res.Description != nil { + desc = *res.Description + descExist = true + } else { + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Description().Config(), "desc") + } + + defaultMtu := uint16(9100) + mtuPath := gnmi.OC().Interface(intf).Mtu() + resolvedMtuPath, _, errs := ygnmi.ResolvePath(mtuPath.Config().PathStruct()) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", mtuPath, err) + } + + if res.Mtu == nil { + t.Fatalf("MTU should not be nil!") + } + + mtu := uint16(2123) + nonDefaultMtu := false // Default value of MTU is 9100 + if *res.Mtu != defaultMtu { + mtu = *res.Mtu + nonDefaultMtu = true + } else { + gnmi.Update(t, dut, gnmi.OC().Interface(intf).Mtu().Config(), mtu) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).Mtu().State(), 20*time.Second, mtu) + } + + defer func() { + // Replace the fields to original values if they existed before the test. + if descExist { + gnmi.Replace(t, dut, descPath.Config(), desc) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).Description().State(), 5*time.Second, desc) + } + if nonDefaultMtu { + gnmi.Replace(t, dut, mtuPath.Config(), mtu) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).Mtu().State(), 5*time.Second, mtu) + } + }() + + paths := []*gpb.Path{resolvedDescPath, resolvedMtuPath} + delRequest := &gpb.SetRequest{ + Prefix: prefix, + Delete: paths, + } + // Fetch raw gNMI client and call Set API to send Set Request. + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + delResp, err := gnmiClient.Set(ctx, delRequest) + if err != nil { + t.Fatalf("Error while calling Set Delete Raw API(%v)", err) + } + t.Logf("SetResponse:\n%v", delResp) + + // Verify desc leaf is deleted. + testt.ExpectFatal(t, func(t testing.TB) { + gnmi.Get(t, dut, descPath.Config()) + }) + // Verify MTU leaf is deleted (set to default value) + if gotMtu := gnmi.Get(t, dut, mtuPath.Config()); gotMtu != defaultMtu { + t.Fatalf("Default MTU matched failed! got:%v, want:%v", gotMtu, defaultMtu) + } +} + +func TestGNMISetDeleteInvalidLeaf(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("bc463964-5e63-417f-bd32-08f17faf84a3").Teardown(t) + dut := ondatra.DUT(t, "DUT") + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + path := &gpb.Path{Elem: []*gpb.PathElem{{Name: "interfaces"}, {Name: "interface", Key: map[string]string{"name": intf}}, {Name: "config"}, {Name: "xyz"}}} + ctx := context.Background() + mtu := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Mtu().Config()) + + paths := []*gpb.Path{path} + + delRequest := &gpb.SetRequest{ + Prefix: prefix, + Delete: paths, + } + // Fetch raw gNMI client and call Set API to send Set Request. + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + delResp, err := gnmiClient.Set(ctx, delRequest) + if err == nil { + t.Fatalf("Set request is expected to fail but it didn't") + } + t.Logf("SetResponse:\n%v", delResp) + + // Verify that other leaf nodes are not changed. + if got := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Mtu().Config()); got != mtu { + t.Fatalf("MTU matched failed! got:%v, want:%v", got, mtu) + } +} + +/********************************************************** +* gNMI SET misc operations +**********************************************************/ + +/* Test to verify the order of SET operations in the same request. + * Verify that specified leaf has been deleted, and other specified leafs + * have been replaced, followed with updating of leaf values. + */ +func TestGNMISetDeleteReplaceUpdateOrderValid(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("fd8b8e2c-69dc-406c-99fd-6bdcf670e17d").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + // Get fields from interface subtree. + res := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + mtuPath := gnmi.OC().Interface(intf).Mtu() + resolvedMtuPath, _, errs := ygnmi.ResolvePath(mtuPath.Config().PathStruct()) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", mtuPath, err) + } + + mtuExist := false + mtu := uint16(9191) + if res.Mtu != nil { + mtu = *res.Mtu + mtuExist = true + } else { + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Mtu().Config(), mtu) + } + + descPath := gnmi.OC().Interface(intf).Description() + resolvedDescPath, _, errs := ygnmi.ResolvePath(descPath.Config().PathStruct()) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", descPath, err) + } + desc := "desc" + descExist := false + if res.Description != nil { + desc = *res.Description + descExist = true + } else { + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Description().Config(), desc) + } + + fqinPath := fmt.Sprintf("/interfaces/interface[name=%s]/config/fully-qualified-interface-name", intf) + resolvedFqinPath, errs := testhelper.StringToYgnmiPath(fqinPath) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", fqinPath, err) + } + fqin := "FQIN" + fqinExist := false + if name := testhelper.FullyQualifiedInterfaceName(t, dut, intf); name != "" { + fqin = name + fqinExist = true + } else { + testhelper.ReplaceFullyQualifiedInterfaceName(t, dut, intf, fqin) + } + wantFqin := "testFQIN" + wantMtu := mtu + 22 + + enabled := res.GetEnabled() + loopBack := res.GetLoopbackMode() + name := res.GetName() + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + defer func() { + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + + // Replace the old values for test cleanup. + if mtuExist { + gnmi.Replace(t, dut, mtuPath.Config(), mtu) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).Mtu().State(), 5*time.Second, mtu) + } else { + delRequest := &gpb.SetRequest{ + Prefix: prefix, + Delete: []*gpb.Path{resolvedMtuPath}, + } + // Fetch set client using the raw gNMI client. + if _, err := gnmiClient.Set(ctx, delRequest); err != nil { + t.Fatalf("Unable to fetch set delete client (%v)", err) + } + } + if descExist { + gnmi.Replace(t, dut, descPath.Config(), desc) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).Description().State(), 5*time.Second, desc) + } else { + delRequest := &gpb.SetRequest{ + Prefix: prefix, + Delete: []*gpb.Path{resolvedDescPath}, + } + // Fetch set client using the raw gNMI client. + if _, err := gnmiClient.Set(ctx, delRequest); err != nil { + t.Fatalf("Unable to fetch set delete client (%v)", err) + } + } + if fqinExist { + testhelper.ReplaceFullyQualifiedInterfaceName(t, dut, intf, fqin) + } else { + delRequest := &gpb.SetRequest{ + Prefix: prefix, + Delete: []*gpb.Path{resolvedFqinPath}, + } + // Fetch set client using the raw gNMI client. + if _, err := gnmiClient.Set(ctx, delRequest); err != nil { + t.Fatalf("Unable to fetch set delete client (%v)", err) + } + } + }() + + setRequest := &gpb.SetRequest{ + Prefix: prefix, + Delete: []*gpb.Path{resolvedMtuPath, resolvedDescPath}, + Replace: []*gpb.Update{ + { + Path: resolvedMtuPath, + Val: &gpb.TypedValue{ + Value: &gpb.TypedValue_JsonIetfVal{ + JsonIetfVal: []byte(strconv.FormatUint(uint64(wantMtu), 10)), + }, + }, + }, + { + Path: resolvedFqinPath, + Val: &gpb.TypedValue{ + Value: &gpb.TypedValue_JsonIetfVal{ + JsonIetfVal: []byte("\"tempFQIN\""), + }, + }, + }, + }, + Update: []*gpb.Update{{ + Path: resolvedFqinPath, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: []byte("\"" + wantFqin + "\"")}}, + }}, + } + + // Fetch raw gNMI client and call Set API to send Set Request. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + if _, err := gnmiClient.Set(ctx, setRequest); err != nil { + t.Fatalf("Error while calling Set Raw API: (%v)", err) + } + + // Verify that the Description leaf is deleted. + testt.ExpectFatal(t, func(t testing.TB) { + gnmi.Get(t, dut, descPath.Config()) + }) + // Get fields from interface subtree. + intfAfterSet := gnmi.Get(t, dut, gnmi.OC().Interface(intf).State()) + if got := testhelper.FullyQualifiedInterfaceName(t, dut, intf); got != wantFqin { + t.Errorf("FullyQualifiedInterfaceName match failed! got %v, want %v", got, wantFqin) + } + if got := intfAfterSet.GetMtu(); got != wantMtu { + t.Errorf("mtu match failed! got: %v, want: %v", got, wantMtu) + } + + // Verify that other leaf nodes are not changed. + if got := intfAfterSet.GetEnabled(); got != enabled { + t.Errorf("enabled match failed! got %v, want %v", got, enabled) + } + if got := intfAfterSet.GetLoopbackMode(); got != loopBack { + t.Errorf("loopback-mode match failed! got: %v, want: %v", got, loopBack) + } + if got := intfAfterSet.GetName(); got != name { + t.Errorf("name match failed! got: %v, want: %v", got, name) + } +} + +/* Test to verify the order of SET operations in the same request. + * Verify that the specified path has been deleted and other + * specified path attributes have been updated. */ +func TestGNMISetDeleteUpdateOrderValid(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("b7f99b79-f4fb-4c56-95be-602ad0361ec0").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + // Get fields from interface subtree. + res := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + descPath := gnmi.OC().Interface(intf).Description() + resolvedDescPath, _, errs := ygnmi.ResolvePath(descPath.Config().PathStruct()) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", descPath, err) + } + desc := "desc" + descExist := false + if res.Description != nil { + desc = *res.Description + descExist = true + } else { + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Description().Config(), desc) + } + wantDesc := "testDescription" + + enabled := res.GetEnabled() + mtu := res.GetMtu() + + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + defer func() { + // Replace the old values for test cleanup. + if descExist { + gnmi.Replace(t, dut, descPath.Config(), desc) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).Description().State(), 5*time.Second, desc) + } else { + delRequest := &gpb.SetRequest{ + Prefix: prefix, + Delete: []*gpb.Path{resolvedDescPath}, + } + // Fetch set client using the raw gNMI client. + if _, err := gnmiClient.Set(ctx, delRequest); err != nil { + t.Fatalf("Unable to fetch set delete client (%v)", err) + } + } + }() + + setRequest := &gpb.SetRequest{ + Prefix: prefix, + Delete: []*gpb.Path{resolvedDescPath}, + Update: []*gpb.Update{{ + Path: resolvedDescPath, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: []byte("\"" + wantDesc + "\"")}}, + }}, + } + + // Fetch raw gNMI client and call Set API to send Set Request. + if _, err := gnmiClient.Set(ctx, setRequest); err != nil { + t.Fatalf("Error while calling Set Raw API: (%v)", err) + } + + // Get fields from interface subtree. + intfAfterSet := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + if got := intfAfterSet.GetDescription(); got != wantDesc { + t.Errorf("Description match failed! got %v, want %v", got, wantDesc) + } + + // Verify that other leaf nodes are not changed. + if got := intfAfterSet.GetEnabled(); got != enabled { + t.Errorf("Enabled match failed! got %v, want %v", got, enabled) + } + if got := intfAfterSet.GetMtu(); got != mtu { + t.Errorf("MTU match failed! got: %v, want: %v", got, mtu) + } +} + +/* Test to verify the order of SET operations in the same request. + * Verify that with delete followed by update in same SET Request, + * an error message related to the invalid update is returned. */ +func TestGNMISetDeleteUpdateOrderInvalid(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("b43ed0ae-22c2-4c25-b50a-96c7243255d9").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + // Get fields from interface subtree. + res := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + descPath := gnmi.OC().Interface(intf).Description() + resolvedDescPath, _, errs := ygnmi.ResolvePath(descPath.Config().PathStruct()) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", descPath, err) + } + desc := "desc" + descExist := false + if res.Description != nil { + desc = *res.Description + descExist = true + } + + enabled := res.GetEnabled() + mtu := res.GetMtu() + + defer func() { + // Replace the old values for test cleanup. + if descExist { + gnmi.Replace(t, dut, descPath.Config(), desc) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).Description().State(), 5*time.Second, desc) + } + }() + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + setRequest := &gpb.SetRequest{ + Prefix: prefix, + Delete: []*gpb.Path{resolvedDescPath}, + Update: []*gpb.Update{{ + Path: resolvedDescPath, + Val: &gpb.TypedValue{Value: &gpb.TypedValue_JsonIetfVal{JsonIetfVal: []byte("123")}}, + }}, + } + + // Verify that error message is returned for invalid update request. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + if _, err := gnmiClient.Set(ctx, setRequest); err == nil { + t.Fatalf("Error expected while calling Set Raw API") + } + + // Get fields from interface subtree. + intfAfterSet := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + + // Verify that other leaf nodes are not changed. + if got := intfAfterSet.GetEnabled(); got != enabled { + t.Errorf("Enabled match failed! got %v, want %v", got, enabled) + } + if got := intfAfterSet.GetMtu(); got != mtu { + t.Errorf("MTU match failed! got: %v, want: %v", got, mtu) + } +} + +/* Test that performs gNMI SET for an empty path. Verify that there are no errors + * returned by the server and a valid response has been sent to the client, + * also verifies that none of the paths are changed. */ +func TestGNMISetEmptyPath(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("0b2e92b0-1295-4aa7-a27d-99073c09e2b7").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + // Get fields from interface subtree. + intfBeforeSet := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + + // Create setRequest message with an empty path. + setRequest := &gpb.SetRequest{ + Prefix: prefix, + } + + // Fetch set client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + if _, err := gnmiClient.Set(ctx, setRequest); err != nil { + t.Fatalf("Error while calling Set API with empty update: (%v)", err) + } + + // Verify that the leaf values did not change. + intfAfterSet := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + cmpOptions := cmp.Options{cmpopts.IgnoreFields(oc.Interface_Ethernet{}, "Counters"), + cmpopts.IgnoreFields(oc.Interface{}, "Counters")} + if diff := cmp.Diff(intfBeforeSet, intfAfterSet, cmpOptions); diff != "" { + t.Fatalf("diff (-want +got): %v", diff) + } +} + +/* Test that performs gNMI SET with two gNMI clients. */ +func TestGNMIMultipleClientSet(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("753e4dfc-5dda-4bfe-8b1c-e377e6c458ad").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + mtuPath := gnmi.OC().Interface(intf).Mtu() + resolvedMtuPath, _, errs := ygnmi.ResolvePath(mtuPath.Config().PathStruct()) + if errs != nil { + t.Fatalf("Failed to resolve path %v: %v", mtuPath, err) + } + mtu := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Mtu().Config()) + defer func() { + // Replace the old value for the MTU field as a test cleanup. + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Mtu().Config(), mtu) + gnmi.Await(t, dut, gnmi.OC().Interface(intf).Mtu().State(), 5*time.Second, mtu) + }() + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + + setRequest1 := &gpb.SetRequest{ + Prefix: prefix, + Replace: []*gpb.Update{ + { + Path: resolvedMtuPath, + Val: &gpb.TypedValue{ + Value: &gpb.TypedValue_JsonIetfVal{ + JsonIetfVal: []byte(strconv.FormatUint(uint64(9100), 10)), + }, + }, + }, + }, + } + setRequest2 := &gpb.SetRequest{ + Prefix: prefix, + Replace: []*gpb.Update{ + { + Path: resolvedMtuPath, + Val: &gpb.TypedValue{ + Value: &gpb.TypedValue_JsonIetfVal{ + JsonIetfVal: []byte(strconv.FormatUint(uint64(9122), 10)), + }, + }, + }, + }, + } + + ctx := context.Background() + newGNMIClient := func() gpb.GNMIClient { + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + return gnmiClient + } + clients := map[string]gpb.GNMIClient{ + "c1": newGNMIClient(), + "c2": newGNMIClient(), + } + + eg, ctx := errgroup.WithContext(context.Background()) + eg.Go(func() error { + _, err := clients["c1"].Set(ctx, setRequest1) + return err + }) + eg.Go(func() error { + _, err := clients["c2"].Set(ctx, setRequest2) + return err + }) + if err := eg.Wait(); err != nil { + t.Fatalf("Error while calling Multiple Set API %v", err) + } +} + +/********************************************************** +* gNMI GET operations +**********************************************************/ +// Sample test that performs gNMI GET using subscribe once on state paths. +func TestGNMIGetPaths(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("75e16f69-06ce-4e9a-bdbb-af50339ca8c4").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // The Get() APIs in this test would panic if the switch does not return + // state value for the Openconfig path, resulting in a test failure. + // The test validates that the switch returns state values for the + // specified interface Openconfig path. + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + // Fetch port MTU. + mtu := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Mtu().State()) + t.Logf("MTU is %v", mtu) + + // Fetch /interfaces/interface[name=]/state subtree. + p := gnmi.Get(t, dut, gnmi.OC().Interface(intf).State()) + // All paths might not be present in the response. Therefore, validate members + // of GoStruct before accessing them. + if p.AdminStatus != oc.Interface_AdminStatus_UNSET { + t.Logf("admin-status: %v", p.AdminStatus) + } + if p.Enabled != nil { + t.Logf("enabled: %v", *p.Enabled) + } + if p.Mtu != nil { + t.Logf("mtu: %v", *p.Mtu) + } + if p.Id != nil { + t.Logf("ID: %v", *p.Id) + } + if p.HoldTime != nil { + h := *p.HoldTime + if h.Down != nil { + t.Logf("hold-time down: %v", *h.Down) + } + if h.Up != nil { + t.Logf("hold-time up: %v", *h.Up) + } + } + + // Fetch /interfaces/interface[name=]/config subtree. + res := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + // All paths might not be present in the response. Therefore, validate members + // of GoStruct before accessing them. + if res.AdminStatus != oc.Interface_AdminStatus_UNSET { + t.Logf("admin-status: %v", res.AdminStatus) + } + if res.Cpu != nil { + t.Logf("IsCpu: %v", *res.Cpu) + } + if res.Enabled != nil { + t.Logf("enabled: %v", *res.Enabled) + } + if res.Description != nil { + t.Logf("description: %v", *res.Description) + } +} + +// Test that performs gNMI GET at module level. +func TestGNMIGetModulePaths(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("66ae2d27-d50e-4012-8be7-5536eb43fae8").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + // Create getRequest message to fetch all components. + getRequest := &gpb.GetRequest{ + Prefix: prefix, + Path: []*gpb.Path{{ + Elem: []*gpb.PathElem{{ + Name: "components", + }, { + Name: "component", + }}, + }}, + Encoding: gpb.Encoding_PROTO, + } + + // Fetch raw gNMI client and call Get API to send Get Request. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + getResp, err := gnmiClient.Get(ctx, getRequest) + if err != nil { + t.Fatalf("Unable to fetch get client (%v)", err) + } + if getResp == nil { + t.Fatal("Get response is nil") + } + + notifs := getResp.GetNotification() + if len(notifs) != 1 { + t.Fatalf("got %d notifications, want 1", len(notifs)) + } + updates := notifs[0].GetUpdate() + if len(updates) == 0 { + t.Fatalf("got %d updates in the notification, want >=1", len(updates)) + } + for i := range updates { + update := updates[i].GetPath() + // Go through all the paths to make sure they are working fine. + if _, err := ygot.PathToString(update); err != nil { + t.Fatalf("Failed to convert path to string (%v) %v", err, prototext.Format(update)) + } + } +} + +// Test that performs gNMI GET at root level. +func TestGNMIGetRootPath(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("dcc2805e-8dda-4899-99ad-3f5a42f1985b").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + // Create getRequest message to fetch root. + getRequest := &gpb.GetRequest{ + Prefix: prefix, + Encoding: gpb.Encoding_PROTO, + } + + // Fetch raw gNMI client and call Get API to send Get Request. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + getResp, err := gnmiClient.Get(ctx, getRequest) + if err != nil { + t.Fatalf("Unable to fetch get client (%v)", err) + } + if getResp == nil { + t.Fatal("Get response is nil") + } + + notifs := getResp.GetNotification() + if len(notifs) < 6 { + t.Fatalf("got %d notifications, want >= 6", len(notifs)) + } + for updates := range notifs { + updates := notifs[updates].GetUpdate() + if len(updates) < 1 { + continue + } + for i := range updates { + update := updates[i].GetPath() + // Go through all the paths to make sure they are working fine. + if _, err := ygot.PathToString(update); err != nil { + t.Fatalf("Failed to convert path to string (%v) %v", err, prototext.Format(update)) + } + } + } +} + +func TestGnmiProtoEncodingGet(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("0c25a72c-9b1a-4f80-90eb-177924f02802").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + // Create getRequest message with ASCII encoding. + getRequest := &gpb.GetRequest{ + Prefix: prefix, + Path: []*gpb.Path{{ + Elem: []*gpb.PathElem{{ + Name: "interfaces", + }, { + Name: "interface", + Key: map[string]string{"name": intf}, + }}, + }}, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_PROTO, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + getResp, err := gnmiClient.Get(ctx, getRequest) + if err != nil { + t.Fatalf("Unable to fetch get client (%v)", err) + } + if getResp == nil { + t.Fatalf("Unable to fetch get client, get response is nil") + } + + notifs := getResp.GetNotification() + if len(notifs) != 1 { + t.Fatalf("got %d notifications, want 1", len(notifs)) + } + updates := notifs[0].GetUpdate() + if len(updates) < 1 { + t.Fatalf("got %d updates in the notification, want >1", len(updates)) + } + for i := range updates { + update := updates[i].GetPath() + // Go through all the paths to make sure they are working fine. + pathStr, err := ygot.PathToString(update) + t.Logf("pathStr: %v", pathStr) + if err != nil { + t.Fatalf("Failed to convert path to string (%v) %v", err, update) + } + } + +} + +func TestGnmiInvalidKeyGet(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("ca3d9356-ca81-482b-aa36-8be5ac9180ba").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + + // Create getRequest message with ASCII encoding. + getRequest := &gpb.GetRequest{ + Prefix: prefix, + Path: []*gpb.Path{{ + Elem: []*gpb.PathElem{{ + Name: "interfaces", + }, { + Name: "interface", + Key: map[string]string{"name": "EthernetY"}, + }}, + }}, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_PROTO, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + if _, err := gnmiClient.Get(ctx, getRequest); err == nil { + t.Fatalf("Set request is expected to fail but it didn't") + } +} + +func TestGnmiInvalidPathGet(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("8f834ec5-da76-4884-9d84-58fc687c4f8c").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + + // Create getRequest message with ASCII encoding. + getRequest := &gpb.GetRequest{ + Prefix: prefix, + Path: []*gpb.Path{{ + Elem: []*gpb.PathElem{{ + Name: "xyz", + }}, + }}, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_PROTO, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + if _, err := gnmiClient.Get(ctx, getRequest); err == nil { + t.Fatalf("Set request is expected to fail but it didn't") + } +} + +func TestGnmiAsciiEncodingGet(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("104a2dbf-fb8d-40af-a592-99e2bfd868d2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Add Prefix information for the GetRequest. + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + + // Select a random front panel interface EthernetX. + intf, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + + // Create getRequest message with ASCII encoding. + getRequest := &gpb.GetRequest{ + Prefix: prefix, + Path: []*gpb.Path{{ + Elem: []*gpb.PathElem{{ + Name: "interfaces", + }, { + Name: "interface", + Key: map[string]string{"name": intf}, + }}, + }}, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_ASCII, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + if _, err = gnmiClient.Get(ctx, getRequest); err == nil { + t.Fatalf("Set request is expected to fail but it didn't") + } +} + +// Test that performs gNMI SET Replace at root level. +// This test will fail till the binding issue for root path is fixed (b/200096572) +func TestGNMISetReplaceRootPath(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("09df0cd9-3e23-4f8c-8a0b-9105de3a83af").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + if err := testhelper.ConfigPush(t, dut, nil); err != nil { + t.Fatalf("Failed to push config: %v", err) + } + + // Select a random front panel interface EthernetX. + var intf string + + info, err := testhelper.FetchPortsOperStatus(t, dut) + if err != nil { + t.Fatalf("Failed to fetch port operation status: %v", err) + } + + if len(info.Up) == 0 { + t.Fatalf("Failed to fetch port with operation status UP: %v", err) + } + + for _, port := range info.Up { + if isParent, err := testhelper.IsParentPort(t, dut, port); err == nil && isParent { + intf = port + break + } + } + + // Get fields from interface subtree. + if intf != "" { + intfIDAfterSet := gnmi.Get(t, dut, gnmi.OC().Interface(intf).State()).GetId() + t.Logf("ID After Set is %v for interface %v", intfIDAfterSet, intf) + } else { + t.Fatalf("Failed to fetch valid parent interface.") + } +} diff --git a/sdn_tests/pins_ondatra/tests/gnmi_stress_test.go b/sdn_tests/pins_ondatra/tests/gnmi_stress_test.go new file mode 100644 index 0000000000..1c5b2ed535 --- /dev/null +++ b/sdn_tests/pins_ondatra/tests/gnmi_stress_test.go @@ -0,0 +1,618 @@ +package gnmi_stress_test + +import ( + "context" + "fmt" + "math/rand" + "sync" + "testing" + "time" + + "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/binding/pinsbind" + "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/testhelper/testhelper" + gst "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/tests/gnmi_stress_helper" + + gpb "github.com/openconfig/gnmi/proto/gnmi" + "github.com/openconfig/ondatra" + "github.com/openconfig/ondatra/gnmi" + "github.com/openconfig/ygot/ygot" + "google.golang.org/grpc" +) + +func TestMain(m *testing.M) { + ondatra.RunTests(m, pinsbind.New) +} + +// gNMI load test - Replacing a single leaf 100 times. +func TestGNMILoadTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("f5e40be6-9913-4926-8d69-505e51f566f1").Teardown(t) + dut := ondatra.DUT(t, "DUT") + port, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + oldMtu := gnmi.Get(t, dut, gnmi.OC().Interface(port).Mtu().Config()) + gst.SanityCheck(t, dut, port) + + for i := gst.MinMtuStepInc; i < gst.MaxMtuStepInc; i++ { + // Configure port MTU and verify that state path reflects configured MTU. + mtu := uint16(1500 + i) + gnmi.Replace(t, dut, gnmi.OC().Interface(port).Mtu().Config(), mtu) + gst.CollectPerformanceMetrics(t, dut) + got := gnmi.Get(t, dut, gnmi.OC().Interface(port).Mtu().Config()) + if got != mtu { + t.Errorf("MTU matched failed! got:%v, want:%v", got, mtu) + } + } + t.Logf("After 10 seconds of idle time, the performance metrics are:") + time.Sleep(gst.IdleTime * time.Second) + gst.CollectPerformanceMetrics(t, dut) + // Replace the old MTU value as a test cleanup. + gnmi.Replace(t, dut, gnmi.OC().Interface(port).Mtu().Config(), oldMtu) + gst.SanityCheck(t, dut, port) + +} + +// gNMI load test short interval(30 minutes). +func TestGNMIShortStressTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("44fa854f-5d85-42aa-9ad0-4ee8dbce7f10").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.StressTestHelper(t, dut, gst.ShortStressTestInterval) + gst.SanityCheck(t, dut) +} + +// gNMI broken client test +func TestGNMIBrokenClientTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("cd36ba68-a2c1-485a-bc1a-c79463ed80d9").Teardown(t) + dut := ondatra.DUT(t, "DUT") + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + gst.SanityCheck(t, dut) + for i := 0; i < gst.MinIteration; i++ { + // Create getRequest message with ASCII encoding. + getRequest := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: []*gpb.Path{{ + Elem: []*gpb.PathElem{{ + Name: "interfaces", + }}, + }}, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_PROTO, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(context.Background(), grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + getResp, err := gnmiClient.Get(ctx, getRequest) + if err == nil { + t.Logf("GetResponse:\n%v", getResp) + t.Fatalf("The getRequest is successfully received on broken client") + } + if getResp != nil { + t.Fatalf("getResponse is received successfully") + } + } + gst.SanityCheck(t, dut) +} + +// gNMI different leaf get test +func TestGNMIGetDifferentLeafTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("08f9ffba-54a9-4d47-a3dc-0e4420fe296b").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.SanityCheck(t, dut) + rand.Seed(time.Now().Unix()) + gst.CollectPerformanceMetrics(t, dut) + for i := 0; i < gst.AvgIteration; i++ { + port, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + reqPath := fmt.Sprintf(gst.Path[rand.Intn(len(gst.Path))], port) + // Create Get Request. + sPath, err := ygot.StringToStructuredPath(reqPath) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + paths := []*gpb.Path{sPath} + + // Create getRequest message with data type. + getRequest := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: paths, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_PROTO, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + getResp, err := gnmiClient.Get(ctx, getRequest) + if err != nil { + t.Fatalf("Error while calling Get Raw API: (%v)", err) + } + + if getResp == nil { + t.Fatalf("Get response is nil") + } + t.Logf("GetResponse:\n%v", getResp) + gst.CollectPerformanceMetrics(t, dut) + } + t.Logf("After 10 seconds of idle time, the performance metrics are:") + time.Sleep(gst.IdleTime * time.Second) + gst.CollectPerformanceMetrics(t, dut) + gst.SanityCheck(t, dut) +} + +// gNMI different subtrees get test +func TestGNMIGetDifferentSubtreeTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("357762b4-4d34-467e-b321-90a2d271d50d").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.SanityCheck(t, dut) + rand.Seed(time.Now().Unix()) + gst.CollectPerformanceMetrics(t, dut) + for i := 0; i < gst.MinIteration; i++ { + reqPath := gst.Subtree[rand.Intn(len(gst.Subtree))] + // Create Get Request. + sPath, err := ygot.StringToStructuredPath(reqPath) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + // Create getRequest message with data type. + getRequest := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: []*gpb.Path{sPath}, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_PROTO, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + getResp, err := gnmiClient.Get(ctx, getRequest) + if err != nil { + t.Fatalf("Error while calling Get Raw API: (%v)", err) + } + + if getResp == nil { + t.Fatalf("Get response is nil") + } + t.Logf("GetResponse:\n%v", getResp) + gst.CollectPerformanceMetrics(t, dut) + } + t.Logf("After 10 seconds of idle time, the performance metrics are:") + time.Sleep(gst.IdleTime * time.Second) + gst.CollectPerformanceMetrics(t, dut) + gst.SanityCheck(t, dut) +} + +// gNMI different Client get test +func TestGNMIGetDifferentClientTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("389641b7-d995-4411-a222-e38caa9291a2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.SanityCheck(t, dut) + ctx := context.Background() + newGNMIClient := func() gpb.GNMIClient { + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + return gnmiClient + } + clients := map[string]gpb.GNMIClient{ + "c1": newGNMIClient(), + "c2": newGNMIClient(), + "c3": newGNMIClient(), + "c4": newGNMIClient(), + "c5": newGNMIClient(), + "c6": newGNMIClient(), + "c7": newGNMIClient(), + "c8": newGNMIClient(), + "c9": newGNMIClient(), + "c10": newGNMIClient(), + } + + var wg sync.WaitGroup + for k := range clients { + v := k + wg.Add(1) + + rand.Seed(time.Now().Unix()) + gst.CollectPerformanceMetrics(t, dut) + port, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + reqPath := fmt.Sprintf(gst.Path[rand.Intn(len(gst.Path))], port) + // Create Get Request. + sPath, err := ygot.StringToStructuredPath(reqPath) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + + // Create getRequest message with data type. + getRequest := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: []*gpb.Path{sPath}, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_PROTO, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + go func() { + getResp, err := clients[v].Get(ctx, getRequest) + if err != nil { + t.Log("Error while calling Get Raw API") + } + if getResp == nil { + t.Log("Get response is nil") + } + t.Logf("GetResponse:\n%v", getResp) + wg.Done() + }() + + gst.CollectPerformanceMetrics(t, dut) + } + wg.Wait() + t.Logf("After 10 seconds of idle time, the performance metrics are:") + time.Sleep(gst.IdleTime * time.Second) + gst.CollectPerformanceMetrics(t, dut) + gst.SanityCheck(t, dut) +} + +// gNMI different leaf get test +func TestGNMISetUpdateDifferentLeafTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("08f9ffba-54a9-4d47-a3dc-0e4420fe296b").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.StressSetTestHelper(t, dut, gst.AvgIteration, false) +} + +// gNMI different leaf set update test +func TestGNMISetReplaceDifferentLeafTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("08f9ffba-54a9-4d47-a3dc-0e4420fe296b").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.StressSetTestHelper(t, dut, gst.AvgIteration, true) +} + +// gNMI different leaf set replace test +func TestGNMISetUpdateDifferentClientTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("389641b7-d995-4411-a222-e38caa9291a2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.SetDifferentClientTest(t, dut, false) +} + +// gNMI different leaf set update test +func TestGNMISetReplaceDifferentClientTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("389641b7-d995-4411-a222-e38caa9291a2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.SetDifferentClientTest(t, dut, true) +} + +// gNMI different leaf set delete test +func TestGNMISetDeleteDifferentLeafTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("08f9ffba-54a9-4d47-a3dc-0e4420fe296b").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.SanityCheck(t, dut) + rand.Seed(time.Now().Unix()) + gst.CollectPerformanceMetrics(t, dut) + for i := 0; i < gst.AvgIteration; i++ { + port, err := testhelper.RandomInterface(t, dut, nil) + if err != nil { + t.Fatalf("Failed to fetch random interface: %v", err) + } + gst.SetDefaultValuesHelper(t, dut, port) + sPath, err := ygot.StringToStructuredPath(gst.RandomDeletePath(port)) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + + paths := []*gpb.Path{sPath} + + // Create getRequest message with data type. + getRequest := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: paths, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_JSON_IETF, + } + t.Logf("GetRequest:\n%v", getRequest) + + // Fetch get client using the raw gNMI client. + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + getResp, err := gnmiClient.Get(ctx, getRequest) + if err != nil { + t.Fatalf("Error while calling Get Raw API: (%v)", err) + } + t.Logf("GetResponse:\n%v", getResp) + + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Delete: []*gpb.Path{sPath}, + } + t.Logf("SetRequest:\n%v", setRequest) + + // Fetch get client using the raw gNMI client. + setResp, err := gnmiClient.Set(ctx, setRequest) + if err != nil { + t.Fatalf("Error while calling Get Raw API: (%v)", err) + } + + if setResp == nil { + t.Fatalf("set response is nil") + } + t.Logf("setResponse:\n%v", setResp) + gst.CollectPerformanceMetrics(t, dut) + + // Restore the old values for the path + + if getResp != nil { + updates, err := gst.UpdatesWithJSONIETF(getResp) + if err != nil { + t.Fatalf("Unable to get updates with JSON IETF: (%v)", err) + } + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Replace: updates, + } + setResp, err := gnmiClient.Set(ctx, setRequest) + if err != nil { + t.Fatalf("Unable to restore the original value using set client (%v)", err) + } + t.Logf("SetResponse:\n%v", setResp) + } + } + t.Logf("After %v seconds of idle time, collecting performance metrics", gst.IdleTime) + time.Sleep(gst.IdleTime * time.Second) + gst.CollectPerformanceMetrics(t, dut) + gst.SanityCheck(t, dut) +} + +// gNMI different subtrees set delete test +func TestGNMISetDeleteDifferentSubtreeTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("357762b4-4d34-467e-b321-90a2d271d50d").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.SanityCheck(t, dut) + rand.Seed(time.Now().Unix()) + gst.CollectPerformanceMetrics(t, dut) + // Restore the config on the DUT after the test. + defer func() { + if err := testhelper.ConfigPush(t, dut, nil); err != nil { + t.Fatalf("Failed to restore config: %v", err) + } + }() + for i := 0; i < gst.MinIteration; i++ { + reqPath := gst.DelSubtree[rand.Intn(len(gst.DelSubtree))] + + // Create Set Delete Request. + sPath, err := ygot.StringToStructuredPath(reqPath) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + + // Create getRequest message with data type. + getRequest := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: []*gpb.Path{sPath}, + Type: gpb.GetRequest_CONFIG, + Encoding: gpb.Encoding_JSON_IETF, + } + t.Logf("GetRequest:\n%v", getRequest) + + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + getResp, err := gnmiClient.Get(ctx, getRequest) + if err != nil { + t.Fatalf("Error while calling Get Raw API: (%v)", err) + } + t.Logf("GetResponse:\n%v", getResp) + + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Delete: []*gpb.Path{sPath}, + } + t.Logf("SetRequest:\n%v", setRequest) + + // Fetch set delete client using the raw gNMI client. + setResp, err := gnmiClient.Set(ctx, setRequest) + if err != nil { + t.Fatalf("Error while calling Get Raw API: (%v)", err) + } + if setResp == nil { + t.Fatalf("set response is nil") + } + t.Logf("setResponse:\n%v", setResp) + + // Restore the old values. + // A defer statement was not used to restore the values because this is in a for loop. The for + // loop deletes a subtree and then restores the values after. The subtree that is chosen could + // be chosen multiple times in the same test, so a defer would not work in this case. + updates, err := gst.UpdatesWithJSONIETF(getResp) + if err != nil { + t.Fatalf("Unable to get updates with JSON IETF: (%v)", err) + } + setRequest = &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Replace: updates, + } + setResp, err = gnmiClient.Set(ctx, setRequest) + if err != nil { + t.Fatalf("Unable to restore the original value using set client (%v)", err) + } + t.Logf("SetResponse:\n%v", setResp) + gst.CollectPerformanceMetrics(t, dut) + } + t.Logf("After %v seconds of idle time, the performance metrics are collected", gst.IdleTime) + time.Sleep(gst.IdleTime * time.Second) + gst.CollectPerformanceMetrics(t, dut) + gst.SanityCheck(t, dut) +} + +// gNMI different Client set delete test +func TestGNMISetDeleteDifferentClientTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("389641b7-d995-4411-a222-e38caa9291a2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.SanityCheck(t, dut) + ctx := context.Background() + newGNMIClient := func() gpb.GNMIClient { + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + return gnmiClient + } + clients := map[string]gpb.GNMIClient{ + "c1": newGNMIClient(), + "c2": newGNMIClient(), + "c3": newGNMIClient(), + "c4": newGNMIClient(), + "c5": newGNMIClient(), + "c6": newGNMIClient(), + "c7": newGNMIClient(), + "c8": newGNMIClient(), + "c9": newGNMIClient(), + "c10": newGNMIClient(), + } + info, err := testhelper.FetchPortsOperStatus(t, dut) + if err != nil || info == nil { + t.Fatalf("Failed to fetch ports oper-status: %v", err) + } + interfaces := info.Up + numIntfs := len(interfaces) + var port string + + var wg sync.WaitGroup + for k := range clients { + if len(interfaces) == 0 { + t.Logf("Less operationally up interfaces than clients: %v interfaces, %v clients", numIntfs, len(clients)) + break + } + v := k + wg.Add(1) + + rand.Seed(time.Now().Unix()) + gst.CollectPerformanceMetrics(t, dut) + port, interfaces = interfaces[0], interfaces[1:] + gst.SetDefaultValuesHelper(t, dut, port) + sPath, err := ygot.StringToStructuredPath(gst.RandomDeletePath(port)) + if err != nil { + t.Fatalf("Unable to convert string to path (%v)", err) + } + + paths := []*gpb.Path{sPath} + + // Create getRequest message with data type. + getRequest := &gpb.GetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Path: paths, + Type: gpb.GetRequest_ALL, + Encoding: gpb.Encoding_PROTO, + } + t.Logf("GetRequest:\n%v", getRequest) + + setRequest := &gpb.SetRequest{ + Prefix: &gpb.Path{Origin: "openconfig", Target: dut.Name()}, + Delete: []*gpb.Path{sPath}, + } + t.Logf("SetRequest:\n%v", setRequest) + ctx := context.Background() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + getResp, err := gnmiClient.Get(ctx, getRequest) + if err != nil { + t.Fatalf("Error while calling Get Raw API: (%v)", err) + } + t.Logf("GetResponse:\n%v", getResp) + + // Fetch get client using the raw gNMI client. + go func() { + // Fetch get client using the raw gNMI client. + setResp, err := clients[v].Set(context.Background(), setRequest) + if err != nil { + t.Log("Error while calling Set delete Raw API") + } + if setResp == nil { + t.Log("Set response is nil") + } + t.Logf("setResponse:\n%v", setResp) + wg.Done() + }() + gst.CollectPerformanceMetrics(t, dut) + // Restore the old values for the path + gst.SetDefaultValuesHelper(t, dut, port) + } + wg.Wait() + t.Logf("After %v seconds of idle time, colecting performance metrics", gst.IdleTime) + time.Sleep(gst.IdleTime * time.Second) + gst.CollectPerformanceMetrics(t, dut) + gst.SanityCheck(t, dut) +} + +// gNMI different leaf subscription poll mode test +func TestGNMISubscribePollDifferentLeafTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("08f9ffba-54a9-4d47-a3dc-0e4420fe296b").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.StressTestSubsHelper(t, dut, false, true) +} + +// gNMI different subtree subscription poll mode test +func TestGNMISubscribePollDifferentSubtreeTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("357762b4-4d34-467e-b321-90a2d271d50d").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.StressTestSubsHelper(t, dut, true, true) +} + +// gNMI different Client Subscribe Poll test +func TestGNMISubscribePollDifferentClientTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("389641b7-d995-4411-a222-e38caa9291a2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.SubscribeDifferentClientTest(t, dut, true) +} + +// gNMI different leaf subscription Sample mode test +func TestGNMISubscribeSampleDifferentLeafTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("08f9ffba-54a9-4d47-a3dc-0e4420fe296b").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.StressTestSubsHelper(t, dut, false, false) +} + +// gNMI different subtree subscription Sample mode test +func TestGNMISubscribeSampleDifferentSubtreeTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("357762b4-4d34-467e-b321-90a2d271d50d").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.StressTestSubsHelper(t, dut, true, false) +} + +// gNMI different Client Subscribe Sample test +func TestGNMISubscribeSampleDifferentClientTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("389641b7-d995-4411-a222-e38caa9291a2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.SubscribeDifferentClientTest(t, dut, false) +} + +// gNMI different Client random operations test +func TestGNMIRandomOpsDifferentClientTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("389641b7-d995-4411-a222-e38caa9291a2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + gst.RandomDifferentClientTestHelper(t, dut, gst.ShortStressTestInterval) +} diff --git a/sdn_tests/pins_ondatra/tests/gnmi_subscribe_modes_test.go b/sdn_tests/pins_ondatra/tests/gnmi_subscribe_modes_test.go index 4e7c9c54d0..cc267429b3 100644 --- a/sdn_tests/pins_ondatra/tests/gnmi_subscribe_modes_test.go +++ b/sdn_tests/pins_ondatra/tests/gnmi_subscribe_modes_test.go @@ -520,6 +520,20 @@ func (c subscribeTest) subModeUpdatesTest(t *testing.T) { } expectedPaths[syncResponse] = operStatus{} + foundPaths, _ := collectResponse(t, subscribeClient, expectedPaths) + if diff := cmp.Diff(expectedPaths, foundPaths, cmpopts.IgnoreUnexported(operStatus{})); diff != "" { + t.Errorf("collectResponse(expectedPaths):\n%v \nResponse mismatch (-missing +extra):\n%s", expectedPaths, diff) + } + if c.mode == gpb.SubscriptionList_ONCE { + return + } + + if c.mode == gpb.SubscriptionList_POLL { + subscribeClient.Send(&gpb.SubscribeRequest{Request: &gpb.SubscribeRequest_Poll{}}) + } + + expectedPaths = c.buildExpectedPaths(t, dut) + foundPaths, delay := collectResponse(t, subscribeClient, expectedPaths) if diff := cmp.Diff(expectedPaths, foundPaths, cmpopts.IgnoreUnexported(operStatus{})); diff != "" { t.Errorf("collectResponse(expectedPaths):\n%v \nResponse mismatch (-missing +extra):\n%s", expectedPaths, diff) @@ -677,3 +691,347 @@ func (c subscribeTest) subModeOnceTest(t *testing.T) { t.Errorf("collectResponse(expectedPaths):\n%v \nResponse mismatch (-missing +extra):\n%s", expectedPaths, diff) } } + +// Test for gNMI Subscribe Stream mode node deletions. +func (c subscribeTest) subModeDeleteTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID(c.uuid).Teardown(t) + if skipTest[t.Name()] { + t.Skip() + } + dut := ondatra.DUT(t, "DUT") + + subscribeRequest := buildRequest(t, c, dut.Name()) + t.Logf("SubscribeRequest:\n%v", prototext.Format(subscribeRequest)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + subscribeClient, err := gnmiClient.Subscribe(ctx) + if err != nil { + t.Fatalf("Unable to get subscribe client (%v)", err) + } + + if err := subscribeClient.Send(subscribeRequest); err != nil { + t.Fatalf("Failed to send gNMI subscribe request (%v)", err) + } + + expectedPaths := c.buildExpectedPaths(t, dut) + expectedPaths[syncResponse] = operStatus{} + + gotName := gnmi.Get(t, dut, gnmi.OC().System().Hostname().State()) + defer gnmi.Update(t, dut, gnmi.OC().System().Hostname().Config(), gotName) + + foundPaths, _ := collectResponse(t, subscribeClient, expectedPaths) + if diff := cmp.Diff(expectedPaths, foundPaths, cmpopts.IgnoreUnexported(operStatus{})); diff != "" { + t.Errorf("collectResponse(expectedPaths):\n%v \nResponse mismatch (-missing +extra):\n%s", expectedPaths, diff) + } + + if c.reqPath == deletePath { + gnmi.Delete(t, dut, gnmi.OC().System().Hostname().Config()) + } + if c.reqPath == deleteTreePath { + gnmi.Delete(t, dut, gnmi.OC().System().Config()) + } + delete(expectedPaths, syncResponse) + for _, v := range expectedPaths { + v.delete = true + } + + foundPaths, delay := collectResponse(t, subscribeClient, expectedPaths) + if diff := cmp.Diff(expectedPaths, foundPaths, cmpopts.IgnoreUnexported(operStatus{})); diff != "" { + t.Errorf("collectResponse(expectedPaths):\n%v \nResponse mismatch (-missing +extra):\n%s", expectedPaths, diff) + } + if delay > time.Duration(c.sampleInterval+(c.sampleInterval/2)) { + t.Errorf("Failed sampleInterval with time of %v", delay) + } +} + +// Test for gNMI Subscribe Poll mode for different levels. +func (c subscribeTest) subModePollTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID(c.uuid).Teardown(t) + if skipTest[t.Name()] { + t.Skip() + } + dut := ondatra.DUT(t, "DUT") + + subscribeRequest := buildRequest(t, c, dut.Name()) + t.Logf("SubscribeRequest:\n%v", prototext.Format(subscribeRequest)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + subscribeClient, err := gnmiClient.Subscribe(ctx) + if err != nil { + t.Fatalf("Unable to get subscribe client (%v)", err) + } + + if err := subscribeClient.Send(subscribeRequest); err != nil { + t.Fatalf("Failed to send gNMI subscribe request (%v)", err) + } + + expectedPaths := c.buildExpectedPaths(t, dut) + expectedPaths[syncResponse] = operStatus{} + + foundPaths, _ := collectResponse(t, subscribeClient, expectedPaths) + if diff := cmp.Diff(expectedPaths, foundPaths, cmpopts.IgnoreUnexported(operStatus{})); diff != "" { + t.Errorf("collectResponse(expectedPaths):\n%v \nResponse mismatch (-missing +extra):\n%s", expectedPaths, diff) + } + + delete(expectedPaths, syncResponse) + subscribeClient.Send(&gpb.SubscribeRequest{Request: &gpb.SubscribeRequest_Poll{}}) + foundPaths, _ = collectResponse(t, subscribeClient, expectedPaths) + if diff := cmp.Diff(expectedPaths, foundPaths, cmpopts.IgnoreUnexported(operStatus{})); diff != "" { + t.Errorf("collectResponse(expectedPaths):\n%v \nResponse mismatch (-missing +extra):\n%s", expectedPaths, diff) + } +} + +func (c subscribeTest) subModeRootTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID(c.uuid).Teardown(t) + if skipTest[t.Name()] { + t.Skip() + } + dut := ondatra.DUT(t, "DUT") + + req := buildRequest(t, c, dut.Name()) + t.Logf("SubscribeRequest:\n%v", prototext.Format(req)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + subscribeClient, err := gnmiClient.Subscribe(ctx) + if err != nil { + t.Fatalf("Unable to get subscribe client (%v)", err) + } + + if err := subscribeClient.Send(req); err != nil { + t.Fatalf("Failed to send gNMI subscribe request (%v)", err) + } + + // First listener returns after sync response + if err := clientListener(t, subscribeClient); err != nil { + t.Errorf("Initial Response failed (%v)", err) + } + + // Second listener returns after fixed time with no errors + if err := clientListener(t, subscribeClient); err != nil { + t.Errorf("Subscribe Response failed (%v)", err) + } + +} + +func collectResponse(t *testing.T, subClient gpb.GNMI_SubscribeClient, expectedPaths map[string]operStatus) (map[string]operStatus, time.Duration) { + t.Helper() + start := time.Now() + // Process response from DUT. + expectedCount := len(expectedPaths) + foundPaths := make(map[string]operStatus) + for pCount := 0; pCount < expectedCount; { + // Wait for response from DUT. + done := make(chan struct{}) + resCh := make(chan *gpb.SubscribeResponse, 1) + errCh := make(chan error, 1) + go func(subClient gpb.GNMI_SubscribeClient, resCh chan<- *gpb.SubscribeResponse, errCh chan<- error) { + res, err := subClient.Recv() + close(done) + resCh <- res + errCh <- err + }(subClient, resCh, errCh) + timer := time.NewTimer(mediumTime) + select { + case <-timer.C: + t.Fatalf("Timed out waiting on stream, expected: \n%+v, \nfound: \n%+v", expectedPaths, foundPaths) + case <-done: + if !timer.Stop() { + <-timer.C + } + } + res := <-resCh + err := <-errCh + if err != nil { + if _, ok := expectedPaths[errorResponse]; ok { + foundPaths[errorResponse] = operStatus{ + match: true, + value: err.Error(), + } + return foundPaths, 0 + } + t.Fatalf("Response error received from DUT (%v)", err) + } + switch v := res.Response.(type) { + case *gpb.SubscribeResponse_Update: + // Process Update message received in SubscribeResponse. + updates := v.Update + prefixStr, err := ygot.PathToString(updates.GetPrefix()) + if err != nil { + t.Fatalf("Failed to convert path to string (%v) %v", err, updates.GetPrefix()) + } + for _, d := range updates.GetDelete() { + elemStr, err := ygot.PathToString(d) + if err != nil { + t.Fatalf("Failed to convert path to string (%v) %v", err, d) + } + + pathStr := prefixStr + elemStr + if !ignorePaths(pathStr) { + _, ok := expectedPaths[syncResponse] + foundPaths[pathStr] = operStatus{match: ok, delete: true} + pCount++ + } + } + + // Perform basic sanity on the Update message. + for _, update := range updates.GetUpdate() { + if update.Path == nil { + t.Fatalf("Invalid nil Path in update: %v", prototext.Format(update)) + } + if update.Val == nil { + t.Fatalf("Invalid nil Val in update: %v", prototext.Format(update)) + } + // Path is partially present in Prefix and partially in Update in the response. + elemStr, err := ygot.PathToString(update.Path) + if err != nil { + t.Fatalf("Failed to convert path to string (%v) %v", err, update.Path) + } + pathStr := prefixStr + elemStr + + if !ignorePaths(pathStr) { + _, ok := expectedPaths[syncResponse] + foundPaths[pathStr] = operStatus{ + match: ok, + value: update.GetVal().GetStringVal(), + } + pCount++ + } + } + + case *gpb.SubscribeResponse_SyncResponse: + _, ok := expectedPaths[syncResponse] + foundPaths[syncResponse] = operStatus{match: ok} + pCount++ + } + } + return foundPaths, time.Since(start) +} + +func clientListener(t *testing.T, sc gpb.GNMI_SubscribeClient) error { + t.Helper() + timeout := time.After(mediumTime) + for { + select { + case <-timeout: + return nil + default: + m, err := sc.Recv() + if err != nil { + if errors.Is(err, context.Canceled) { + return nil + } + return err + } + switch m.Response.(type) { + case *gpb.SubscribeResponse_SyncResponse: + return nil + } + } + } +} + +func (c *subscribeTest) buildExpectedPaths(t *testing.T, dut *ondatra.DUTDevice) map[string]operStatus { + t.Helper() + expectedPaths := make(map[string]operStatus) + if c.expectError { + expectedPaths[errorResponse] = operStatus{} + return expectedPaths + } + prefix := &gpb.Path{Origin: "openconfig", Target: dut.Name()} + + resolvedPath, errs := ygot.StringToStructuredPath(c.reqPath) + if errs != nil { + t.Fatal(c.reqPath + " " + errs.Error()) + } + req := &gpb.GetRequest{ + Prefix: prefix, + Path: func(want string) []*gpb.Path { + if want == rootPath { + return nil + } + return []*gpb.Path{&gpb.Path{Elem: resolvedPath.Elem}} + }(c.reqPath), + Encoding: gpb.Encoding_PROTO, + } + + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + gnmiClient, err := dut.RawAPIs().BindingDUT().DialGNMI(ctx, grpc.WithBlock()) + if err != nil { + t.Fatalf("Unable to get gNMI client (%v)", err) + } + resp, err := gnmiClient.Get(ctx, req) + if err != nil { + t.Fatalf("GetResponse error received from DUT (%v)", err) + } + for _, notification := range resp.GetNotification() { + if notification == nil { + t.Fatalf("GetResponse contained no Notification (%v)", prototext.Format(resp)) + } + prefixStr, err := ygot.PathToString(notification.GetPrefix()) + if err != nil { + t.Fatalf("Failed to convert path to string (%v) %v", err, notification.GetPrefix()) + } + for _, update := range notification.GetUpdate() { + if update.Path == nil { + t.Fatalf("Invalid nil Path in update: %v", prototext.Format(update)) + } + elemStr, err := ygot.PathToString(update.Path) + if err != nil { + t.Fatalf("Failed to convert path to string (%v) %v", err, update.Path) + } + path := prefixStr + elemStr + + if !ignorePaths(path) { + expectedPaths[path] = operStatus{} + } + } + } + return expectedPaths +} + +// buildRequest creates a SubscribeRequest message using the specified +// parameters that include the list of paths to be added in the request. +func buildRequest(t *testing.T, params subscribeTest, target string) *gpb.SubscribeRequest { + t.Helper() + resolvedPath, errs := ygot.StringToStructuredPath(params.reqPath) + if errs != nil { + t.Fatal(params.reqPath + " " + errs.Error()) + } + + prefix := &gpb.Path{Origin: "openconfig", Target: target} + return &gpb.SubscribeRequest{ + Request: &gpb.SubscribeRequest_Subscribe{ + Subscribe: &gpb.SubscriptionList{ + Prefix: prefix, + Subscription: []*gpb.Subscription{ + &gpb.Subscription{ + Path: &gpb.Path{Elem: resolvedPath.Elem}, + Mode: params.subMode, + SampleInterval: params.sampleInterval, + SuppressRedundant: params.suppressRedundant, + HeartbeatInterval: params.heartbeatInterval, + }}, + Mode: params.mode, + Encoding: gpb.Encoding_PROTO, + UpdatesOnly: params.updatesOnly, + }, + }, + } +} diff --git a/sdn_tests/pins_ondatra/tests/gnmi_wildcard_subscription_test.go b/sdn_tests/pins_ondatra/tests/gnmi_wildcard_subscription_test.go new file mode 100644 index 0000000000..ef12d82336 --- /dev/null +++ b/sdn_tests/pins_ondatra/tests/gnmi_wildcard_subscription_test.go @@ -0,0 +1,1292 @@ +package gnmi_wildcard_subscription_test + +import ( + "context" + "math/rand" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/openconfig/ondatra" + "github.com/openconfig/ondatra/gnmi" + "github.com/openconfig/ondatra/gnmi/oc" + "github.com/openconfig/ygnmi/ygnmi" + "github.com/openconfig/ygot/ygot" + "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/binding/pinsbind" + "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/testhelper/testhelper" + "github.com/pkg/errors" + + gpb "github.com/openconfig/gnmi/proto/gnmi" +) + +const ( + intfPrefix = "Ethernet" + intfKey = "name" + componentKey = "name" + queueKey = "name" + intfIDKey = "interface-id" + + shortWait = 5 * time.Second + longWait = 20 * time.Second +) + +func TestMain(m *testing.M) { + ondatra.RunTests(m, pinsbind.New) +} + +// Test for gNMI Subscribe for Wildcard OnChange subscriptions. +func TestWCOnChangeAdminStatus(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("0cd4c21e-0af8-41d6-b637-07b6b90ba23d").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Collect every interface through a GET to be compared to the SUBSCRIBE. + wantUpdates := make(map[string]int) + var portList []string + ports := gnmi.GetAll(t, dut, gnmi.OC().InterfaceAny().Name().State()) + for _, port := range ports { + if strings.Contains(port, intfPrefix) { + wantUpdates[port]++ + portList = append(portList, port) + } + } + + intf, err := testhelper.RandomInterface(t, dut, &testhelper.RandomInterfaceParams{PortList: portList}) + if err != nil { + t.Fatal("No enabled interface found") + } + + state := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Enabled().State()) + predicate := oc.Interface_AdminStatus_UP + if state { + predicate = oc.Interface_AdminStatus_DOWN + } + + // Open a parallel client to watch changes to the oper-status. + // The async call needs to see the initial value be changed to + // its updated value. If this doesn't happen in a reasonable + // time (20 seconds) the test is failed. + finalValueCall := gnmi.WatchAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + InterfaceAny(). + AdminStatus().State(), longWait, func(val *ygnmi.Value[oc.E_Interface_AdminStatus]) bool { + port, err := fetchPathKey(val.Path, intfKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + } + status, present := val.Val() + return port == intf && present && status == predicate + }) + + // Collect interfaces through subscription to be compared to the previous GET. + initialValues := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + InterfaceAny(). + AdminStatus().State(), shortWait). + Await(t) + + gotUpdates := make(map[string]int) + for _, val := range initialValues { + port, err := fetchPathKey(val.Path, intfKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if val.IsPresent() && strings.Contains(port, intfPrefix) { + gotUpdates[port]++ + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got): %v", diff) + } + + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Enabled().Config(), !state) + defer gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Enabled().Config(), state) + + _, foundUpdate := finalValueCall.Await(t) + if !foundUpdate { + t.Errorf("Interface did not receive an update for %v enabled %v", intf, !state) + } +} + +func TestWCOnChangeOperStatus(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("d0a07207-b6a2-4045-8f16-243b8ad693b6").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Collect every interface through a GET to be compared to the SUBSCRIBE. + wantUpdates := make(map[string]bool) + ports := gnmi.GetAll(t, dut, gnmi.OC().InterfaceAny().Name().State()) + for _, port := range ports { + if strings.Contains(port, intfPrefix) { + wantUpdates[port] = true + } + } + + intf, err := testhelper.RandomInterface(t, dut, &testhelper.RandomInterfaceParams{}) + if err != nil { + t.Fatal("No enabled interface found") + } + + // Open a parallel client to watch changes to the oper-status. + // The async call needs to see the initial value be changed to + // its updated value. If this doesn't happen in a reasonable + // time (20 seconds) the test is failed. + finalValueCall := gnmi.WatchAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + InterfaceAny(). + OperStatus().State(), longWait, func(val *ygnmi.Value[oc.E_Interface_OperStatus]) bool { + port, err := fetchPathKey(val.Path, intfKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + } + state, present := val.Val() + return port == intf && present && state == oc.Interface_OperStatus_DOWN + }) + + // Collect Interfaces through Subscription to be compared to the previous GET. + initialValues := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + InterfaceAny(). + OperStatus().State(), shortWait). + Await(t) + + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Enabled().Config(), false) + defer gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Enabled().Config(), true) + + gotUpdates := make(map[string]bool) + for _, val := range initialValues { + port, err := fetchPathKey(val.Path, intfKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if val.IsPresent() && strings.Contains(port, intfPrefix) { + gotUpdates[port] = true + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got): %v", diff) + } + + _, foundUpdate := finalValueCall.Await(t) + if !foundUpdate { + t.Errorf("Interface did not receive an update to %s", intf) + } +} + +func TestWCOnChangePortSpeed(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("59669bd7-e2e2-4734-869a-4bf4110b4cdc").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Collect every interface through a GET to be compared to the SUBSCRIBE. + wantUpdates := make(map[string]int) + paths := gnmi.CollectAll(t, dut, gnmi.OC().InterfaceAny().Ethernet(). + PortSpeed().State(), shortWait). + Await(t) + for _, path := range paths { + port, isfp, err := extractFrontPanelPortName(path.Path) + if err != nil { + t.Errorf("extractFrontPanelPortName(%v) failed: %v", path.Path, err) + continue + } + if !isfp { + continue + } + if strings.Contains(port, intfPrefix) { + wantUpdates[port]++ + } + } + + var intf string + var speed oc.E_IfEthernet_ETHERNET_SPEED + newSpeed := oc.IfEthernet_ETHERNET_SPEED_UNSET + + // Find an interface with an alternate speed + for port := range wantUpdates { + if empty, err := testhelper.TransceiverEmpty(t, dut, port); err != nil || empty { + continue + } + speed = gnmi.Get(t, dut, gnmi.OC().Interface(port).Ethernet().PortSpeed().State()) + speeds, err := testhelper.SupportedSpeedsForPort(t, dut, port) + if err != nil { + t.Logf("SupportedSpeedsForPort(%v) failed: %v", port, err) + continue + } + if len(speeds) > 1 { + intf = port + for _, s := range speeds { + if s != speed { + newSpeed = s + break + } + } + break + } + } + if newSpeed == oc.IfEthernet_ETHERNET_SPEED_UNSET { + t.Fatal("No alternate speeds found") + } + + // Open a parallel client to watch changes to the oper-status. + // The async call needs to see the initial value be changed to + // its updated value. If this doesn't happen in a reasonable + // time (20 seconds) the test is failed. + finalValueCall := gnmi.WatchAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + InterfaceAny(). + Ethernet(). + PortSpeed().State(), longWait, func(val *ygnmi.Value[oc.E_IfEthernet_ETHERNET_SPEED]) bool { + port, err := fetchPathKey(val.Path, intfKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + } + if speed, present := val.Val(); port == intf && present && speed == newSpeed { + return true + } + return false + }) + + initialValues := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + InterfaceAny(). + Ethernet(). + PortSpeed().State(), shortWait). + Await(t) + + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Ethernet().PortSpeed().Config(), newSpeed) + defer gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Ethernet().PortSpeed().Config(), speed) + + gotUpdates := make(map[string]int) + for _, val := range initialValues { + port, err := fetchPathKey(val.Path, "name") + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if val.IsPresent() { + if _, ok := wantUpdates[port]; !ok { + t.Errorf("Port not found in On Change update: %v", port) + } + gotUpdates[port]++ + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got): %v", diff) + } + + _, foundUpdate := finalValueCall.Await(t) + if !foundUpdate { + t.Errorf("Interface did not receive an update for %v %v to %v", intf, speed, newSpeed) + } +} + +func fetchPathKey(path *gpb.Path, id string) (string, error) { + if path == nil { + return "", errors.New("received nil path") + } + pathStr, err := ygot.PathToString(path) + if err != nil { + return "", errors.Errorf("ygot.PathToString() failed: %v", err) + } + for _, e := range path.GetElem() { + if e.GetKey() == nil { + continue + } + if key, ok := e.GetKey()[id]; ok { + return key, nil + } + return "", errors.Errorf("failed to get key from path: %v", pathStr) + } + return "", errors.Errorf("failed to find key for path: %v", pathStr) +} + +func TestWCOnChangeId(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("f7e8ab6b-4d10-4986-811c-63044295a74d").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Collect every interface through a GET to be compared to the SUBSCRIBE. + wantUpdates := make(map[string]bool) + var portList []string + paths := gnmi.CollectAll(t, dut, gnmi.OC().InterfaceAny().Id().State(), 5*time.Second).Await(t) + for _, path := range paths { + port, isfp, err := extractFrontPanelPortName(path.Path) + if err != nil || !isfp { + continue + } + if strings.Contains(string(port), intfPrefix) { + wantUpdates[port] = true + portList = append(portList, port) + } + } + + // Randomly select one intf + intf, err := testhelper.RandomInterface(t, dut, &testhelper.RandomInterfaceParams{PortList: portList, OperDownOk: true}) + if err != nil { + t.Fatal("No interface found") + } + + res := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Config()) + idPath := gnmi.OC().Interface(intf).Id() + + var originalID uint32 = 0 + var modifiedID uint32 + var idExist bool = false + gotUpdates := make(map[string]uint32) + + if res.Id != nil { + // If the interface has already an ID set, save it so it can be restored. + originalID = *res.Id + idExist = true + } else { + rand.Seed(time.Now().Unix()) + originalID = uint32(rand.Intn(100)) + } + + modifiedID = uint32(originalID + 800) + + var wg sync.WaitGroup + wg.Add(1) + + go func(intf string) { + defer wg.Done() + // This goroutine runs ON_CHANGE subscription, wait change of id + // We are checking only on interface, so we just need the one got checked + value := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + InterfaceAny(). + Id().State(), 30*time.Second).Await(t) + + for _, v := range value { + // Extract front panel port. + fp, isfp, err := extractFrontPanelPortName(v.Path) + if err != nil { + t.Errorf("extractFrontPanelPortName() failed: %v", err) + continue + } + if !isfp { + continue + } + + if upd, present := v.Val(); present { + if intf == fp { + if wantUpdates[fp] { + gotUpdates[fp] = upd + } + } + } + } + }(intf) + + // Replace the originalID value with modifiedID if originalID exists before the test. + if idExist { + gnmi.Delete(t, dut, idPath.Config()) + gnmi.Update(t, dut, idPath.Config(), modifiedID) + } else { + gnmi.Update(t, dut, idPath.Config(), modifiedID) + } + time.Sleep(30 * time.Second) + wg.Wait() + + // When originalID does exist, sets the originalID back at the end of the test + defer func() { + if idExist { + gnmi.Delete(t, dut, idPath.Config()) + gnmi.Update(t, dut, idPath.Config(), originalID) + afterCall := gnmi.Watch(t, dut, gnmi.OC().Interface(intf).Id().State(), longWait, func(val *ygnmi.Value[uint32]) bool { + v, ok := val.Val() + return ok && v == originalID + }) + valueAfterCall, _ := afterCall.Await(t) + t.Logf("Modified ID got replaced back to originalID %v", valueAfterCall) + } else { + gnmi.Delete(t, dut, idPath.Config()) + } + }() + + if wantUpdates[intf] { + if modifiedID != gotUpdates[intf] { + t.Errorf("ID is updated for %v. Want to get %v, got %v ", intf, modifiedID, gotUpdates[intf]) + } + } +} + +// Returns the port name, whether its front-panel or not, and an error +func extractFrontPanelPortName(path *gpb.Path) (string, bool, error) { + if path == nil { + return "", false, errors.New("received nil path") + } + + pathStr, err := ygot.PathToString(path) + if err != nil { + return "", false, errors.Errorf("ygot.PathToString() failed: %v", err) + } + + if len(path.GetElem()) < 3 { + return "", false, errors.Errorf("No valid front panel name from path: %v", pathStr) + } + + fpEle := path.GetElem()[1] + if fpEle == nil { + return "", false, errors.Errorf("failed to get key from path: %v", pathStr) + } + + fpKey, ok := fpEle.GetKey()["name"] + if !ok { + return "", false, errors.Errorf("failed to get key from path: %v", pathStr) + } + + if !strings.Contains(fpKey, "Ethernet") { + return "", false, nil + } + + return fpKey, true, nil +} + +func TestWCOnChangeEthernetMacAddress(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("c83a6e46-95a4-4dc7-a934-48c92fa0f136").Teardown(t) + t.Skip() + dut := ondatra.DUT(t, "DUT") + + // Collect every interface through a GET to be compared to the SUBSCRIBE. + wantUpdates := make(map[string]int) + var portList []string + ports := gnmi.GetAll(t, dut, gnmi.OC().InterfaceAny().Name().State()) + for _, port := range ports { + if strings.Contains(port, intfPrefix) { + wantUpdates[port]++ + portList = append(portList, port) + } + } + + intf, err := testhelper.RandomInterface(t, dut, &testhelper.RandomInterfaceParams{PortList: portList, OperDownOk: true}) + if err != nil { + t.Fatal("No interface found") + } + + origMAC := gnmi.Get(t, dut, gnmi.OC().Interface(intf).Ethernet().MacAddress().State()) + newMAC := "00:11:22:33:44:55" + if origMAC == newMAC { + newMAC = "55:44:33:22:11:00" + } + + // Open a parallel client to watch changes to the `mac-address`. + // The async call needs to see the initial value be changed to + // its updated value. If this doesn't happen in a reasonable + // time (20 seconds) the test is failed. + finalValueCall := gnmi.WatchAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + InterfaceAny(). + Ethernet(). + MacAddress().State(), longWait, func(val *ygnmi.Value[string]) bool { + port, err := fetchPathKey(val.Path, intfKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + } + mac, present := val.Val() + return port == intf && present && mac == newMAC + }) + + // Collect interfaces through subscription to be compared to the previous GET. + initialValues := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + InterfaceAny(). + Ethernet(). + MacAddress().State(), shortWait). + Await(t) + + gotUpdates := make(map[string]int) + for _, val := range initialValues { + port, err := fetchPathKey(val.Path, intfKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if val.IsPresent() && strings.Contains(port, intfPrefix) { + gotUpdates[port]++ + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got): %v", diff) + } + + gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Ethernet().MacAddress().Config(), newMAC) + defer gnmi.Replace(t, dut, gnmi.OC().Interface(intf).Ethernet().MacAddress().Config(), origMAC) + + _, foundUpdate := finalValueCall.Await(t) + if !foundUpdate { + t.Errorf("Interface did not receive an update for %v `id` %s", intf, newMAC) + } +} + +func TestWCOnChangeIntegratedCircuitNodeId(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("7dd451b1-4d2b-4c79-90f5-1d419bdecc67").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Collect every integrated circuit component through a GET to be compared to the SUBSCRIBE. + wantUpdates := make(map[string]int) + var icList []string + components := gnmi.GetAll(t, dut, gnmi.OC().ComponentAny().Name().State()) + for _, component := range components { + if component == "" { + continue + } + compTypeVal, present := testhelper.LookupComponentTypeOCCompliant(t, dut, component) + if !present || compTypeVal != "INTEGRATED_CIRCUIT" { + continue + } + wantUpdates[component]++ + icList = append(icList, component) + } + if len(icList) == 0 { + t.Fatal("No integrated circuit components found") + } + + ic := icList[len(icList)-1] + + origNodeID := gnmi.Get(t, dut, gnmi.OC().Component(ic).IntegratedCircuit().NodeId().State()) + newNodeID := origNodeID + 1 + + // Open a parallel client to watch changes to the `node-id`. + // The async call needs to see the initial value be changed to + // its updated value. If this doesn't happen in a reasonable + // time (20 seconds) the test is failed. + finalValueCall := gnmi.WatchAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + ComponentAny(). + IntegratedCircuit(). + NodeId().State(), longWait, func(val *ygnmi.Value[uint64]) bool { + component, err := fetchPathKey(val.Path, componentKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + } + id, present := val.Val() + return component == ic && present && id == newNodeID + }) + + // Collect interfaces through subscription to be compared to the previous GET. + initialValues := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + ComponentAny(). + IntegratedCircuit(). + NodeId().State(), shortWait). + Await(t) + + gotUpdates := make(map[string]int) + for _, val := range initialValues { + component, err := fetchPathKey(val.Path, componentKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if val.IsPresent() { + gotUpdates[component]++ + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got): %v", diff) + } + + gnmi.Replace(t, dut, gnmi.OC().Component(ic).IntegratedCircuit().NodeId().Config(), newNodeID) + defer gnmi.Replace(t, dut, gnmi.OC().Component(ic).IntegratedCircuit().NodeId().Config(), origNodeID) + + _, foundUpdate := finalValueCall.Await(t) + if !foundUpdate { + t.Errorf("Interface did not receive an update for %v `id` %v", ic, newNodeID) + } +} + +func TestWCOnChangeComponentOperStatus(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("794672b0-e15f-4a72-8619-f5a0bbb45e9b").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Collect every component with an oper-status through a GET to be compared to the SUBSCRIBE. + wantUpdates := make(map[string]int) + vals := gnmi.LookupAll(t, dut, gnmi.OC().ComponentAny().OperStatus().State()) + for _, val := range vals { + if !val.IsPresent() { + continue + } + component, err := fetchPathKey(val.Path, componentKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + wantUpdates[component]++ + } + + // Collect components through Subscription to be compared to the previous GET. + initialValues := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + ComponentAny(). + OperStatus().State(), shortWait). + Await(t) + + gotUpdates := make(map[string]int) + for _, val := range initialValues { + component, err := fetchPathKey(val.Path, intfKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if val.IsPresent() { + gotUpdates[component]++ + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got):\n%v", diff) + } +} + +type counterSubscription struct { + dut *ondatra.DUTDevice + t *testing.T +} + +func (c counterSubscription) subToInUnicastPkts() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + InUnicastPkts().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToInBroadcastPkts() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + InBroadcastPkts().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToInMulticastPkts() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + InMulticastPkts().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToOutUnicastPkts() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + OutUnicastPkts().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToOutBroadcastPkts() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + OutBroadcastPkts().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToOutMulticastPkts() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + OutMulticastPkts().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToInOctets() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + InOctets().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToOutOctets() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + OutOctets().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToInDiscards() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + InDiscards().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToOutDiscards() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + OutDiscards().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToInErrors() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + InErrors().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToOutErrors() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + OutErrors().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToInFcsErrors() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + InterfaceAny(). + Counters(). + InFcsErrors().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToTransmitPkts() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + Qos(). + InterfaceAny(). + Output(). + QueueAny(). + TransmitPkts().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToTransmitOctets() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + Qos(). + InterfaceAny(). + Output(). + QueueAny(). + TransmitOctets().State(), longWait). + Await(c.t) +} + +func (c counterSubscription) subToDroppedPkts() []*ygnmi.Value[uint64] { + return gnmi.CollectAll(c.t, gnmiOpts(c.t, c.dut, gpb.SubscriptionMode_TARGET_DEFINED), gnmi.OC(). + Qos(). + InterfaceAny(). + Output(). + QueueAny(). + DroppedPkts().State(), longWait). + Await(c.t) +} + +type counterTest struct { + uuid string + subfun func() []*ygnmi.Value[uint64] +} + +func TestWcTargetDefinedCounters(t *testing.T) { + dut := ondatra.DUT(t, "DUT") + testCases := []struct { + name string + function func(*testing.T) + }{ + { + name: "InUnicastPkts", + function: counterTest{ + uuid: "488f9daa-9b8d-455f-b580-5dd5491d64b5", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToInUnicastPkts, + }.targetDefinedCounterTest, + }, + { + name: "InBroadcastPkts", + function: counterTest{ + uuid: "edffb8d9-9040-4188-b9d3-bdb083a61f27", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToInBroadcastPkts, + }.targetDefinedCounterTest, + }, + { + name: "InMulticastPkts", + function: counterTest{ + uuid: "1b20f216-1bb5-4ed6-b2ed-5a09942a2eee", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToInMulticastPkts, + }.targetDefinedCounterTest, + }, + { + name: "OutUnicastPkts", + function: counterTest{ + uuid: "712042e8-b057-4f0c-bd0b-cde2861a3555", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToOutUnicastPkts, + }.targetDefinedCounterTest, + }, + { + name: "OutBroadcastPkts", + function: counterTest{ + uuid: "06cf226d-36ad-4dec-958e-89a6c2d42506", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToOutBroadcastPkts, + }.targetDefinedCounterTest, + }, + { + name: "OutMulticastPkts", + function: counterTest{ + uuid: "4b81e22d-5dd2-4ea2-95bd-25d3655978a3", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToOutMulticastPkts, + }.targetDefinedCounterTest, + }, + { + name: "InOctets", + function: counterTest{ + uuid: "30e25f1b-f79c-4824-adac-4fa6feba2f02", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToInOctets, + }.targetDefinedCounterTest, + }, + { + name: "OutOctets", + function: counterTest{ + uuid: "9a55189e-c1a4-42c9-a02d-eec3d8ec5d1b", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToOutOctets, + }.targetDefinedCounterTest, + }, + { + name: "InDiscards", + function: counterTest{ + uuid: "744c4e37-e6d2-400f-999b-adb6a81b461d", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToInDiscards, + }.targetDefinedCounterTest, + }, + { + name: "OutDiscards", + function: counterTest{ + uuid: "69075f31-d825-4eb2-9bbe-e9bfb95b36a6", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToOutDiscards, + }.targetDefinedCounterTest, + }, + { + name: "InErrors", + function: counterTest{ + uuid: "b416b679-f336-4d3c-9e70-cc907508cda1", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToInErrors, + }.targetDefinedCounterTest, + }, + { + name: "OutErrors", + function: counterTest{ + uuid: "76b515f6-9464-42ee-aa31-2210c5c9fc29", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToOutErrors, + }.targetDefinedCounterTest, + }, + { + name: "InFcsErrors", + function: counterTest{ + uuid: "6e6aac39-eaa4-4d10-890b-aec13e981733", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToInFcsErrors, + }.targetDefinedCounterTest, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, testCase.function) + } +} + +func TestWcTargetDefinedQosCountersWc(t *testing.T) { + dut := ondatra.DUT(t, "DUT") + + testCases := []struct { + name string + function func(*testing.T) + }{ + { + name: "TransmitPkts", + function: counterTest{ + uuid: "7b1133b0-b934-4948-b087-d63108418dcb", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToTransmitPkts, + }.targetDefinedQosCounterWcTest, + }, + { + name: "TransmitOctets", + function: counterTest{ + uuid: "9c36c0ca-2551-4a7d-a7de-d47fef53d158", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToTransmitOctets, + }.targetDefinedQosCounterWcTest, + }, + { + name: "DroppedPkts", + function: counterTest{ + uuid: "88985776-2b1e-4afc-8e7b-dd1114f7070c", + subfun: counterSubscription{ + dut: dut, + t: t, + }.subToDroppedPkts, + }.targetDefinedQosCounterWcTest, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, testCase.function) + } +} + +func (c counterTest) targetDefinedCounterTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID(c.uuid).Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Collect every interface through a GET to be compared to the SUBSCRIBE. + wantUpdates := make(map[string]int) + ports := gnmi.GetAll(t, dut, gnmi.OC().InterfaceAny().Name().State()) + for _, port := range ports { + if strings.Contains(port, intfPrefix) { + wantUpdates[port] = 2 // initial subscription plus one timed update. + } + } + + // Collect interfaces through subscription to be compared to the previous GET. + subcriptionValues := c.subfun() + + gotUpdates := make(map[string]int) + for _, val := range subcriptionValues { + port, err := fetchPathKey(val.Path, intfKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if val.IsPresent() && strings.Contains(port, intfPrefix) { + gotUpdates[port]++ + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got):\n%v", diff) + } +} + +// UMF default queue name is interface-id:queueName +// This name is used for the queue name without proper number->string mapping +func isQueueConfigured(qname, iname string) bool { + return !strings.HasPrefix(qname, iname) +} + +func (c counterTest) targetDefinedQosCounterWcTest(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID(c.uuid).Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Collect every interface through a GET to be compared to the SUBSCRIBE. + wantUpdates := make(map[string]uint64) + qosInterfaces := gnmi.Get(t, dut, gnmi.OC().Qos().State()).Interface + for intf := range qosInterfaces { + queueNames := gnmi.GetAll(t, dut, gnmi.OC().Qos().Interface(intf).Output().QueueAny().Name().State()) + for _, queueName := range queueNames { + if !isQueueConfigured(queueName, intf) { + continue + } + wantUpdates[intf+","+queueName] = 2 // initial subscription plus one timed update. + } + } + + // Collect interfaces through subscription to be compared to the previous GET. + subcriptionValues := c.subfun() + + gotUpdates := make(map[string]uint64) + for _, val := range subcriptionValues { + intfQueue, err := fetchQosKey(val.Path) + if err != nil { + if !strings.Contains(err.Error(), "unconfigured") { + t.Logf("fetchQosKey() failed: %v", err) + } + continue + } + + interfaceID, queueName := intfQueue[0], intfQueue[1] + if val.IsPresent() { + gotUpdates[interfaceID+","+queueName]++ + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got):\n%v", diff) + } +} + +func fetchQosKey(path *gpb.Path) ([]string, error) { + if path == nil { + return nil, errors.New("received nil path") + } + pathStr, err := ygot.PathToString(path) + if err != nil { + return nil, errors.Errorf("ygot.PathToString() failed: %v", err) + } + + if len(path.GetElem()) != 8 { + return nil, errors.Errorf("no valid interface id or queue name from path: %v", pathStr) + } + + interfaceEle := path.GetElem()[2].GetKey() + if interfaceEle == nil { + return nil, errors.Errorf("no valid interface id from path: %v", pathStr) + } + interfaceKey, ok := interfaceEle[intfIDKey] + if !ok { + return nil, errors.Errorf("no valid interface id from path: %v", pathStr) + } + + queueEle := path.GetElem()[5].GetKey() + if queueEle == nil { + return nil, errors.Errorf("no valid queue name from path: %v", pathStr) + } + queueName, ok := queueEle[queueKey] + if !ok { + return nil, errors.Errorf("no valid queue name from path: %v", pathStr) + } + if !isQueueConfigured(queueName, interfaceKey) { + return nil, errors.Errorf("unconfigured queue found: %v", pathStr) + } + + return []string{interfaceKey, queueName}, nil +} + +func TestWCOnChangeSoftwareModuleModuleType(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("ff05e9c6-b57b-4128-9535-e8543dc5aedc").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Collect every software module component through a GET to be compared to the SUBSCRIBE. + wantUpdates := make(map[string]int) + components := gnmi.GetAll(t, dut, gnmi.OC().ComponentAny().Name().State()) + for _, component := range components { + if component == "" { + continue + } + compTypeVal, present := testhelper.LookupComponentTypeOCCompliant(t, dut, component) + if !present || compTypeVal != "SOFTWARE_MODULE" { + continue + } + wantUpdates[component]++ + } + if len(wantUpdates) == 0 { + t.Fatal("No software module components found") + } + + // Collect interfaces through subscription to be compared to the previous GET. + initialValues := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + ComponentAny(). + SoftwareModule(). + ModuleType().State(), shortWait). + Await(t) + + gotUpdates := make(map[string]int) + for _, val := range initialValues { + component, err := fetchPathKey(val.Path, componentKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if val.IsPresent() { + gotUpdates[component]++ + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got):\n%v", diff) + } +} + +func TestWCOnChangeHardwarePort(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("acfc84d1-b76f-45b3-bb8f-267abca3b2d2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Collect every interface through a GET to be compared to the SUBSCRIBE. + wantUpdates := make(map[string]int) + ports := gnmi.GetAll(t, dut, gnmi.OC().InterfaceAny().Name().State()) + for _, port := range ports { + if strings.Contains(port, intfPrefix) { + wantUpdates[port]++ + } + } + + // Collect interfaces through subscription to be compared to the previous GET. + initialValues := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + InterfaceAny(). + HardwarePort().State(), shortWait). + Await(t) + + gotUpdates := make(map[string]int) + for _, val := range initialValues { + port, err := fetchPathKey(val.Path, intfKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if val.IsPresent() && strings.Contains(port, intfPrefix) { + gotUpdates[port]++ + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got): %v", diff) + } +} + +func TestWCOnChangeComponentType(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("e07ea34c-b217-4aef-99ac-2516b0b5c393").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Determine that updates are received from all expected components. + wantUpdates := make(map[string]int) + components := gnmi.GetAll(t, dut, gnmi.OC().ComponentAny().State()) + for _, component := range components { + if component.GetName() == "" { + continue + } + + if _, present := testhelper.LookupComponentTypeOCCompliant(t, dut, component.GetName()); !present { + continue + } + + wantUpdates[component.GetName()]++ + } + + // Collect components updates from ON_CHANGE subscription. + initialValues := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + ComponentAny(). + Type().State(), shortWait). + Await(t) + + gotUpdates := make(map[string]int) + for _, val := range initialValues { + component, err := fetchPathKey(val.Path, componentKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if val.IsPresent() { + gotUpdates[component] = 1 + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got):\n%v", diff) + } +} + +func TestWCOnChangeComponentParent(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("9c889baf-c3c2-4ce3-bb74-36b78c5b77ca").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Determine expected components. + wantUpdates := make(map[string]string) + components := gnmi.GetAll(t, dut, gnmi.OC().ComponentAny().State()) + for _, component := range components { + if component == nil || component.Name == nil || component.GetName() == "" { + continue + } + if component.Parent != nil && component.GetParent() != "" { + wantUpdates[component.GetName()] = component.GetParent() + } + } + + // Collect component updates from ON_CHANGE subscription. + initialValues := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + ComponentAny(). + Parent().State(), shortWait). + Await(t) + + gotUpdates := make(map[string]string) + for _, val := range initialValues { + component, err := fetchPathKey(val.Path, componentKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if upd, present := val.Val(); present && upd != "" { + gotUpdates[component] = upd + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got):\n%v", diff) + } +} + +func TestWCOnChangeComponentSoftwareVersion(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("25e36fae-82e7-4d51-8f60-df6fb139f6ca").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Determine expected components. + wantUpdates := make(map[string]string) + components := gnmi.GetAll(t, dut, gnmi.OC().ComponentAny().State()) + for _, component := range components { + if component == nil || component.Name == nil || component.GetName() == "" { + continue + } + if component.SoftwareVersion != nil && component.GetSoftwareVersion() != "" { + wantUpdates[component.GetName()] = component.GetSoftwareVersion() + } + } + + // Collect component updates from ON_CHANGE subscription. + initialValues := gnmi.CollectAll(t, gnmiOpts(t, dut, gpb.SubscriptionMode_ON_CHANGE), gnmi.OC(). + ComponentAny(). + SoftwareVersion().State(), shortWait). + Await(t) + + gotUpdates := make(map[string]string) + for _, val := range initialValues { + component, err := fetchPathKey(val.Path, componentKey) + if err != nil { + t.Errorf("fetchPathKey() failed: %v", err) + continue + } + if upd, present := val.Val(); present && upd != "" { + gotUpdates[component] = upd + } + } + + if diff := cmp.Diff(wantUpdates, gotUpdates); diff != "" { + t.Errorf("Update notifications comparison failed! (-want +got):\n%v", diff) + } +} + +func gnmiOpts(t *testing.T, dut *ondatra.DUTDevice, mode gpb.SubscriptionMode) *gnmi.Opts { + client, err := dut.RawAPIs().BindingDUT().DialGNMI(context.Background()) + if err != nil { + t.Fatalf("DialGNMI() failed: %v", err) + } + return dut.GNMIOpts(). + WithClient(client). + WithYGNMIOpts(ygnmi.WithSubscriptionMode(mode)) +} diff --git a/sdn_tests/pins_ondatra/tests/inband_sw_interface_dual_switch_test.go b/sdn_tests/pins_ondatra/tests/inband_sw_interface_dual_switch_test.go index 29c720b8fa..c15c098e29 100644 --- a/sdn_tests/pins_ondatra/tests/inband_sw_interface_dual_switch_test.go +++ b/sdn_tests/pins_ondatra/tests/inband_sw_interface_dual_switch_test.go @@ -25,7 +25,7 @@ type counters struct { var ( inbandSwIntfName = "Loopback0" interfaceIndex = uint32(0) - configuredIPv4Path = "6.7.8.9" + configuredIPv4Path = "10.10.10.10" configuredIPv4PrefixLength = uint8(32) configuredIPv6Path = "3000::2" configuredIPv6PrefixLength = uint8(128) @@ -160,7 +160,7 @@ func TestGNMIInbandSwLoopbackInCnts(t *testing.T) { Version: 4, TTL: 64, Protocol: layers.IPProtocolTCP, - SrcIP: net.ParseIP("2.2.2.2").To4(), + SrcIP: net.ParseIP("10.10.20.30").To4(), DstIP: net.ParseIP(configuredIPv4Path).To4(), } tcp := &layers.TCP{ @@ -282,7 +282,7 @@ func TestGNMIInbandSwLoopbackOutCnts(t *testing.T) { TTL: 64, Protocol: layers.IPProtocolTCP, SrcIP: net.ParseIP(configuredIPv4Path).To4(), - DstIP: net.ParseIP("2.2.2.2").To4(), + DstIP: net.ParseIP("10.10.20.30").To4(), } tcp := &layers.TCP{ SrcPort: 10000, diff --git a/sdn_tests/pins_ondatra/tests/inband_sw_interface_test.go b/sdn_tests/pins_ondatra/tests/inband_sw_interface_test.go index a02dfadab3..edf21c1ea5 100644 --- a/sdn_tests/pins_ondatra/tests/inband_sw_interface_test.go +++ b/sdn_tests/pins_ondatra/tests/inband_sw_interface_test.go @@ -16,11 +16,11 @@ import ( var ( inbandSwIntfName = "Loopback0" interfaceIndex = uint32(0) - configuredIPv4Path = "6.7.8.9" + configuredIPv4Path = "10.10.40.20" configuredIPv4PrefixLength = uint8(32) configuredIPv6Path = "3000::2" configuredIPv6PrefixLength = uint8(128) - newConfiguredIPv4Path = "7.8.9.6" + newConfiguredIPv4Path = "10.10.50.15" newConfiguredIPv4PrefixLength = uint8(32) newConfiguredIPv6Path = "3022::2345" newConfiguredIPv6PrefixLength = uint8(128) @@ -210,7 +210,7 @@ func TestGNMIInbandSwIntfSetInvalidIPv4AddrOrPrefixLength(t *testing.T) { } // Set IPv4 address with invalid prefix length for the loopback0 interface. - var tryConfiguredIPv4Path = "70.80.90.60" + var tryConfiguredIPv4Path = "10.10.60.30" var badConfiguredIPv4PrefixLength = uint8(24) newV4 := iface.GetOrCreateIpv4().GetOrCreateAddress(tryConfiguredIPv4Path) newV4.Ip = &tryConfiguredIPv4Path diff --git a/sdn_tests/pins_ondatra/tests/lacp_test.go b/sdn_tests/pins_ondatra/tests/lacp_test.go index cd9076cd6a..82e96af62c 100644 --- a/sdn_tests/pins_ondatra/tests/lacp_test.go +++ b/sdn_tests/pins_ondatra/tests/lacp_test.go @@ -25,7 +25,7 @@ import ( const defaultGNMIWait = 15 * time.Second // IEEE 802.3ad defines the Link Aggregation standard used by LACP where connected ports can -// exchange control packets between each other. Based on these packets the switch can group matching +// experimental control packets between each other. Based on these packets the switch can group matching // ports into a LAG/Trunk/PortChannel. // // Local state is maintained for each member of a LAG to monitor the health of that given member. @@ -556,3 +556,128 @@ func TestLacpConfiguredOnOnlyOneSwitch(t *testing.T) { t.Errorf("LACP state is not blocking: %v", err) } } + +func TestMembersArePartiallyConfigured(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("42eea9f9-043c-45c4-95fd-5c1e00fef959").Teardown(t) + + host := ondatra.DUT(t, "DUT") + peer := ondatra.DUT(t, "CONTROL") + t.Logf("Host Device: %v", host.Name()) + t.Logf("Peer Device: %v", peer.Name()) + + // Find a set of peer ports between the 2 switches. + peerPorts, err := testhelper.PeerPortGroupWithNumMembers(t, host, peer, 2) + if err != nil { + t.Fatalf("Failed to get enough peer ports: %v", err) + } + t.Logf("Using peer ports: %v", peerPorts) + + // We bring up one PortChannel on both the host and peer device so we can reuse the same device + // configuration on both without issue. + portChannel := "PortChannel200" + portChannelConfig := testhelper.GeneratePortChannelInterface(portChannel) + portChannelConfigs := map[string]*oc.Interface{portChannel: &portChannelConfig} + + lacpConfig := testhelper.GenerateLACPInterface(portChannel) + var lacpConfigs oc.Lacp + lacpConfigs.AppendInterface(&lacpConfig) + + deviceConfig := &oc.Root{ + Interface: portChannelConfigs, + Lacp: &lacpConfigs, + } + + // Push the PortChannel configs and clean them up after the test finishes so they won't affect + // future tests + gnmi.Replace(t, host, gnmi.OC().Config(), deviceConfig) + defer func() { + if err := testhelper.RemovePortChannelFromDevice(t, defaultGNMIWait, host, portChannel); err != nil { + t.Fatalf("Failed to remove %v:%v: %v", host.Name(), portChannel, err) + } + }() + gnmi.Replace(t, peer, gnmi.OC().Config(), deviceConfig) + defer func() { + if err := testhelper.RemovePortChannelFromDevice(t, defaultGNMIWait, peer, portChannel); err != nil { + t.Fatalf("Failed to remove %v:%v: %v", peer.Name(), portChannel, err) + } + }() + + // On the host assign both ports to the PortChannel, but on the peer only assign 1. + testhelper.AssignPortsToAggregateID(t, host, portChannel, peerPorts[0].Host, peerPorts[1].Host) + testhelper.AssignPortsToAggregateID(t, peer, portChannel, peerPorts[0].Peer) + + // Ensure ports are enabled before trying to verify state. + gnmi.Await(t, host, gnmi.OC().Interface(peerPorts[0].Host).Enabled().State(), defaultGNMIWait, true) + gnmi.Await(t, host, gnmi.OC().Interface(peerPorts[1].Host).Enabled().State(), defaultGNMIWait, true) + gnmi.Await(t, peer, gnmi.OC().Interface(peerPorts[0].Peer).Enabled().State(), defaultGNMIWait, true) + + if err := verifyInSyncState(t, host, portChannel, peerPorts[0].Host); err != nil { + t.Errorf("LACP state is not in-sync: %v", err) + } + if err := verifyInSyncState(t, peer, portChannel, peerPorts[0].Peer); err != nil { + t.Errorf("LACP state is not in-sync: %v", err) + } + if err := verifyBlockingState(t, host, portChannel, peerPorts[1].Host); err != nil { + t.Errorf("LACP state is not blocking: %v", err) + } +} + +func TestPortDownEvent(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("8b585121-80c1-4ca1-9847-5433ded2ebe6").Teardown(t) + + host := ondatra.DUT(t, "DUT") + peer := ondatra.DUT(t, "CONTROL") + t.Logf("Host Device: %v", host.Name()) + t.Logf("Peer Device: %v", peer.Name()) + + // Find a set of peer ports between the 2 switches. + peerPorts, err := testhelper.PeerPortGroupWithNumMembers(t, host, peer, 2) + if err != nil { + t.Fatalf("Failed to get enough peer ports: %v", err) + } + t.Logf("Using peer ports: %v", peerPorts) + + // The same PortChannel settings will be used on the host and peer devices. + portChannel1 := "PortChannel200" + portChannelConfig := testhelper.GeneratePortChannelInterface(portChannel1) + portChannelConfigs := map[string]*oc.Interface{portChannel1: &portChannelConfig} + + var lacpConfigs oc.Lacp + lacpConfig := testhelper.GenerateLACPInterface(portChannel1) + lacpConfigs.AppendInterface(&lacpConfig) + + deviceConfig := &oc.Root{ + Interface: portChannelConfigs, + Lacp: &lacpConfigs, + } + gnmi.Replace(t, host, gnmi.OC().Config(), deviceConfig) + defer func() { + if err := testhelper.RemovePortChannelFromDevice(t, defaultGNMIWait, host, portChannel1); err != nil { + t.Fatalf("Failed to remove %v:%v: %v", host.Name(), portChannel1, err) + } + }() + gnmi.Replace(t, peer, gnmi.OC().Config(), deviceConfig) + defer func() { + if err := testhelper.RemovePortChannelFromDevice(t, defaultGNMIWait, peer, portChannel1); err != nil { + t.Fatalf("Failed to remove %v:%v: %v", peer.Name(), portChannel1, err) + } + }() + + // Assign the port to each PortChannel and wait for the links to become active. + testhelper.AssignPortsToAggregateID(t, host, portChannel1, peerPorts[0].Host, peerPorts[1].Host) + testhelper.AssignPortsToAggregateID(t, peer, portChannel1, peerPorts[0].Peer, peerPorts[1].Peer) + gnmi.Await(t, host, gnmi.OC().Interface(portChannel1).Enabled().State(), defaultGNMIWait, true) + gnmi.Await(t, peer, gnmi.OC().Interface(portChannel1).Enabled().State(), defaultGNMIWait, true) + + // Bring the port down on the peer side. + gnmi.Replace(t, peer, gnmi.OC().Interface(peerPorts[0].Peer).Enabled().Config(), false) + defer func() { + gnmi.Replace(t, peer, gnmi.OC().Interface(peerPorts[0].Peer).Enabled().Config(), true) + }() + + // Wait for the port to go down on the peer then verify the host side is in a blocking state. + gnmi.Await(t, peer, gnmi.OC().Interface(peerPorts[0].Peer).Enabled().State(), defaultGNMIWait, false) + if err := verifyBlockingState(t, host, portChannel1, peerPorts[0].Host); err != nil { + t.Errorf("LACP state is not blocking: %v", err) + } +} diff --git a/sdn_tests/pins_ondatra/tests/lacp_timeout_test.go b/sdn_tests/pins_ondatra/tests/lacp_timeout_test.go index 153a5bf565..4c0cdcc2af 100644 --- a/sdn_tests/pins_ondatra/tests/lacp_timeout_test.go +++ b/sdn_tests/pins_ondatra/tests/lacp_timeout_test.go @@ -10,7 +10,7 @@ import ( "github.com/openconfig/ondatra/gnmi" "github.com/openconfig/ondatra/gnmi/oc" "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/binding/pinsbind" - "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure//testhelper/testhelper" + "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/testhelper/testhelper" "github.com/pkg/errors" ) @@ -131,9 +131,10 @@ func verifyLACPTimeout(t *testing.T, hostActivity oc.E_Lacp_LacpActivityType, ho // Choose a random port to test, and get the LACPDU count. peerportslen := len(peerPorts) - max := big.NewInt(peerportslen) + max := big.NewInt(int64(peerportslen)) randomIndex, _ := rand.Int(rand.Reader, max) - port := randomIndex + port_64 := randomIndex.Int64() + port := int(port_64) hostBefore := gnmi.Get(t, host, gnmi.OC().Lacp().Interface(portChannel).Member(peerPorts[port].Host).Counters().State()) peerBefore := gnmi.Get(t, peer, gnmi.OC().Lacp().Interface(portChannel).Member(peerPorts[port].Peer).Counters().State()) diff --git a/sdn_tests/pins_ondatra/tests/link_event_damping_test.go b/sdn_tests/pins_ondatra/tests/link_event_damping_test.go index e5d8db775e..f9cc1689e0 100644 --- a/sdn_tests/pins_ondatra/tests/link_event_damping_test.go +++ b/sdn_tests/pins_ondatra/tests/link_event_damping_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/openconfigondatra" + "github.com/openconfig/ondatra" "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/binding/pinsbind" "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/testhelper/testhelper" "github.com/pkg/errors" diff --git a/sdn_tests/pins_ondatra/tests/mgmt_interface_test.go b/sdn_tests/pins_ondatra/tests/mgmt_interface_test.go new file mode 100644 index 0000000000..e9bca613c9 --- /dev/null +++ b/sdn_tests/pins_ondatra/tests/mgmt_interface_test.go @@ -0,0 +1,594 @@ +package mgmt_interface_test + +// This suite of tests exercises the gNMI paths associated with the management + +import ( + "errors" + "fmt" + "math/rand" + "net" + "testing" + "time" + + "github.com/openconfig/ondatra" + "github.com/openconfig/ondatra/gnmi" + "github.com/openconfig/ondatra/gnmi/oc" + "github.com/openconfig/testt" + "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/binding/pinsbind" + "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/testhelper/testhelper" +) + +func TestMain(m *testing.M) { + ondatra.RunTests(m, pinsbind.New) +} + +var ( + bond0Name = "bond0" + interfaceIndex = uint32(0) + calledMockConfigPush = false + managementInterfaces = []string{ + bond0Name, + } +) + +type ipAddressInfo struct { + address string + prefixLength uint8 +} + +func fetchMgmtIPv4AddressAndPrefix(t *testing.T) (ipAddressInfo, error) { + // Reads the existing management interface IPv4 address. For these tests, + // we will not be able to change the address without breaking connection of + // the proxy used by the test. + dut := ondatra.DUT(t, "DUT") + ipInfo := gnmi.Get(t, dut, gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv4().State()) + for _, v := range ipInfo.Address { + addr := v.GetIp() + if addr == "" { + continue + } + return ipAddressInfo{address: addr, prefixLength: v.GetPrefixLength()}, nil + } + return ipAddressInfo{}, errors.New("no IPv4 management interface has been configured") +} + +func fetchMgmtIPv6AddressAndPrefix(t *testing.T) (ipAddressInfo, error) { + // Reads the existing management interface IPv6 address. For these tests, + // we will not be able to change the address without breaking connection of + // the proxy used by the test. + dut := ondatra.DUT(t, "DUT") + ipInfo := gnmi.Get(t, dut, gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv6().State()) + for _, v := range ipInfo.Address { + addr := v.GetIp() + if addr == "" { + continue + } + return ipAddressInfo{address: addr, prefixLength: v.GetPrefixLength()}, nil + } + return ipAddressInfo{}, errors.New("no IPv6 management interface has been configured") +} + +func mockConfigPush(t *testing.T) { + // Performs a mock config push by ensuring the management interface database + // entries expected for IPv4 and IPv6 addresses have been setup. + // TODO: Remove calls to this function once the helper function + // to perform a default config during setup is available. + dut := ondatra.DUT(t, "DUT") + d := &oc.Root{} + + // Create the bond0 interface. + if !calledMockConfigPush { + t.Logf("Config push for %v", bond0Name) + newIface := d.GetOrCreateInterface(bond0Name) + newIface.Name = &bond0Name + newIface.Type = oc.IETFInterfaces_InterfaceType_ieee8023adLag + gnmi.Replace(t, dut, gnmi.OC().Interface(bond0Name).Config(), newIface) + calledMockConfigPush = true + } +} + +// ----------------------------------------------------------------------------- +// Generic management interface path tests +// ----------------------------------------------------------------------------- + +func TestGetInterfaceDefaultInfo(t *testing.T) { + // This test confirms generic management interface information is correct. + // Paths tested: + // /interfaces/interface[name=]/ethernet/state/mac-address + // /interfaces/interface[name=]/state/name + // /interfaces/interface[name=]/state/oper-status + // /interfaces/interface[name=]/state/type + defer testhelper.NewTearDownOptions(t).WithID("1b0707dd-4112-4c0f-ad74-1998df876747").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + for _, iface := range managementInterfaces { + mgmtInterface := gnmi.OC().Interface(iface) + macAddress := gnmi.Get(t, dut, mgmtInterface.Ethernet().MacAddress().State()) + if _, err := net.ParseMAC(macAddress); err != nil { + t.Errorf("MGMT component (%v) has invalid mac-address format! got:%v: %v", iface, macAddress, err) + } + if mgmtName := gnmi.Get(t, dut, mgmtInterface.Name().State()); mgmtName != iface { + t.Errorf("MGMT component (%v) name match failed! got:%v, want:%v", iface, mgmtName, iface) + } + if operStatus, statusWant := gnmi.Get(t, dut, mgmtInterface.OperStatus().State()), oc.Interface_OperStatus_UP; operStatus != statusWant { + t.Errorf("MGMT component (%v) oper-status match failed! got:%v, want:%v", iface, operStatus, statusWant) + } + if ifaceType, typeWant := gnmi.Get(t, dut, mgmtInterface.Type().State()), oc.IETFInterfaces_InterfaceType_ieee8023adLag; ifaceType != typeWant { + t.Errorf("MGMT component (%v) type match failed! got:%v, want:%v", iface, ifaceType, typeWant) + } + } +} + +func TestSetName(t *testing.T) { + // This test confirms the name of a management interface can be written. + // Paths tested: + // /interfaces/interface[name=]/config/name + // /interfaces/interface[name=]/state/name + defer testhelper.NewTearDownOptions(t).WithID("826882ed-0534-499c-880a-91cb3c078a03").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + mgmtInterfaceState := gnmi.OC().Interface(bond0Name) + mgmtInterfaceConfig := gnmi.OC().Interface(bond0Name) + + gnmi.Replace(t, dut, mgmtInterfaceConfig.Name().Config(), bond0Name) + gnmi.Await(t, dut, mgmtInterfaceState.Name().State(), 5*time.Second, bond0Name) + + // Expect name on state path to have changed. + if configuredName := gnmi.Get(t, dut, mgmtInterfaceConfig.Name().Config()); configuredName != bond0Name { + t.Errorf("MGMT component (%v) name match failed! set:%v, config-path-value:%v (want:%v)", bond0Name, bond0Name, configuredName, bond0Name) + } +} + +func TestSetInvalidType(t *testing.T) { + // This test confirms that an invalid type cannot be set for the management + // interface. + // Paths tested: + // /interfaces/interface[name=]/config/type + // /interfaces/interface[name=]/state/type + defer testhelper.NewTearDownOptions(t).WithID("0c838fb4-6846-4f5e-a5db-cbcabacdb020").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + for _, iface := range managementInterfaces { + mgmtInterfaceState := gnmi.OC().Interface(iface) + mgmtInterfaceConfig := gnmi.OC().Interface(iface) + originalType := gnmi.Get(t, dut, mgmtInterfaceState.Type().State()) + originalConfigType := gnmi.Get(t, dut, mgmtInterfaceConfig.Type().Config()) + + invalidType := oc.IETFInterfaces_InterfaceType_softwareLoopback + // This call should fail, since the type is invalid for a management interface. + testt.ExpectFatal(t, func(t testing.TB) { + gnmi.Replace(t, dut, mgmtInterfaceConfig.Type().Config(), invalidType) + }) + stateType := gnmi.Get(t, dut, mgmtInterfaceState.Type().State()) + configuredType := gnmi.Get(t, dut, mgmtInterfaceConfig.Type().Config()) + + // Invalid type should not have gone through. + if stateType != originalType || configuredType == invalidType { + t.Errorf("MGMT component (%v) type match failed! set:%v, config-path-value:%v (want:%v), state-path-value:%v (want:%v)", iface, invalidType, configuredType, originalConfigType, stateType, originalType) + } + } +} + +func TestSetInvalidName(t *testing.T) { + // This test confirms that an invalid name cannot be set for the management + // interface. + // Paths tested: + // /interfaces/interface[name=]/config/name + // /interfaces/interface[name=]/state/name + defer testhelper.NewTearDownOptions(t).WithID("67a8c951-34cc-4148-9f09-a779e7976d03").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + mgmtInterfaceState := gnmi.OC().Interface(bond0Name) + mgmtInterfaceConfig := gnmi.OC().Interface(bond0Name) + originalName := gnmi.Get(t, dut, mgmtInterfaceState.Name().State()) + originalConfigName := gnmi.Get(t, dut, mgmtInterfaceConfig.Name().Config()) + + invalidName := "mybond0" + // Setting invalid name should be ignored. + // TODO: This replace call should fail. + gnmi.Replace(t, dut, mgmtInterfaceConfig.Name().Config(), invalidName) + gnmi.Await(t, dut, mgmtInterfaceState.Name().State(), 5*time.Second, originalName) + + // Invalid name should not be accepted. + if configuredName := gnmi.Get(t, dut, mgmtInterfaceConfig.Name().Config()); configuredName == invalidName { + t.Errorf("MGMT component (%v) name match failed! set:%v, config-path-value:%v (want:%v)", bond0Name, invalidName, configuredName, originalConfigName) + } +} + +// ----------------------------------------------------------------------------- +// Counter path tests +// ----------------------------------------------------------------------------- + +func verifyInCounters(counters *oc.Interface_Counters) []error { + var rv []error + if counters.InDiscards == nil { + rv = append(rv, errors.New("in-discards")) + } + if counters.InErrors == nil { + rv = append(rv, errors.New("in-errors")) + } + if counters.InOctets == nil { + rv = append(rv, errors.New("in-octets")) + } + if counters.InPkts == nil { + rv = append(rv, errors.New("in-pkts")) + } + return rv +} + +func verifyOutCounters(counters *oc.Interface_Counters) []error { + var rv []error + if counters.OutDiscards == nil { + rv = append(rv, errors.New("out-discards")) + } + if counters.OutErrors == nil { + rv = append(rv, errors.New("out-errors")) + } + if counters.OutOctets == nil { + rv = append(rv, errors.New("out-octets")) + } + if counters.OutPkts == nil { + rv = append(rv, errors.New("out-pkts")) + } + return rv +} + +func TestInCounters(t *testing.T) { + // This test confirms that the input counters (RX side) are updated by packet + // events. Note: the management interface is the connection by which gNMI + // operations take place, so it is difficult to get a precise count of the + // expected differences in counter values. + // Paths tested: + // /interfaces/interface[name=]/state/counters/in-discards + // /interfaces/interface[name=]/state/counters/in-errors + // /interfaces/interface[name=]/state/counters/in-octets + // /interfaces/interface[name=]/state/counters/in-pkts + defer testhelper.NewTearDownOptions(t).WithID("b0801966-fb60-456d-9c91-ee3191c5e7e1").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + counters := gnmi.OC().Interface(bond0Name).Counters() + initialState := gnmi.Get(t, dut, counters.State()) + if errors := verifyInCounters(initialState); len(errors) != 0 { + t.Fatalf("MGMT component (%v) has invalid initial input counters: %v", bond0Name, errors) + } + + t.Logf("Initial in-counters state has:") + t.Logf(" in-discards:%v in-errors:%v in-octets:%v in-pkts: %v", initialState.GetInDiscards(), initialState.GetInErrors(), initialState.GetInOctets(), initialState.GetInPkts()) + + // The management interface is active. That is how gNMI operations are + // communicated to the switch. In this test, we verify that packets have been + // received and that there were no errors. + // Initiate a handful of Get operations to ensure there is traffic. + gnmi.Get(t, dut, gnmi.OC().Interface(bond0Name).Name().State()) + gnmi.Get(t, dut, gnmi.OC().Interface(bond0Name).Name().State()) + gnmi.Get(t, dut, gnmi.OC().Interface(bond0Name).Name().State()) + + nextState := gnmi.Get(t, dut, counters.State()) + if errors := verifyInCounters(nextState); len(errors) != 0 { + t.Fatalf("MGMT component (%v) has invalid next input counters: %v", bond0Name, errors) + } + + t.Logf("Next in-counters state has:") + t.Logf(" in-discards:%v in-errors:%v in-octets:%v in-pkts: %v", nextState.GetInDiscards(), nextState.GetInErrors(), nextState.GetInOctets(), nextState.GetInPkts()) + + if initialState.GetInDiscards() > nextState.GetInDiscards() { + t.Errorf("MGMT component (%v) has unexpected decrease in in-discards %v -> %v", bond0Name, initialState.GetInDiscards(), nextState.GetInDiscards()) + } + if nextState.GetInDiscards() != 0 { + t.Logf("MGMT component (%v) has non-zero in-discards: %v", bond0Name, nextState.GetInDiscards()) + } + if initialState.GetInErrors() > nextState.GetInErrors() { + t.Errorf("MGMT component (%v) has unexpected decrease in in-errors %v -> %v", bond0Name, initialState.GetInErrors(), nextState.GetInErrors()) + } + if nextState.GetInErrors() != 0 { + t.Logf("MGMT component (%v) has non-zero in-errors: %v", bond0Name, nextState.GetInErrors()) + } + if initialState.GetInOctets() >= nextState.GetInOctets() { + t.Errorf("MGMT component (%v) in-octets did not increase as expected %v -> %v", bond0Name, initialState.GetInOctets(), nextState.GetInOctets()) + } + if initialState.GetInPkts() >= nextState.GetInPkts() { + t.Errorf("MGMT component (%v) in-pkts did not increase as expected %v -> %v", bond0Name, initialState.GetInPkts(), nextState.GetInPkts()) + } +} + +func TestOutCounters(t *testing.T) { + // This test confirms that the output counters (TX side) are updated by packet + // events. Note: the management interface is the connection by which gNMI + // operations take place, so it is difficult to get a precise count of the + // expected differences in counter values. + // Paths tested: + // /interfaces/interface[name=]/state/counters/out-discards + // /interfaces/interface[name=]/state/counters/out-errors + // /interfaces/interface[name=]/state/counters/out-octets + // /interfaces/interface[name=]/state/counters/out-pkts + defer testhelper.NewTearDownOptions(t).WithID("ccec883e-0b87-4084-8861-77393460976b").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + counters := gnmi.OC().Interface(bond0Name).Counters() + initialState := gnmi.Get(t, dut, counters.State()) + if errors := verifyOutCounters(initialState); len(errors) != 0 { + t.Fatalf("MGMT component (%v) has invalid initial output counters: %v", bond0Name, errors) + } + + t.Logf("Initial out-counters state has:") + t.Logf(" out-discards:%v out-errors:%v out-octets:%v out-pkts: %v", initialState.GetOutDiscards(), initialState.GetOutErrors(), initialState.GetOutOctets(), initialState.GetOutPkts()) + + // The management interface is active. That is how gNMI operations are + // communicated to the switch. In this test, we verify that packets have been + // received and that there were no errors. + // Initiate a handful of Get operations to ensure there is traffic. + gnmi.Get(t, dut, gnmi.OC().Interface(bond0Name).Name().State()) + gnmi.Get(t, dut, gnmi.OC().Interface(bond0Name).Name().State()) + gnmi.Get(t, dut, gnmi.OC().Interface(bond0Name).Name().State()) + + nextState := gnmi.Get(t, dut, counters.State()) + if errors := verifyOutCounters(nextState); len(errors) != 0 { + t.Fatalf("MGMT component (%v) has invalid next output counters: %v", bond0Name, errors) + } + + t.Logf("Next out-counters state has:") + t.Logf(" out-discards:%v out-errors:%v out-octets:%v out-pkts: %v", nextState.GetOutDiscards(), nextState.GetOutErrors(), nextState.GetOutOctets(), nextState.GetOutPkts()) + + if initialState.GetOutDiscards() > nextState.GetOutDiscards() { + t.Errorf("MGMT component (%v) has unexpected decrease in out-discards %v -> %v", bond0Name, initialState.GetOutDiscards(), nextState.GetOutDiscards()) + } + if nextState.GetOutDiscards() != 0 { + t.Logf("MGMT component (%v) has non-zero out-discards: %v", bond0Name, nextState.GetOutDiscards()) + } + if initialState.GetOutErrors() > nextState.GetOutErrors() { + t.Errorf("MGMT component (%v) has unexpected decrease in out-errors %v -> %v", bond0Name, initialState.GetOutErrors(), nextState.GetOutErrors()) + } + if nextState.GetOutErrors() != 0 { + t.Logf("MGMT component (%v) has non-zero out-errors: %v", bond0Name, nextState.GetOutErrors()) + } + if initialState.GetOutOctets() >= nextState.GetOutOctets() { + t.Errorf("MGMT component (%v) out-octets did not increase as expected %v -> %v", bond0Name, initialState.GetOutOctets(), nextState.GetOutOctets()) + } + if initialState.GetOutPkts() >= nextState.GetOutPkts() { + t.Errorf("MGMT component (%v) out-pkts did not increase as expected %v -> %v", bond0Name, initialState.GetOutPkts(), nextState.GetOutPkts()) + } +} + +// ----------------------------------------------------------------------------- +// IPv4 path tests +// ----------------------------------------------------------------------------- +func TestSetIPv4AddressAndPrefixLength(t *testing.T) { + // This test confirms that a new IPv4 address and prefix-length can be added. + // Note: the entire "tree" has to be added in one gNMI operation. (The IP and + // prefix length cannot be written separately.) + // formed. + // Paths tested: + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv4/addresses/address[ip=
]/config/ip + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv4/addresses/address[ip=
]/config/prefix-length + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv4/addresses/address[ip=
]/state/ip + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv4/addresses/address[ip=
]/state/prefix-length + defer testhelper.NewTearDownOptions(t).WithID("64003075-93a5-41b3-b962-74e9f36dde94").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + // We can't change the management interface IP address; the connection via the + // proxy would be lost. We can, however, write the existing value again. + newIPv4Info, err := fetchMgmtIPv4AddressAndPrefix(t) + restoreIPv4State := false + + if err != nil { + // If IPv4 is not used in the testbed, we can set a valid address. + t.Logf("Unable to fetch IPv4 management address: %v", err) + t.Logf("We will create an unused one.") + // Address is [16:126].[0:255].[0:255].[0:255]. + start, end := 16, 126 + firstPrefixPart1 := make([]int, end-start+1) + for i := range firstPrefixPart1 { + firstPrefixPart1[i] = i + start + } + start, end = 128, 223 + firstPrefixPart2 := make([]int, end-start+1) + for i := range firstPrefixPart2 { + firstPrefixPart2[i] = i + start + } + firstPrefix := append(firstPrefixPart1, firstPrefixPart2...) + + newAddr := fmt.Sprintf("%d.%d.%d.%d", firstPrefix[rand.Int()%len(firstPrefix)], rand.Intn(256), rand.Intn(256), rand.Intn(256)) + newPrefix := uint8(rand.Intn(27) + 5) // 5 to 31 + newIPv4Info = ipAddressInfo{address: newAddr, prefixLength: newPrefix} + restoreIPv4State = true + } + + d := &oc.Root{} + iface := d.GetOrCreateInterface(bond0Name).GetOrCreateSubinterface(interfaceIndex) + newV4 := iface.GetOrCreateIpv4().GetOrCreateAddress(newIPv4Info.address) + newV4.Ip = &newIPv4Info.address + newV4.PrefixLength = &newIPv4Info.prefixLength + + ipv4 := gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv4().Address(newIPv4Info.address) + gnmi.Replace(t, dut, gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv4().Address(newIPv4Info.address).Config(), newV4) + if restoreIPv4State { + defer gnmi.Delete(t, dut, gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv4().Address(newIPv4Info.address).Config()) + } + // Give the configuration a chance to become active. + time.Sleep(1 * time.Second) + + if observed := gnmi.Get(t, dut, ipv4.State()); observed.GetIp() != newIPv4Info.address || observed.GetPrefixLength() != newIPv4Info.prefixLength { + t.Errorf("MGMT component (%v) address match failed! state-path-value:%v/%v (want:%v/%v)", bond0Name, observed.GetIp(), observed.GetPrefixLength(), newIPv4Info.address, newIPv4Info.prefixLength) + } +} + +func TestSetIPv4InvalidAddress(t *testing.T) { + // This test confirms that an invalid IPv4 address cannot be set. + // IPv4 addresses that begin with 0 or 255 (e.g. 255.1.2.3 or 0.4.5.6) are + // considered invalid. + // Paths tested: + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv4/addresses/address[ip=
]/config/ip + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv4/addresses/address[ip=
]/state/ip + defer testhelper.NewTearDownOptions(t).WithID("00acbce9-069e-43e1-a511-9b45bb3ad5b0").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + var invalidIPPaths = []string{ + fmt.Sprintf("255.%v.%v.%v", rand.Intn(256), rand.Intn(256), rand.Intn(256)), + fmt.Sprintf("0.%v.%v.%v", rand.Intn(256), rand.Intn(256), rand.Intn(256)), + } + configuredIPv4PrefixLength := uint8(16) + + for _, invalidIPPath := range invalidIPPaths { + + d := &oc.Root{} + iface := d.GetOrCreateInterface(bond0Name).GetOrCreateSubinterface(interfaceIndex) + newV4 := iface.GetOrCreateIpv4().GetOrCreateAddress(invalidIPPath) + newV4.Ip = &invalidIPPath + newV4.PrefixLength = &configuredIPv4PrefixLength + + ipv4 := gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv4().Address(invalidIPPath) + ipv4Config := gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv4().Address(invalidIPPath) + // Cannot write invalid IPv4 address. + testt.ExpectFatal(t, func(t testing.TB) { + gnmi.Replace(t, dut, ipv4Config.Config(), newV4) + }) + + // There should be no IP set with the invalid IPv4 address. + testt.ExpectFatal(t, func(t testing.TB) { + observedIP := gnmi.Get(t, dut, ipv4.Ip().State()) + t.Logf("MGMT component (%v) observed IPv4 address: %v.", bond0Name, observedIP) + }) + } +} + +// ----------------------------------------------------------------------------- +// IPv6 path tests +// ----------------------------------------------------------------------------- +func TestGetIPv6DefaultInfo(t *testing.T) { + // This test confirms that generic IPv6 information can be read and is well + // formed. + // Paths tested: + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv6/addresses/address[ip=
]/state/ip + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv6/addresses/address[ip=
]/state/prefix-length + defer testhelper.NewTearDownOptions(t).WithID("5bc725a2-befe-4154-bd7d-d390c87dc4d8").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + configuredIPv6Info, err := fetchMgmtIPv6AddressAndPrefix(t) + if err != nil { + t.Fatalf("Unable to fetch IPv6 management address: %v", err) + } + ipv6 := gnmi.Get(t, dut, gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv6().Address(configuredIPv6Info.address).State()) + + if *ipv6.PrefixLength >= 128 { + t.Errorf("MGMT component (%v) has an incorrect prefix-length: %v (want: [0:127]) on subinterface %v with IP %v", bond0Name, *ipv6.PrefixLength, interfaceIndex, configuredIPv6Info.address) + } + parsedIP := net.ParseIP(*ipv6.Ip) + if parsedIP == nil { + t.Fatalf("MGMT component (%v) has an incorrectly formatted IPv6 address: %v", bond0Name, *ipv6.Ip) + } + ipAsBytes := parsedIP.To16() + if ipAsBytes == nil { + t.Fatalf("MGMT component (%v) has an incorrectly formatted IPv6 address: %v could not be parsed", bond0Name, *ipv6.Ip) + } + if len(ipAsBytes) != 16 { + t.Fatalf("MGMT component (%v) IPv6 address is only %v bytes.", bond0Name, len(ipAsBytes)) + } +} + +func TestSetIPv6AddressAndPrefixLength(t *testing.T) { + // This test confirms that a new IPv6 address and prefix-length can be added. + // Note: the entire "tree" has to be added in one gNMI operation. (The IP and + // prefix length cannot be written separately.) + // formed. + // Paths tested: + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv6/addresses/address[ip=
]/config/ip + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv6/addresses/address[ip=
]/config/prefix-length + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv6/addresses/address[ip=
]/state/ip + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv6/addresses/address[ip=
]/state/prefix-length + defer testhelper.NewTearDownOptions(t).WithID("0f79f318-b0de-4352-a045-540aa1da94d4").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + // We can't change the management interface IP address; the connection via the + // proxy would be lost. We can, however, write the existing value again. + newIPInfo, err := fetchMgmtIPv6AddressAndPrefix(t) + if err != nil { + t.Fatalf("Unable to fetch IPv6 management address: %v", err) + } + + d := &oc.Root{} + iface := d.GetOrCreateInterface(bond0Name).GetOrCreateSubinterface(interfaceIndex) + newV6 := iface.GetOrCreateIpv6().GetOrCreateAddress(newIPInfo.address) + newV6.Ip = &newIPInfo.address + newV6.PrefixLength = &newIPInfo.prefixLength + + ipv6 := gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv6().Address(newIPInfo.address) + gnmi.Replace(t, dut, gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv6().Address(newIPInfo.address).Config(), newV6) + // Give the configuration a chance to become active. + time.Sleep(1 * time.Second) + + if observed := gnmi.Get(t, dut, ipv6.State()); *observed.Ip != newIPInfo.address || *observed.PrefixLength != newIPInfo.prefixLength { + t.Errorf("MGMT component (%v) address match failed! state-path-value:%v/%v (want:%v/%v)", bond0Name, *observed.Ip, *observed.PrefixLength, newIPInfo.address, newIPInfo.prefixLength) + } +} + +func TestSetIPv6InvalidPrefixLength(t *testing.T) { + // This test confirms that an invalid IPv6 prefix-length cannot be set. + // Any prefix length in the range [0:128] is supported. + // Paths tested: + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv6/addresses/address[ip=
]/config/prefix-length + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv6/addresses/address[ip=
]/state/prefix-length + defer testhelper.NewTearDownOptions(t).WithID("7813ab28-1d8c-43ca-ab21-d4106a733e47").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + configuredIPv6Info, err := fetchMgmtIPv6AddressAndPrefix(t) + if err != nil { + t.Fatalf("Unable to fetch IPv6 management address: %v", err) + } + ipv6 := gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv6().Address(configuredIPv6Info.address) + ipv6Config := gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv6().Address(configuredIPv6Info.address) + originalPrefixLength := gnmi.Get(t, dut, ipv6.PrefixLength().State()) + + invalidPrefixLength := uint8(129) + testt.ExpectFatal(t, func(t testing.TB) { + gnmi.Replace(t, dut, ipv6Config.PrefixLength().Config(), invalidPrefixLength) + }) + gnmi.Await(t, dut, ipv6.PrefixLength().State(), 5*time.Second, originalPrefixLength) + configuredPrefixLength := gnmi.Get(t, dut, ipv6Config.PrefixLength().Config()) + + if configuredPrefixLength == invalidPrefixLength { + t.Errorf("MGMT component (%v) prefix-length match failed! set:%v, config-path-value:%v (want:%v)", bond0Name, invalidPrefixLength, configuredPrefixLength, originalPrefixLength) + } +} + +func TestSetIPv6InvalidAddress(t *testing.T) { + // This test confirms that an invalid IPv6 address cannot be set. + // Paths tested: + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv6/addresses/address[ip=
]/config/ip + // /interfaces/interface[name=]/subinterfaces/subinterface[index=]/ipv6/addresses/address[ip=
]/state/ip + defer testhelper.NewTearDownOptions(t).WithID("c58360a6-4d7f-442d-a9f7-9f1d72682ee2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + mockConfigPush(t) + + invalidIPPath := "ffff:ffff:ffff:ffff:ffff:f25c:77ff:fe7f:69be" + configuredIPv6PrefixLength := uint8(64) + ipv6 := gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv6().Address(invalidIPPath) + ipv6Config := gnmi.OC().Interface(bond0Name).Subinterface(interfaceIndex).Ipv6().Address(invalidIPPath) + + d := &oc.Root{} + iface := d.GetOrCreateInterface(bond0Name).GetOrCreateSubinterface(interfaceIndex) + newV6 := iface.GetOrCreateIpv6().GetOrCreateAddress(invalidIPPath) + newV6.Ip = &invalidIPPath + newV6.PrefixLength = &configuredIPv6PrefixLength + + // Cannot write invalid IPv6 address. + testt.ExpectFatal(t, func(t testing.TB) { + gnmi.Replace(t, dut, ipv6Config.Config(), newV6) + }) + + // There should be no IP set with the invalid IPv6 address. + testt.ExpectFatal(t, func(t testing.TB) { + observedIP := gnmi.Get(t, dut, ipv6.Ip().State()) + t.Logf("MGMT component (%v) observed IPv6 address: %v.", bond0Name, observedIP) + }) +} diff --git a/sdn_tests/pins_ondatra/tests/platforms_hardware_component_test.go b/sdn_tests/pins_ondatra/tests/platforms_hardware_component_test.go index 057d94f057..4697111b15 100644 --- a/sdn_tests/pins_ondatra/tests/platforms_hardware_component_test.go +++ b/sdn_tests/pins_ondatra/tests/platforms_hardware_component_test.go @@ -1,8 +1,8 @@ package platforms_hardware_component_test import ( - "reflect" "regexp" + "reflect" "testing" "time" @@ -113,7 +113,7 @@ func TestSetValidICName(t *testing.T) { gnmi.Replace(t, dut, componentPath.Name().Config(), name) gnmi.Await(t, dut, componentPath.Name().State(), awaitTime, name) - fullyQualifiedName := "ju1u1m1b1s1i1.ibs40.net.google.com" + fullyQualifiedName := "abc.def.test.com" testhelper.ReplaceFullyQualifiedName(t, dut, name, fullyQualifiedName) testhelper.AwaitFullyQualifiedName(t, dut, name, awaitTime, fullyQualifiedName) } @@ -187,7 +187,7 @@ func TestPersistenceAfterReboot(t *testing.T) { t.Fatalf("Failed to fetch integrated-circuit info: %v", err) } - fullyQualifiedName := "ju1u1m1b1s1i1.ibs40.net.google.com" + fullyQualifiedName := "abc.def.test.com" nodeID := uint64(12345678) t.Log("Configuring config paths before reboot") @@ -253,7 +253,233 @@ func TestPersistenceAfterReboot(t *testing.T) { } } -/ Health-indicator test. +// FPGA tests. +func TestGetFPGAInfo(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("52a71049-40dc-4f2d-b074-4b0f649064f0").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + fpgas, err := testhelper.FPGAInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch FPGA info: %v", err) + } + + var fpgaResetCounts []uint8 + for _, fpga := range fpgas { + name := fpga.GetName() + componentPath := gnmi.OC().Component(name) + wantType := "FPGA" + if gotType := testhelper.FPGAType(t, dut, &fpga); gotType != wantType { + t.Errorf("%v type match failed! got:%v, want:%v", name, gotType, wantType) + } + + if mfgName := gnmi.Get(t, dut, componentPath.MfgName().State()); mfgName != fpga.GetMfgName() { + t.Errorf("%v manufacturer name match failed! got:%v, want:%v", name, mfgName, fpga.GetMfgName()) + } + + if description := gnmi.Get(t, dut, componentPath.Description().State()); description != fpga.GetDescription() { + t.Errorf("%v description match failed! got:%v, want:%v", name, description, fpga.GetDescription()) + } + + if err := verifyRegexMatch(fpga.GetFirmwareVersionRegex(), gnmi.Get(t, dut, componentPath.FirmwareVersion().State())); err != nil { + t.Errorf("%v firmware version match failed! %v", name, err) + } + + resetCauseMap := testhelper.FPGAResetCauseMap(t, dut, &fpga) //fpgaInfo.ResetCause + if got, want := len(resetCauseMap), fpga.GetResetCauseNum(); got != want { + t.Errorf("%v invalid number of reset causes! got:%v, want:%v", name, got, want) + } + for index, resetCause := range resetCauseMap { + if got, want := resetCause.GetIndex(), index; got != want { + t.Errorf("%v reset-cause-index: %v index match failed! got:%v, want:%v", name, index, got, want) + } + if got := resetCause.GetCause(); got < testhelper.ResetCause_Cause_POWER || got > testhelper.ResetCause_Cause_CPU { + t.Errorf("%v reset-cause-index: %v cause match failed! got:%v, want:range(%v-%v)", name, index, got, testhelper.ResetCause_Cause_POWER, testhelper.ResetCause_Cause_CPU) + } + } + + // Need to know current reset count, since after reboot it should be current count + 1. + fpgaResetCounts = append(fpgaResetCounts, testhelper.FPGAResetCount(t, dut, &fpga)) + } + + // Reboot DUT and verify that the latest reset cause is SOFTWARE. + waitTime, err := testhelper.RebootTimeForDevice(t, dut) + if err != nil { + t.Fatalf("Unable to get reboot wait time: %v", err) + } + params := testhelper.NewRebootParams().WithWaitTime(waitTime).WithCheckInterval(30 * time.Second).WithRequest(syspb.RebootMethod_COLD) + if err := testhelper.Reboot(t, dut, params); err != nil { + t.Fatalf("Failed to reboot DUT: %v", err) + } + // Wait for the switch to update FPGA information. + time.Sleep(time.Minute) + + for i, fpga := range fpgas { + name := fpga.GetName() + + if got, want := testhelper.FPGAResetCount(t, dut, &fpga), fpgaResetCounts[i]+1; got != want { + t.Errorf("%v latest reset count match failed after reboot! got:%v, want:%v", name, got, want) + } + + if fpga.GetResetCauseNum() == 0 { + // This FPGA doesn't support reset causes. + continue + } + if got, want := testhelper.FPGAResetCause(t, dut, &fpga, 0), testhelper.ResetCause_Cause_SOFTWARE; got != want { + t.Errorf("%v latest reset cause match failed after reboot! got:%v, want:%v", name, got, want) + } + } +} + +// Temperature sensor tests. +func TestGetTemperatureSensorDefaultInformation(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("b68ca974-590c-4685-9da4-4c344c74a056").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + type sensorInfo struct { + ocType oc.E_PlatformTypes_OPENCONFIG_HARDWARE_COMPONENT + subType string + } + sensorInfoMap := map[testhelper.TemperatureSensorType]sensorInfo{ + testhelper.CPUTempSensor: { + ocType: oc.PlatformTypes_OPENCONFIG_HARDWARE_COMPONENT_CPU, + }, + testhelper.HeatsinkTempSensor: { + ocType: oc.PlatformTypes_OPENCONFIG_HARDWARE_COMPONENT_SENSOR, + subType: "HEAT_SINK_TEMPERATURE_SENSOR", + }, + testhelper.ExhaustTempSensor: { + ocType: oc.PlatformTypes_OPENCONFIG_HARDWARE_COMPONENT_SENSOR, + subType: "EXHAUST_TEMPERATURE_SENSOR", + }, + testhelper.InletTempSensor: { + ocType: oc.PlatformTypes_OPENCONFIG_HARDWARE_COMPONENT_SENSOR, + subType: "INLET_TEMPERATURE_SENSOR", + }, + testhelper.DimmTempSensor: { + ocType: oc.PlatformTypes_OPENCONFIG_HARDWARE_COMPONENT_SENSOR, + subType: "DIMM_TEMPERATURE_SENSOR", + }, + } + + tests := []struct { + name string + sensorType testhelper.TemperatureSensorType + }{ + { + name: "CPUTemperatureSensorInfo", + sensorType: testhelper.CPUTempSensor, + }, + { + name: "HeatsinkTemperatureSensorInfo", + sensorType: testhelper.HeatsinkTempSensor, + }, + { + name: "ExhaustTemperatureSensorInfo", + sensorType: testhelper.ExhaustTempSensor, + }, + { + name: "InletTemperatureSensorInfo", + sensorType: testhelper.InletTempSensor, + }, + { + name: "DimmTemperatureSensorInfo", + sensorType: testhelper.DimmTempSensor, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expectedInfo, ok := sensorInfoMap[tt.sensorType] + if !ok { + t.Fatalf("Sensor type: %v not found in expected info map", tt.sensorType) + } + + sensors, err := testhelper.TemperatureSensorInfoForDevice(t, dut, tt.sensorType) + if err != nil { + t.Fatalf("Failed to fetch temperature info for %v: %v", expectedInfo, err) + } + + for _, sensor := range sensors { + name := sensor.GetName() + info := gnmi.Get(t, dut, gnmi.OC().Component(name).State()) + + if got, want := info.GetName(), name; got != want { + t.Errorf("%v name match failed! got:%v, want:%v", name, got, want) + } + if got, want := info.GetParent(), "chassis"; got != want { + t.Errorf("%v parent match failed! got:%v, want:%v", name, got, want) + } + if got, want := info.GetType(), expectedInfo.ocType; got != want { + t.Errorf("%v type match failed! got:%v, want:%v", name, got, want) + } + if got, want := info.GetLocation(), sensor.GetLocation(); got != want { + t.Errorf("%v location match failed! got:%v, want:%v", name, got, want) + } + + // Sensor sub-type is not applicable for CPU temperature sensor. + if tt.sensorType == testhelper.CPUTempSensor { + continue + } + + if got, want := testhelper.SensorType(t, dut, &sensor), expectedInfo.subType; got != want { + t.Errorf("%v sensor sub-type match failed! got:%v, want:%v", name, got, want) + continue + } + } + + }) + } +} + +func TestGetTemperatureSensorTemperatureInformation(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("294bf647-cff4-47d6-a701-ad9dfe7ff8f3").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + tests := []struct { + name string + sensorType testhelper.TemperatureSensorType + }{ + { + name: "CPUTemperatureSensorInfo", + sensorType: testhelper.CPUTempSensor, + }, + { + name: "HeatsinkTemperatureSensorInfo", + sensorType: testhelper.HeatsinkTempSensor, + }, + { + name: "ExhaustTemperatureSensorInfo", + sensorType: testhelper.ExhaustTempSensor, + }, + { + name: "InletTemperatureSensorInfo", + sensorType: testhelper.InletTempSensor, + }, + { + name: "DimmTemperatureSensorInfo", + sensorType: testhelper.DimmTempSensor, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sensors, err := testhelper.TemperatureSensorInfoForDevice(t, dut, tt.sensorType) + if err != nil { + t.Fatalf("Failed to fetch temperature info for sensor type %v: %v", tt.sensorType, err) + } + + for _, sensor := range sensors { + name := sensor.GetName() + if got, want := gnmi.Get(t, dut, gnmi.OC().Component(name).Temperature().Instant().State()), sensor.GetMaxTemperature(); want != 0 && got > want { + t.Errorf("%v temperature threshold exceeded! got:%v, want:<=%v", name, got, want) + } + } + + }) + } +} + +// Health-indicator test. func TestSetPortHealthIndicator(t *testing.T) { defer testhelper.NewTearDownOptions(t).WithID("77865f9c-5919-467f-8be2-19a08d6803f9").Teardown(t) dut := ondatra.DUT(t, "DUT") @@ -272,3 +498,411 @@ func TestSetPortHealthIndicator(t *testing.T) { testhelper.AwaitHealthIndicator(t, dut, port, 5*time.Second, healthIndicator) } } + +// Storage device test. +func TestStorageDeviceInformation(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("b5db258b-2e3f-4880-96dc-db2ac452afe9").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + devices, err := testhelper.StorageDeviceInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch storage devices: %v", err) + } + + // Removable storage devices may not be present in the switch. This will cause + // dut.Telemetry().Component(name).Get() API to fail fatally. Instead, fetch the + // entire component subtree and validate storage device information. + components := gnmi.GetAll(t, dut, gnmi.OC().ComponentAny().State()) + + for _, device := range devices { + name := device.GetName() + + var info *oc.Component + for _, component := range components { + if component.GetName() == name { + info = component + break + } + } + if info == nil { + if device.GetIsRemovable() == false { + t.Errorf("%v information is missing in DUT", name) + } else { + t.Logf("Skipping verification for removable storage device %v since it is not present in DUT", name) + } + continue + } + t.Logf("Validating information for storage device: %v", name) + + if info.Name == nil { + t.Errorf("%v missing name leaf", name) + } else { + if got, want := info.GetName(), name; got != want { + t.Errorf("%v name match failed! got:%v, want:%v", name, got, want) + } + } + if info.Type == nil { + t.Errorf("%v missing type leaf", name) + } else { + if got, want := info.GetType(), oc.PlatformTypes_OPENCONFIG_HARDWARE_COMPONENT_STORAGE; got != want { + t.Errorf("%v type match failed! got:%v, want:%v", name, got, want) + } + } + if info.PartNo == nil { + t.Errorf("%v missing part-no leaf", name) + } else if info.GetPartNo() == "" { + t.Errorf("%v has empty part-no", name) + } + if info.SerialNo == nil { + t.Errorf("%v missing serial-no leaf", name) + } else if info.GetSerialNo() == "" { + t.Errorf("%v has empty serial-no", name) + } + + if info.Removable == nil { + t.Errorf("%v missing removable leaf", name) + } else { + if got, want := info.GetRemovable(), device.GetIsRemovable(); got != want { + t.Errorf("%v removable match failed! got:%v, want:%v", name, got, want) + } + } + + // Only check io-error information for non-removable storage devices. + if device.GetIsRemovable() { + continue + } + if got, want := testhelper.StorageIOErrors(t, dut, &device), device.GetIoErrorsThreshold(); got > want { + t.Errorf("%v io-errors threshold exceeded! got:%v, want:<=%v", name, got, want) + } + } +} + +// Storage device SMART info test. +func TestStorageDeviceSmartInformation(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("c5fe2192-9759-4829-9231-8fdb4ecc4245").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + devices, err := testhelper.StorageDeviceInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch storage devices: %v", err) + } + + // Removable storage devices may not be present in the switch. This will cause + // dut.Telemetry().Component(name).Get() API to fail fatally. Instead, fetch the + // entire component subtree and validate storage device information. + components := gnmi.GetAll(t, dut, gnmi.OC().ComponentAny().State()) + + for _, device := range devices { + // Only check SMART information for non-removable storage devices. + if device.GetIsRemovable() { + continue + } + + name := device.GetName() + + var info *oc.Component + for _, component := range components { + if component.GetName() == name { + info = component + break + } + } + if info == nil { + t.Errorf("%v information is missing in DUT", name) + continue + } + t.Logf("Validating SMART information for storage device: %v", name) + + smartDataInfo := device.GetSmartDataInfo() + { + got := testhelper.StorageWriteAmplificationFactor(t, dut, &device) + thresholds := smartDataInfo.GetWriteAmplificationFactorThresholds() + if !thresholds.IsValid(got) { + t.Errorf("%v write-amplification-factor thresholds not met! got:%v, thresholds:[%v]", + name, got, thresholds) + } + } + { + got := testhelper.StorageRawReadErrorRate(t, dut, &device) + thresholds := smartDataInfo.GetRawReadErrorRateThresholds() + if !thresholds.IsValid(got) { + t.Errorf("%v raw-read-error-rate thresholds not met! got:%v, thresholds:[%v]", + name, got, thresholds) + } + } + { + got := testhelper.StorageThroughputPerformance(t, dut, &device) + thresholds := smartDataInfo.GetThroughputPerformanceThresholds() + if !thresholds.IsValid(got) { + t.Errorf("%v throughput-performance thresholds not met! got:%v, thresholds:[%v]", + name, got, thresholds) + } + } + { + got := testhelper.StorageReallocatedSectorCount(t, dut, &device) + thresholds := smartDataInfo.GetReallocatedSectorCountThresholds() + if !thresholds.IsValid(got) { + t.Errorf("%v reallocated-sector-count thresholds not met! got:%v, thresholds:[%v]", + name, got, thresholds) + } + } + { + got := testhelper.StoragePowerOnSeconds(t, dut, &device) + thresholds := smartDataInfo.GetPowerOnSecondsThresholds() + if !thresholds.IsValid(got) { + t.Errorf("%v power-on-seconds thresholds not met! got:%v, thresholds:[%v]", + name, got, thresholds) + } + } + { + got := testhelper.StorageSsdLifeLeft(t, dut, &device) + thresholds := smartDataInfo.GetSsdLifeLeftThresholds() + if !thresholds.IsValid(got) { + t.Errorf("%v ssd-life-left thresholds not met! got:%v, thresholds:[%v]", + name, got, thresholds) + } + } + { + got := testhelper.StorageAvgEraseCount(t, dut, &device) + thresholds := smartDataInfo.GetAvgEraseCountThresholds() + if !thresholds.IsValid(got) { + t.Errorf("%v avg-erase-count thresholds not met! got:%v, thresholds:[%v]", + name, got, thresholds) + } + } + { + got := testhelper.StorageMaxEraseCount(t, dut, &device) + thresholds := smartDataInfo.GetMaxEraseCountThresholds() + if !thresholds.IsValid(got) { + t.Errorf("%v max-erase-count thresholds not met! got:%v, thresholds:[%v]", + name, got, thresholds) + } + } + } +} + + +// Fan tests. +func TestFanInformation(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("a394f0d4-61a9-45a8-a05a-c738fa4fa4b2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + fans, err := testhelper.FanInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch fan information: %v", err) + } + + for _, fan := range fans { + name := fan.GetName() + // Even though fan components might be removable, we expect all fans to be + // present in the switch (unlike storage devices). Hence, we are fetching + // fan component information instead of fetching the entire component subtree. + info := gnmi.Get(t, dut, gnmi.OC().Component(name).State()) + + if info.Type == nil { + t.Errorf("%v missing type leaf", name) + } else { + if got, want := info.GetType(), oc.PlatformTypes_OPENCONFIG_HARDWARE_COMPONENT_FAN; got != want { + t.Errorf("%v type match failed! got:%v, want:%v", name, got, want) + } + } + if info.Location == nil { + t.Errorf("%v missing location leaf", name) + } else { + if got, want := info.GetLocation(), fan.GetLocation(); got != want { + t.Errorf("%v location match failed! got:%v, want:%v", name, got, want) + } + } + if info.Parent == nil { + t.Errorf("%v missing parent leaf", name) + } else { + if got, want := info.GetParent(), fan.GetParent(); got != want { + t.Errorf("%v parent match failed! got:%v, want:%v", name, got, want) + } + } + if info.Removable == nil { + t.Errorf("%v missing removable leaf", name) + } + if got, want := info.GetRemovable(), fan.GetIsRemovable(); got != want { + t.Errorf("%v removable match failed! got:%v, want:%v", name, got, want) + } + if info.Empty == nil { + t.Errorf("%v missing Empty leaf", name) + } else { + if info.GetEmpty() { + t.Errorf("%v is unexpectedly empty.", name) + } + } + + // Only removable fans have FRU information. + if fan.GetIsRemovable() == false { + t.Logf("Not checking FRU information for %v since it is not removable", name) + continue + } + + if info.PartNo == nil { + t.Errorf("%v missing part-no leaf", name) + } else if info.GetPartNo() == "" { + t.Errorf("%v has empty part-no", name) + } + if info.SerialNo == nil { + t.Errorf("%v missing serial-no leaf", name) + } else if info.GetSerialNo() == "" { + t.Errorf("%v has empty serial-no", name) + } + + // Fetch mfg-date leaf separately since we want the test to fail in case + // of non-compliance errors with respect to the date format. Ondatra ignores + // non-compliance errors at sub-tree level Get() but fails the test if there + // is non-compliance at leaf level Get(). + if got := gnmi.Get(t, dut, gnmi.OC().Component(name).MfgDate().State()); got == "" { + t.Errorf("%v has empty mfg-date", name) + } + } + + fantrays, err := testhelper.FanTrayInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch fan information: %v", err) + } + + for _, fantray := range fantrays { + name := fantray.GetName() + // Likewise for fan trays, we expect all to be present regardless of whether they are removable. + info := gnmi.Get(t, dut, gnmi.OC().Component(name).State()) + if info.Type == nil { + t.Errorf("%v missing type leaf", name) + } + // } else { + // if got, want := info.GetType(), oc.PlatformTypes_OPENCONFIG_HARDWARE_COMPONENT_FANTRAY; got != want { + // t.Errorf("%v type match failed! got:%v, want:%v", name, got, want) + // } + // } + if info.Location == nil { + t.Errorf("%v missing location leaf", name) + } else { + if got, want := info.GetLocation(), fantray.GetLocation(); got != want { + t.Errorf("%v location match failed! got:%v, want:%v", name, got, want) + } + } + if info.Parent == nil { + t.Errorf("%v missing parent leaf", name) + } else { + if got, want := info.GetParent(), fantray.GetParent(); got != want { + t.Errorf("%v parent match failed! got:%v, want:%v", name, got, want) + } + } + if info.Removable == nil { + t.Errorf("%v missing removable leaf", name) + } + if got, want := info.GetRemovable(), fantray.GetIsRemovable(); got != want { + t.Errorf("%v removable match failed! got:%v, want:%v", name, got, want) + } + if info.Empty == nil { + t.Errorf("%v missing Empty leaf", name) + } else { + if info.GetEmpty() { + t.Errorf("%v is unexpectedly empty.", name) + } + } + } +} + +func TestFanSpeedInformation(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("804f6dbb-5480-4e1d-a215-e259530fa801").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + fans, err := testhelper.FanInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch fan information: %v", err) + } + + for _, fan := range fans { + name := fan.GetName() + info := gnmi.Get(t, dut, gnmi.OC().Component(name).Fan().State()) + + if info.Speed == nil { + t.Errorf("%v missing speed leaf", name) + } else { + if got, want := info.GetSpeed(), fan.GetMaxSpeed(); got > want { + t.Errorf("%v speed threshold exceeded! got:%v, want:<=%v", name, got, want) + } + } + { + if got := testhelper.FanSpeedControlPct(t, dut, &fan); got == 0 || got > 100 { + t.Errorf("%v speed-control-pct failed! got:%v, want:range(0,100]", name, got) + } + } + } +} + +func validatePcieInformation(info any) error { + if info == nil { + return errors.New("PCIe information is nil") + } + + var err error + var totalErrors uint64 + var individualErrors uint64 + rv := reflect.ValueOf(info) + rv = rv.Elem() + for i := 0; i < rv.NumField(); i++ { + name := rv.Type().Field(i).Name + field := rv.Field(i) + if field.IsNil() { + err = testhelper.WrapError(err, "%v leaf is nil", name) + continue + } + field = field.Elem() + if got, want := field.Kind(), reflect.Uint64; got != want { + err = testhelper.WrapError(err, "%v leaf has invalid value type! got:%v, want:%v", name, got, want) + continue + } + + value := field.Uint() + if name == "TotalErrors" { + totalErrors = value + } else { + individualErrors += value + } + } + + if totalErrors > individualErrors { + err = testhelper.WrapError(err, "total-errors:%v should be <= cumulative-individual-errors:%v", totalErrors, individualErrors) + } else if totalErrors == 0 && individualErrors > 0 { + err = testhelper.WrapError(err, "total-errors count cannot be 0 if individual errors are detected (count:%v)", individualErrors) + } + + return err +} + +func TestPcieInformation(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("82e1ef7b-46db-4523-b0e5-f94a2e0a8a12").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + devices, err := testhelper.PcieInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch PCIe information: %v", err) + } + + for _, device := range devices { + name := device.GetName() + + c := gnmi.Get(t, dut, gnmi.OC().Component(name).Pcie().CorrectableErrors().State()) + if err := validatePcieInformation(c); err != nil { + t.Errorf("Correctable error information validation failed for device:%v\n%v", name, err) + } + + f := gnmi.Get(t, dut, gnmi.OC().Component(name).Pcie().FatalErrors().State()) + if err := validatePcieInformation(f); err != nil { + t.Errorf("Fatal error information validation failed for device:%v\n%v", name, err) + } + if f != nil && f.GetTotalErrors() != 0 { + t.Errorf("%v fatal errors detected on %v", f.GetTotalErrors(), dut.Name()) + } + + n := gnmi.Get(t, dut, gnmi.OC().Component(name).Pcie().NonFatalErrors().State()) + if err := validatePcieInformation(n); err != nil { + t.Errorf("Non-fatal error information validation failed for device:%v\n%v", name, err) + } + } +} diff --git a/sdn_tests/pins_ondatra/tests/platforms_software_component_test.go b/sdn_tests/pins_ondatra/tests/platforms_software_component_test.go index eaa2f9bb99..cd6f4f939b 100644 --- a/sdn_tests/pins_ondatra/tests/platforms_software_component_test.go +++ b/sdn_tests/pins_ondatra/tests/platforms_software_component_test.go @@ -269,8 +269,8 @@ func TestGetChassisDefaultInformation(t *testing.T) { t.Errorf("Chassis component part-no length validation failed! got:%v, want:%v(atmost)", len(partNo), maxLength) } - if platform := testhelper.ComponentChassisPlatform(t, dut, name); platform != "BRIXIA" { - t.Errorf("Chassis component platform match failed! got:%v, want:BRIXIA", platform) + if platform := testhelper.ComponentChassisPlatform(t, dut, name); platform != "experimental" { + t.Errorf("Chassis component platform match failed! got:%v, want:experimental", platform) } serialNo := gnmi.Get(t, dut, componentPath.SerialNo().State()) @@ -296,9 +296,9 @@ func TestSetChassisNamePaths(t *testing.T) { componentPath := gnmi.OC().Component(key) testStrings := []string{ - "ju09u1m1.sqs99.net.google.com", - "df50f001.mtv16.net.google.com", - "mn120ab012.xyz16.prod.google.com", + "abc.s1.test.com", + "def.s2.test.com", + "xyz.xyz16.test.com", } for _, fqdn := range testStrings { @@ -357,7 +357,7 @@ func TestChassisInfoPersistsAfterReboot(t *testing.T) { key := "chassis" // Configure fully-qualified-name on the chassis. - fqdn := "mn120ab012.xyz16.prod.google.com" + fqdn := "xy120ab012.xyz.test.com" testhelper.ReplaceFullyQualifiedName(t, dut, key, fqdn) testhelper.AwaitFullyQualifiedName(t, dut, key, 5*time.Second, fqdn) diff --git a/sdn_tests/pins_ondatra/tests/port_debug_data_test.go b/sdn_tests/pins_ondatra/tests/port_debug_data_test.go index 554a81368c..2fc368b8cc 100644 --- a/sdn_tests/pins_ondatra/tests/port_debug_data_test.go +++ b/sdn_tests/pins_ondatra/tests/port_debug_data_test.go @@ -17,7 +17,8 @@ func TestMain(m *testing.M) { func TestGetPortDebugDataInvalidInterface(t *testing.T) { defer testhelper.NewTearDownOptions(t).WithID("dba77fa7-b0d1-4412-8136-22dea24ed935").Teardown(t) var intfName = "Ethernet99999" - if _, err := testhelper.HealthzGetPortDebugData(t, ondatra.DUT(t, "DUT"), intfName); err == nil { + err := testhelper.HealthzGetPortDebugData(t, ondatra.DUT(t, "DUT"), intfName); + if err == nil { t.Fatalf("Expected RPC failure due to invalid interface %v", intfName) } } @@ -39,24 +40,11 @@ func TestGetPortDebugDataWithTranscevierInserted(t *testing.T) { } t.Logf("Get port debug data from interface %v on xcvr present port %v", intfName, xcvrName) - data, err := testhelper.HealthzGetPortDebugData(t, dut, intfName) + err := testhelper.HealthzGetPortDebugData(t, dut, intfName) if err != nil { t.Fatalf("Expected RPC success, got error %v", err) } - if data.GetPhyData() == "" { - t.Errorf("Got empty phy_data from PortDebugData for intfName %v", intfName) - } - - if len(data.GetTransceiverEepromPages()) == 0 { - t.Errorf("Got empty transceiver_eeprom_pages from PortDebugData for intfName %v", intfName) - } - - for _, eepromPage := range data.GetTransceiverEepromPages() { - if len(eepromPage.GetEepromContent()) == 0 { - t.Errorf("Got empty eeprom_content on page %v from PortDebugData for intfName %v", eepromPage.GetPageNum(), intfName) - } - } } } @@ -77,17 +65,10 @@ func TestGetPortDebugDataWithoutTranscevierInserted(t *testing.T) { } t.Logf("Get port debug data from interface %v on xcvr empty port %v", intfName, xcvrName) - data, err := testhelper.HealthzGetPortDebugData(t, dut, intfName) + err := testhelper.HealthzGetPortDebugData(t, dut, intfName) if err != nil { t.Fatalf("Expected RPC success, got error %v", err) } - if data.GetPhyData() == "" { - t.Errorf("Got empty phy_data from PortDebugData for intfName %v", intfName) - } - - if len(data.GetTransceiverEepromPages()) != 0 { - t.Errorf("Got non-empty transceiver_eeprom_pages from PortDebugData for intfName %v", intfName) - } } } diff --git a/sdn_tests/pins_ondatra/tests/system_paths_test.go b/sdn_tests/pins_ondatra/tests/system_paths_test.go new file mode 100644 index 0000000000..7a9338a7b1 --- /dev/null +++ b/sdn_tests/pins_ondatra/tests/system_paths_test.go @@ -0,0 +1,804 @@ +package system_paths_test + +import ( + "fmt" + "math/rand" + "strings" + "testing" + "time" + + log "github.com/golang/glog" + "github.com/google/go-cmp/cmp" + "github.com/openconfig/ondatra" + "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/binding/pinsbind" + "github.com/sonic-net/sonic-mgmt/sdn_tests/pins_ondatra/infrastructure/testhelper/testhelper" + "github.com/pkg/errors" + + syspb "github.com/openconfig/gnoi/system" + "github.com/openconfig/ondatra/gnmi" + "github.com/openconfig/ondatra/gnmi/oc" +) + +func TestMain(m *testing.M) { + ondatra.RunTests(m, pinsbind.New) +} + +func verifyAddress(address string, addresses []string) error { + for _, addr := range addresses { + if addr == address { + return nil + } + } + return errors.New("unknown address") +} + +func TestGetRemoteServerAddressInfo(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("c2873412-1016-4c89-9e59-79fcfec642bb").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + logInfo, err := testhelper.LoggingServerAddressesForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch remote server logging info: %v", err) + } + + // Collect remote server addresses. + foundAddresses := gnmi.GetAll(t, dut, gnmi.OC().System().Logging().RemoteServerAny().Host().State()) + + // Determine if configured addresses are IPv4 or IPv6. We are only allowed to have one or the other. + hasIpv4, hasIpv6 := false, false + for _, addr := range foundAddresses { + if err := verifyAddress(addr, logInfo.IPv4RemoteAddresses); err == nil { + hasIpv4 = true + } + if err := verifyAddress(addr, logInfo.IPv6RemoteAddresses); err == nil { + hasIpv6 = true + } + } + + if !hasIpv4 && !hasIpv6 { + t.Fatalf("Remote server addresses do not match device logging server addresses: got: %v vs want: %v or want: %v ", strings.Join(foundAddresses, ", "), strings.Join(logInfo.IPv4RemoteAddresses, ", "), strings.Join(logInfo.IPv6RemoteAddresses, ", ")) + } + if hasIpv4 && hasIpv6 { + t.Fatalf("Remote server addresses are not expected to mix IPv4 and IPv6 addresses: got: %v", strings.Join(foundAddresses, ", ")) + } + + addresses := logInfo.IPv4RemoteAddresses + if hasIpv6 { + addresses = logInfo.IPv6RemoteAddresses + } + + // Addresses configured may only be what device configuration allows. + if foundLen, addressLen := len(foundAddresses), len(addresses); foundLen != addressLen { + t.Errorf("Unexpected number of remote logging server addresses: %v (want %v).", foundLen, addressLen) + } + + addressSet := make(map[string]bool) + for _, addr := range foundAddresses { + addressSet[addr] = true + } + // Addresses may not be repeated. + if setLen, foundLen := len(addressSet), len(foundAddresses); setLen != foundLen { + t.Errorf("Remote logging addresses are not unique: %v", foundAddresses) + } + + // Addresses configured may only be what device configuration allows. + for _, addr := range foundAddresses { + if err := verifyAddress(addr, addresses); err != nil { + t.Errorf("Remote logging address is unsupported: %v", addr) + } + } + + // Check that state host value matches the rest of the path. + for _, addr := range foundAddresses { + if readAddress := gnmi.Get(t, dut, gnmi.OC().System().Logging().RemoteServer(addr).Host().State()); readAddress != addr { + t.Errorf("Remote logging host address does not match path: %v vs %v", readAddress, addr) + } + } +} + +func TestGetCurrentDateAndTime(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("8ec03425-b9ab-4e13-8b01-1564b5043d68").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + t1 := time.Now() + time.Sleep(1 * time.Second) + dutTime, err := time.Parse(time.RFC3339, gnmi.Get(t, dut, gnmi.OC().System().CurrentDatetime().State())) + if err != nil { + t.Fatalf("Failed to parse DUT time: %v", err) + } + t2 := time.Now() + + // Time reported by DUT should be between the time the request was sent and received. + if dutTime.Before(t1) || dutTime.After(t2) { + t.Errorf("Time comparison failed! got:%v, want:(greater than:%v, less than:%v)", dutTime, t1, t2) + } +} + +func TestGetBootTime(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("c2bcb460-e79a-4ae2-9a74-d1b3d6ec62ae").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // boot-time should be the same before rebooting switch. We give a 1 second buffer to account for + // jitter in boot-time calculation. + want := gnmi.Get(t, dut, gnmi.OC().System().BootTime().State()) + time.Sleep(5 * time.Second) + sec := uint64(time.Second.Nanoseconds()) + if got := gnmi.Get(t, dut, gnmi.OC().System().BootTime().State()); got < want-sec || got > want+sec { + t.Errorf("boot-time comparison before reboot failed! got:%v, want:%v(+-1s)", got, want) + } + + waitTime, err := testhelper.RebootTimeForDevice(t, dut) + if err != nil { + t.Fatalf("Unable to get reboot wait time: %v", err) + } + params := testhelper.NewRebootParams().WithWaitTime(waitTime).WithCheckInterval(30 * time.Second).WithRequest(syspb.RebootMethod_COLD) + if err := testhelper.Reboot(t, dut, params); err != nil { + t.Fatalf("Failed to reboot DUT: %v", err) + } + + // boot-time should be later than the previous boot-time after rebooting switch. + if got := gnmi.Get(t, dut, gnmi.OC().System().BootTime().State()); got <= want { + t.Errorf("boot-time comparison after reboot failed! got:%v, want:(greater than)%v", got, want) + } +} + +func TestGetHostname(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("01c119ae-2550-4949-8fd7-3605b8d2981c").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + hostname := gnmi.Get(t, dut, gnmi.OC().System().Hostname().State()) + if len(hostname) == 0 || len(hostname) > 253 { + t.Errorf("Invalid hostname length! got:%v, want:(0-253)", len(hostname)) + } + if hostname != dut.Name() { + t.Errorf("Hostname match failed! got:%v, want:%v", hostname, dut.Name()) + } +} + +func TestConfigMetaData(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("366f4520-79f7-49ac-a67d-c53b48b11535").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Perform an initial config push on config-meta-data path. + // TODO: Remove this step when default config push is available. + testhelper.ReplaceConfigMetaData(t, dut, "initial metadata") + + origMetaData := testhelper.SystemConfigMetaData(t, dut) + if len(origMetaData) == 0 { + t.Error("Invalid initial metadata length! got:0, want:(greater than) 0") + } + // Configure a different value of at config-meta-data path. + newMetaData := "test1" + if newMetaData == origMetaData { + newMetaData = "test2" + } + + testhelper.ReplaceConfigMetaData(t, dut, newMetaData) + + if got, want := testhelper.SystemConfigMetaData(t, dut), newMetaData; got != want { + t.Errorf("Invalid value for config-meta-data state path! got:%v, want:%v", got, want) + } + if got, want := testhelper.SystemConfigMetaDataFromConfig(t, dut), newMetaData; got != want { + t.Errorf("Invalid value for config-meta-data config path! got:%v, want:%v", got, want) + } +} + +func TestCPUIndexes(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("093c4411-c748-4b7c-bee7-fd73b8c2a473").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + cpuInfo, err := testhelper.CPUInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch CPU information: %v", err) + } + + // Convert index in expected CPU information to System_Cpu_Index_Union type since this + // type will be returned in the GET response by the switch. + wantIndexes := make(map[oc.System_Cpu_Index_Union]bool) + for _, cpu := range cpuInfo { + index, err := (&oc.System_Cpu{}).To_System_Cpu_Index_Union(cpu.GetIndex()) + if err != nil { + t.Fatalf("To_System_Cpu_Index_Union() failed for index:%v (%v)", cpu.GetIndex(), err) + } + wantIndexes[index] = true + } + + gotIndexes := make(map[oc.System_Cpu_Index_Union]bool) + for i, info := range gnmi.GetAll(t, dut, gnmi.OC().System().CpuAny().State()) { + if info.Index == nil { + t.Errorf("CPU index not present in information iteration %v", i) + continue + } + gotIndexes[info.GetIndex()] = true + } + + if !cmp.Equal(wantIndexes, gotIndexes) { + t.Errorf("CPU index match failed! (-want +got):%v", cmp.Diff(wantIndexes, gotIndexes)) + } +} + +func TestCPUUsage(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("4806f97b-1c4e-4763-a9e3-58671bda144a").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + cpuInfo, err := testhelper.CPUInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch CPU information: %v", err) + } + + wantUsage := make(map[oc.System_Cpu_Index_Union]uint8) + for _, cpu := range cpuInfo { + index, err := (&oc.System_Cpu{}).To_System_Cpu_Index_Union(cpu.GetIndex()) + if err != nil { + t.Fatalf("To_System_Cpu_Index_Union() failed for index:%v (%v)", cpu.GetIndex(), err) + } + wantUsage[index] = cpu.GetMaxAverageUsage() + } + + gotInfo := gnmi.GetAll(t, dut, gnmi.OC().System().CpuAny().State()) + if len(gotInfo) != len(wantUsage) { + t.Errorf("Invalid number of CPU indexes received from switch! got:%v, want:%v", len(gotInfo), len(wantUsage)) + } + + // Fetch the average utilization for each CPU for 2 minutes. In each iteration, validate + // that the utilization is less than the specified threshold. Also, store the cumulative + // utilization for each CPU. At the end of 2 minutes, validate that the cumulative + // utilization for each CPU is non-zero since it is highly unlikely that a CPU is not + // being utilized during this entire time interval. + waitTime := 2 * time.Minute + interval := 10 * time.Second + cumulativeUsage := make(map[oc.System_Cpu_Index_Union]int) + log.Infof("Fetching average CPU utilization in %v intervals for %v", interval, waitTime) + for timeout := time.Now().Add(waitTime); time.Now().Before(timeout); { + log.Info("========== CPU average usage stats ==========") + for i, info := range gotInfo { + if info.Index == nil { + t.Errorf("CPU index not present in information iteration %v", i) + continue + } + + index := info.GetIndex() + if _, ok := wantUsage[index]; !ok { + t.Errorf("Invalid index:%v received from DUT", index) + continue + } + + got := gnmi.Get(t, dut, gnmi.OC().System().Cpu(index).Total().Avg().State()) + if wantUsage[index] != 0 && got > wantUsage[index] { + t.Errorf("CPU (index:%v) average usage validation failed! got:%v, want:<%v", index, got, wantUsage[index]) + } + log.Infof("CPU (index:%v): %v", index, got) + cumulativeUsage[index] += int(got) + } + + time.Sleep(interval) + } + + for i, u := range cumulativeUsage { + if u == 0 { + t.Errorf("CPU (index:%v) cumulative average got:0, want:>0", i) + } + } + +} + +func TestCPUInterval(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("534f595e-06b6-434c-b7cb-20d856efacdb").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + cpuInfo, err := testhelper.CPUInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch CPU information: %v", err) + } + + wantIndexes := make(map[oc.System_Cpu_Index_Union]bool) + for _, cpu := range cpuInfo { + index, err := (&oc.System_Cpu{}).To_System_Cpu_Index_Union(cpu.GetIndex()) + if err != nil { + t.Fatalf("To_System_Cpu_Index_Union() failed for index:%v (%v)", cpu.GetIndex(), err) + } + wantIndexes[index] = true + } + + gotInfo := gnmi.GetAll(t, dut, gnmi.OC().System().CpuAny().State()) + for i, info := range gotInfo { + if info.Index == nil { + t.Errorf("CPU index not present in information iteration %v", i) + continue + } + + index := info.GetIndex() + if _, ok := wantIndexes[index]; !ok { + t.Errorf("Invalid index:%v received from DUT", index) + continue + } + if got := gnmi.Get(t, dut, gnmi.OC().System().Cpu(index).Total().Interval().State()); got == 0 { + t.Errorf("CPU (index:%v) interval validation failed! got:%v, want:>0", index, got) + } + } + + if len(gotInfo) != len(wantIndexes) { + t.Errorf("Invalid number of CPU indexes received from switch! got:%v, want:%v", len(gotInfo), len(wantIndexes)) + } +} + +// This method performs validations on process information leafs. +// It returns whether the validations need to be retried along with the error +// encountered while performing the validations. +// The condition for retry is that there is a missing leaf for the PID. If the +// leaf validation itself fails, the API returns retry = false along with the +// errors encountered. +func validateProcessInformation(procInfo *oc.System_Process, bootTime uint64, systemMemory uint64) (bool, error) { + var err error + infoMissing := false + validationFailed := false + pid := procInfo.GetPid() + + processString := func() string { + name := procInfo.GetName() + if name == "" { + name = "" + } + return fmt.Sprintf("%s (pid:%d)", name, pid) + } + + if procInfo.CpuUtilization == nil { + infoMissing = true + err = testhelper.WrapError(err, "Invalid cpu-utilization for %v! got:, want:range(0-100)", processString()) + } else if got := procInfo.GetCpuUtilization(); got > 100 { + // Not checking for UMF default value since actual CPU utilization + // of the process can be 0. + validationFailed = true + err = testhelper.WrapError(err, "Invalid cpu-utilization for %v! got:%v, want:range(0-100)", processString()) + } + + if procInfo.MemoryUsage == nil { + infoMissing = true + err = testhelper.WrapError(err, "Invalid memory-usage for %v! got:, want:(<=)%v", processString(), systemMemory) + } else if got := procInfo.GetMemoryUsage(); got > systemMemory { + // Not checking for UMF default value since actual memory usage + // of the process can be 0. + validationFailed = true + err = testhelper.WrapError(err, "Invalid memory-usage for %v! got:%v, want:(<=)%v", processString(), got, systemMemory) + } + + if procInfo.StartTime == nil { + infoMissing = true + err = testhelper.WrapError(err, "Invalid start-time for %v! got:, want:(>=)%v", processString(), bootTime) + } else { + got := procInfo.GetStartTime() + if got == 0 { + // UMF sends 0 by default. start-time of a process cannot be 0. + // This indicates missing DB information. + infoMissing = true + err = testhelper.WrapError(err, "Invalid start-time for %v! got:%v, want:(>=)%v", processString(), got, bootTime) + } else if got < bootTime { + validationFailed = true + err = testhelper.WrapError(err, "Invalid start-time for %v! got:%v, want:(>=)%v", processString(), got, bootTime) + } + } + + if procInfo.Name == nil { + infoMissing = true + err = testhelper.WrapError(err, "Invalid name for pid:%v! got:, want:", pid) + } else if procInfo.GetName() == "" { + // UMF sends empty name by default. name of a process cannot be empty. + // This indicates missing DB information. + infoMissing = true + err = testhelper.WrapError(err, "Invalid name for pid:%v! got:, want:", pid) + } + + // CPU usage time might not be present since the process might be ephemeral. In such cases, + // only log that the information is not present. + if procInfo.CpuUsageUser == nil { + log.Infof("cpu-usage-user not reported for %v", processString()) + } + if procInfo.CpuUsageSystem == nil { + log.Infof("cpu-usage-system not reported for %v", processString()) + } + + if validationFailed { + // Don't retry if validation failed for a particular leaf. + return false, err + } + + // None of the validations failed, so retry if information for a leaf + // is missing. + return infoMissing, err +} + +func TestMemoryStatistics(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("d2f4917b-3813-4e81-b195-8f6b9222d615").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + expectedInfo, err := testhelper.MemoryInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch memory information for device: %v", err) + } + info := gnmi.Get(t, dut, gnmi.OC().System().Memory().State()) + + if info.Physical == nil { + t.Error("Physical memory information not received from DUT") + } else { + // Physical memory value returned by the switch might not be an exact match. + // Provide 1GB error margin. + errMargin := uint64(1073741824) + if got, want := info.GetPhysical(), expectedInfo.GetPhysical(); got > want || want-got > errMargin { + t.Errorf("Physical memory validation failed! got:%v, want:%v (error margin: -%v)", got, want, errMargin) + } + } + + if info.Free == nil { + t.Error("Free memory information not received from DUT") + } else { + if got, want := info.GetFree(), info.GetPhysical(); got > want { + t.Errorf("Free memory (%v) more than physical memory (%v)", got, want) + } + if expectedInfo.GetFreeThreshold() != 0 { + // Free memory threshold specified for the device. + if got, want := info.GetFree(), expectedInfo.GetFreeThreshold(); got < want { + t.Errorf("Free memory threshold validation failed! got:%v, want:>=%v", got, want) + } + } + } + + if info.Used == nil { + t.Error("Used memory information not received from DUT") + } else { + if got, want := info.GetUsed(), info.GetPhysical(); got > want { + t.Errorf("Used memory (%v) more than physical memory (%v)", got, want) + } + if expectedInfo.GetUsedThreshold() != 0 { + // Used memory threshold specified for the device. + if got, want := info.GetUsed(), expectedInfo.GetUsedThreshold(); got > want { + t.Errorf("Used memory threshold validation failed! got:%v, want:<=%v", got, want) + } + } + } +} + +func TestMemoryErrorStatistics(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("6300ee99-ac15-4913-a8a8-9231bb92a498").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + expectedInfo, err := testhelper.MemoryInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch memory information for device: %v", err) + } + info := gnmi.Get(t, dut, gnmi.OC().System().Memory().Counters().State()) + + if info.CorrectableEccErrors == nil { + t.Errorf("correctable-ecc-errors information not received from DUT") + } else { + if got, want := info.GetCorrectableEccErrors(), expectedInfo.GetCorrectableEccErrorThreshold(); want != 0 && got > want { + t.Errorf("correctable-ecc-errors threshold exceeded! got:%v, want:<=%v", got, want) + } + } + + if info.UncorrectableEccErrors == nil { + t.Errorf("uncorrectable-ecc-errors information not received from DUT") + } else { + if got := info.GetUncorrectableEccErrors(); got != 0 { + t.Errorf("uncorrectable-ecc-errors detected on the DUT! got:%v, want:0", got) + } + } +} + +func TestNTPServerInformation(t *testing.T) { + dut := ondatra.DUT(t, "DUT") + + tests := []struct { + name string + uuid string + checkAllInformation bool + }{ + { + name: "TestServerAddress", + uuid: "a0cd293a-0a26-4b2a-bf8f-3819e885ed1a", + }, + { + name: "TestServerInfo", + uuid: "0b83e93f-5d85-4100-ba81-a5f1af058763", + checkAllInformation: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + expectedInfo, err := testhelper.NTPServerInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch NTP server information for device: %v", err) + } + + ntp := gnmi.Get(t, dut, gnmi.OC().System().Ntp().State()) + if ntp == nil { + t.Fatalf("No NTP information received from DUT") + } + serverInfo := ntp.Server + + // Each NTP server IP address is returned separately from the DUT. + // Therefore, all expected IP addresses need to be aggregated before + // comparing with the received information. + expectedServerNum := 0 + for _, info := range expectedInfo { + expectedServerNum += len(info.GetIPv4Address()) + len(info.GetIPv6Address()) + } + + if got, want := len(serverInfo), expectedServerNum; got != want { + t.Errorf("Invalid number of NTP servers! got:%v, want:%v", got, want) + } + + for _, info := range expectedInfo { + // For each expected NTP server, fetch the IPv4 and IPv6 address from the + // gNMI response and perform validations. + ipv4Reachable := false + ipv6Reachable := false + expectedAddresses := []struct { + addresses []string + isIPv4 bool + }{ + { + addresses: info.GetIPv4Address(), + isIPv4: true, + }, + { + addresses: info.GetIPv6Address(), + }, + } + for _, e := range expectedAddresses { + for _, address := range e.addresses { + // Ensure that the server address is present in the gNMI response. + log.Infof("Validating NTP server: %s", address) + if _, ok := serverInfo[address]; !ok { + t.Errorf("%v NTP server not reported by DUT", address) + continue + } + server := serverInfo[address] + if got, want := server.GetAddress(), address; got != want { + t.Errorf("%v NTP server has invalid address field! got:%v, want:%v", address, got, want) + } + + // Only perform additional checks if checkAllInformation is set. + if !tt.checkAllInformation { + continue + } + + // Do not perform value checks for unreachable servers. + // Only ensure that fields are present in the response. + checkValue := false + + if server.Stratum == nil { + t.Errorf("%v NTP server doesn't have stratum information", address) + } else if checkValue { + if got, want := server.GetStratum(), info.GetStratumThreshold(); want != 0 && got > want { + t.Errorf("%v NTP server has invalid stratum field! got:%v, want:<=%v", address, got, want) + } + } + + if server.RootDelay == nil { + t.Errorf("%v NTP server doesn't have root-delay information", address) + } else if checkValue { + if got := server.GetRootDelay(); got == 0 { + t.Errorf("%v NTP server has invalid root-delay field! got:%v, want:>0", address, got) + } + } + + if server.PollInterval == nil { + t.Errorf("%v NTP server doesn't have poll-interval information", address) + } else { + // Poll interval value should always be checked since the NTP + // client must poll the server at non-zero intervals. + if got := server.GetPollInterval(); got == 0 { + t.Errorf("%v NTP server has invalid poll-interval field! got:%v, want:>0", address, got) + } + } + + if server.RootDispersion == nil { + t.Errorf("%v NTP server doesn't have root-dispersion information", address) + } else if checkValue { + if got := server.GetRootDispersion(); got == 0 { + t.Errorf("%v NTP server has invalid root-dispersion field! got:%v, want:>0", address, got) + } + } + } + } + + // Server should be reachable via an IPv4 or IPv6 address. + if tt.checkAllInformation && !ipv4Reachable && !ipv6Reachable { + ipAddresses := append(info.GetIPv4Address(), info.GetIPv6Address()...) + t.Errorf("NTP server with IP addresses: %v is not reachable", strings.Join(ipAddresses, ",")) + } + } + }) + } +} + +func TestMountPointsInformation(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("c0cf81e1-e99d-4b57-b273-9fe87c713881").Teardown(t) + + dut := ondatra.DUT(t, "DUT") + + mountPoints, err := testhelper.MountPointsInfoForDevice(t, dut) + if err != nil { + t.Fatalf("Failed to fetch mount points information for device: %v", err) + } + + // Validate that DUT returns at least the required number of mount points. + if got := gnmi.GetAll(t, dut, gnmi.OC().System().MountPointAny().State()); len(got) < len(mountPoints) { + t.Errorf("Invalid number of mount points! got:%v, want:>=%v", len(got), len(mountPoints)) + } + + for _, mp := range mountPoints { + name := mp.GetName() + info := gnmi.Get(t, dut, gnmi.OC().System().MountPoint(name).State()) + + if info.Size == nil { + t.Errorf("%v missing size leaf", name) + } else if info.GetSize() == 0 { + t.Errorf("%v has invalid size! got:0, want:>0", name) + } + + if info.Available == nil { + t.Errorf("%v missing available leaf", name) + } + + if info.Size != nil && info.Available != nil { + if size, available := info.GetSize(), info.GetAvailable(); available > size { + t.Errorf("available space:%v exceeds size:%v for mount point %v", available, size, name) + } + } + } +} + +func printProcessStatistics(t *testing.T, stats map[uint64]oc.System_Process) { + logString := "\n***************************************\n" + logString += "\tProcess Statistics" + logString += "\n***************************************\n" + for pid, info := range stats { + logString += fmt.Sprintf("Process: %s\n", info.GetName()) + logString += fmt.Sprintf("PID: %d\n", pid) + startTime := info.GetStartTime() + // Nanoseconds to seconds. + divider := uint64(1000000000) + logString += fmt.Sprintf("Start Time: %d (%s)\n", startTime, time.Unix(int64(startTime/divider), int64(startTime%divider))) + m := info.GetMemoryUsage() / 1000000 + suffix := "MB" + if m > 1000 { + m = m / 1000 + suffix = "GB" + } + logString += fmt.Sprintf("Memory Usage: %d bytes (%d %s)\n", info.GetMemoryUsage(), m, suffix) + logString += fmt.Sprintf("Memory Utilization: %d%%\n", info.GetMemoryUtilization()) + logString += fmt.Sprintf("CPU Utilization: %d%%\n", info.GetCpuUtilization()) + logString += "--------------------------------------------------------------------\n" + } + t.Log(logString) +} + +// This is a GPINs-specific test since it makes assumptions about the +// functionality of systemstatsd back-end. +func TestProcessStatistics(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("dc958005-d45b-429d-9ee1-c14cc1eefcf2").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + info := gnmi.GetAll(t, dut, gnmi.OC().System().ProcessAny().State()) + if len(info) == 0 { + t.Fatalf("Invalid number of PIDs! got:0, want:>=1") + } + bootTime := gnmi.Get(t, dut, gnmi.OC().System().BootTime().State()) + systemMemory := gnmi.Get(t, dut, gnmi.OC().System().Memory().Physical().State()) + + // Systemstatsd updates process attributes one-by-one in the DB in the + // following order: + // cpu-utilization, memory-usage, start-time, name. + // If any of the above attributes are empty or not present, retry fetching + // information for that process. If that also fails, only then declare the + // test as failure. + var retryPID []uint64 + stats := make(map[uint64]oc.System_Process) + for _, procInfo := range info { + pid := procInfo.GetPid() + if pid == 0 { + t.Errorf("Invalid PID value! got:0, want:>=1") + continue + } + stats[pid] = *procInfo + + retry, err := validateProcessInformation(procInfo, bootTime, systemMemory) + if retry { + log.Infof("Adding pid:%v to retry list. Failures seen:\n %v", pid, err) + retryPID = append(retryPID, pid) + } else if err != nil { + // At least one validation failed. + t.Error(err) + } + } + + time.Sleep(1 * time.Second) + for _, pid := range retryPID { + log.Infof("Retrying information validation for pid:%v", pid) + procInfo := gnmi.Get(t, dut, gnmi.OC().System().Process(pid).State()) + if _, err := validateProcessInformation(procInfo, bootTime, systemMemory); err != nil { + t.Errorf("Validation failed for pid:%v\n %v", pid, err) + } + stats[pid] = *procInfo + } + + printProcessStatistics(t, stats) +} + +func generateLabel(existingLabelsSubtree []*testhelper.System_FeatureLabel) (label uint32, ok bool) { + currLabels := map[uint32]bool{} + for _, val := range existingLabelsSubtree { + currLabels[val.GetLabel()] = true + } + // Loop (for a maximum of 100 times) until a label is generated that is not an existing label on + // the switch. Exit if unable to generate a suitable label. + rand.Seed(time.Now().UnixNano()) + for i := 0; i < 100; i++ { + label = rand.Uint32() + if !currLabels[label] { + return label, true + } + } + return label, false +} + +func TestFeatureLabels(t *testing.T) { + defer testhelper.NewTearDownOptions(t).WithID("b5c1a559-21b6-4fa0-b5ff-1f15080b7b0f").Teardown(t) + dut := ondatra.DUT(t, "DUT") + + // Get the existing feature-labels tree. + existingLabelsSubtree := testhelper.SystemFeatureLabels(t, dut) + + // Generate a unique label not already configured on the switch. + label, labelGenerated := generateLabel(existingLabelsSubtree) + if !labelGenerated { + t.Fatalf("Couldn't generate a new label from feature-labels: %v", existingLabelsSubtree) + } + + featureLabel := testhelper.CreateFeatureLabel(label) + gnmi.Replace(t, dut, testhelper.SystemFeatureLabelPath(gnmi.OC().System(), label).Config(), featureLabel) + testhelper.AwaitSystemFeatureLabel(t, dut, 5*time.Second, featureLabel) + + defer func() { + // Remove the configured feature-label after the test. + gnmi.Delete(t, dut, testhelper.SystemFeatureLabelPath(gnmi.OC().System(), label).Config()) + labelsSubtree := testhelper.SystemFeatureLabels(t, dut) + if len(labelsSubtree) != len(existingLabelsSubtree) { + t.Errorf("Incorrect number of feature-labels found after FeatureLabel(%v).Delete; got:%v, want:%v", label, len(labelsSubtree), len(existingLabelsSubtree)) + } + for _, val := range labelsSubtree { + if val.GetLabel() == label { + t.Errorf("Path did not get deleted; got:%v, want:", label) + } + } + }() + + // Get the new feature-labels tree after Set Replace. + newLabelsSubtree := testhelper.SystemFeatureLabels(t, dut) + if got, want := len(newLabelsSubtree), len(existingLabelsSubtree)+1; got != want { + t.Fatalf("Incorrect number of feature-labels found after FeatureLabel(%v).Replace; got:%v, want:%v", label, got, want) + } + + // Verify that the new feature-label is present in the feature-labels subtree. + labelFound := false + for _, val := range newLabelsSubtree { + if val.GetLabel() == label { + labelFound = true + break + } + } + if !labelFound { + t.Fatalf("Couldn't find configured label: %v in feature-labels: %v", label, newLabelsSubtree) + } + + // Verify the GET response for the feature-label state and config leaf paths. + if got, want := testhelper.SystemFeatureLabel(t, dut, label).GetLabel(), label; got != want { + t.Errorf("gnmi.Get(t, dut, gnmi.OC().System().FeatureLabel(label).State()).GetLabel() got:%v, want:%v", got, want) + } + if got, want := testhelper.SystemFeatureLabelFromConfig(t, dut, label).GetLabel(), label; got != want { + t.Errorf("gnmi.GetConfig(t, dut, gnmi.OC().System().FeatureLabel(label).Config()).GetLabel() got:%v, want:%v", got, want) + } +} diff --git a/sdn_tests/pins_ondatra/tests/transceiver_test.go b/sdn_tests/pins_ondatra/tests/transceiver_test.go index 6bcf770a7b..54bdd83b6a 100644 --- a/sdn_tests/pins_ondatra/tests/transceiver_test.go +++ b/sdn_tests/pins_ondatra/tests/transceiver_test.go @@ -77,7 +77,6 @@ func FindPresentOpticalTransceiver(t *testing.T, dut *ondatra.DUTDevice) (string // Check if a transceiver is optical by getting cable-length, // which should only be positive only for copper tranceivers. // If cable-length is 0, then the transceiver is optical. -// Since cable-length is only defined in buzznik's // openconfig-platform-ext.yang which is unavailable to Ondatra, // it is necessary to use raw gNMI get. func IsOptical(gnmiClient gpb.GNMIClient, dut *ondatra.DUTDevice, xcvrName string) (bool, error) { diff --git a/setup-container.sh b/setup-container.sh index fe1300c275..90bae4ef4f 100755 --- a/setup-container.sh +++ b/setup-container.sh @@ -34,6 +34,8 @@ declare -r VERBOSE_INFO="4" declare -r VERBOSE_MAX="${VERBOSE_INFO}" declare -r VERBOSE_MIN="${VERBOSE_ERROR}" +declare EXISTING_CONTAINER_NAME="" + # # Arguments ----------------------------------------------------------------------------------------------------------- # @@ -295,13 +297,54 @@ EOF fi } +function container_exists() { + container_id=`docker ps --all --filter=name=$CONTAINER_NAME --format '{{.ID}}'` + count=`echo $container_id | wc -w` + return $count +} + +function get_existing_container() { + img_id=${IMAGE_ID} + if [ -z ${img_id} ] + then + img_id=${LOCAL_IMAGE} + fi + container_id=`docker container ls --all --filter=ancestor=${img_id} --format '{{.ID}}'` + count=`echo $container_id | wc -w` + case $count in + 0) + EXISTING_CONTAINER_NAME="" + ;; + 1) + container_name=`docker inspect $container_id --format '{{.Name}}'` + if [[ $container_name == /* ]] + then + container_name="${container_name:1}" + fi + EXISTING_CONTAINER_NAME=${container_name} + ;; + *) + echo "Multiple container IDs found: ${container_id}" + EXISTING_CONTAINER_NAME="" + ;; + esac +} + function start_local_container() { - log_info "creating a container: ${CONTAINER_NAME} ..." - eval "docker run -d -t ${PUBLISH_PORTS} -h ${CONTAINER_NAME} \ - -v \"$(dirname "${SCRIPT_DIR}"):${LINK_DIR}:rslave\" ${MOUNT_POINTS} \ - --name \"${CONTAINER_NAME}\" \"${LOCAL_IMAGE}\" /bin/bash ${SILENT_HOOK}" || \ - exit_failure "failed to start a container: ${CONTAINER_NAME}" + container_exists + local exists=$? + if [ $exists -eq 1 ] + then + log_info "starting existing container ${CONTAINER_NAME} ..." + docker start ${CONTAINER_NAME} + else + log_info "creating a container: ${CONTAINER_NAME} ..." + eval "docker run -d -t ${PUBLISH_PORTS} -h ${CONTAINER_NAME} \ + -v \"$(dirname "${SCRIPT_DIR}"):${LINK_DIR}:rslave\" ${MOUNT_POINTS} \ + --name \"${CONTAINER_NAME}\" \"${LOCAL_IMAGE}\" /bin/bash ${SILENT_HOOK}" || \ + exit_failure "failed to start a container: ${CONTAINER_NAME}" + fi eval "docker exec --user root \"${CONTAINER_NAME}\" \ bash -c \"service ssh restart\" ${SILENT_HOOK}" || \ @@ -321,8 +364,15 @@ function start_local_container() { } function parse_arguments() { + if [[ -z "${CONTAINER_NAME}" ]]; then - exit_failure "container name is not set" + get_existing_container + if [ -z $EXISTING_CONTAINER_NAME ] + then + exit_failure "container name is not set." + else + exit_failure "found existing container (\"docker start $EXISTING_CONTAINER_NAME\")" + fi fi if [[ -z "${LINK_DIR}" ]]; then diff --git a/test_reporting/kusto/setup.kql b/test_reporting/kusto/setup.kql index d1b6a8079a..b7e8e2399b 100644 --- a/test_reporting/kusto/setup.kql +++ b/test_reporting/kusto/setup.kql @@ -183,7 +183,7 @@ ############################################################################### // Create table command //////////////////////////////////////////////////////////// -.create table ['RebootTimingData'] (ReportId: string, ['RebootType']:string, +.create table ['RebootTimingData'] (ReportId: string, TrackingId: string, Hostname: string, BaseImage:string, TargetImage:string, HwSku:string, ['RebootType']:string, ['ControlplaneArpPing']:string, ['ControlplaneDowntime']:string, ['DataplaneLostPackets']:string, ['DataplaneDowntime']:string, ['OffsetFromKexecDatabase']:string, ['OffsetFromKexecFinalizer']:string, ['OffsetFromKexecInitView']:string, ['OffsetFromKexecSyncdCreateSwitch']:string, ['OffsetFromKexecPortInit']:string, ['OffsetFromKexecPortReady']:string, ['OffsetFromKexecSaiCreateSwitch']:string, ['OffsetFromKexecNeighborEntry']:string, @@ -196,6 +196,11 @@ //////////////////////////////////////////////////////////// .create table ['RebootTimingData'] ingestion json mapping 'RebootTimingDataMapping' @'[' '{"column":"ReportId","Properties":{"path":"$.id"}},' + '{"column":"TrackingId","Properties":{"path":"$.tracking_id"}},' + '{"column":"Hostname","Properties":{"path":"$[\'hostname\']"}},' + '{"column":"HwSku","Properties":{"path":"$[\'hwsku\']"}},' + '{"column":"BaseImage","Properties":{"path":"$[\'base_ver\']"}},' + '{"column":"TargetImage","Properties":{"path":"$[\'target_ver\']"}},' '{"column":"OffsetFromKexecDatabase", "Properties":{"Path":"$[\'offset_from_kexec\'][\'database\']"}},' '{"column":"OffsetFromKexecFinalizer", "Properties":{"Path":"$[\'offset_from_kexec\'][\'finalizer\']"}},' '{"column":"OffsetFromKexecPortInit", "Properties":{"Path":"$[\'offset_from_kexec\'][\'port_init\']"}},' diff --git a/test_reporting/report_data_storage.py b/test_reporting/report_data_storage.py index 0130786751..a02350f827 100644 --- a/test_reporting/report_data_storage.py +++ b/test_reporting/report_data_storage.py @@ -255,9 +255,10 @@ def upload_pdu_status_data(self, pdu_status_output: List) -> None: pdu_status_data = {"data": pdu_output} self._ingest_data(self.RAW_PDU_STATUS_TABLE, pdu_status_data) - def upload_reboot_report(self, path_name: str = "", report_guid: str = "") -> None: + def upload_reboot_report(self, path_name: str = "", tracking_id: str = "", report_guid: str = "") -> None: reboot_timing_data = { - "id": report_guid + "id": report_guid, + "tracking_id": tracking_id } reboot_timing_dict = validate_json_file(path_name) reboot_timing_data.update(reboot_timing_dict) diff --git a/test_reporting/report_uploader.py b/test_reporting/report_uploader.py index ef9bdbeb52..59ef3eb2bf 100644 --- a/test_reporting/report_uploader.py +++ b/test_reporting/report_uploader.py @@ -80,9 +80,9 @@ def _run_script(): for path_name in args.path_list: try: reboot_data_regex = re.compile( - '.*test.*_(reboot|sad.*|upgrade_path)_(summary|report).json') + '.*test.*_(reboot|sad|upgrade_path).*_(summary|report).json') if reboot_data_regex.match(path_name): - kusto_db.upload_reboot_report(path_name, report_guid) + kusto_db.upload_reboot_report(path_name, tracking_id, report_guid) else: if args.json: test_result_json = validate_junit_json_file(path_name) diff --git a/tests/acl/conftest.py b/tests/acl/conftest.py index 4321f78f93..36da21a367 100644 --- a/tests/acl/conftest.py +++ b/tests/acl/conftest.py @@ -2,5 +2,9 @@ @pytest.fixture(scope='module') -def get_function_conpleteness_level(pytestconfig): +def get_function_completeness_level(pytestconfig): return pytestconfig.getoption("--completeness_level") + + +def pytest_configure(config): + config.asic_db = dict() diff --git a/tests/acl/custom_acl_table/test_custom_acl_table.py b/tests/acl/custom_acl_table/test_custom_acl_table.py index d0ac5d56c1..617e9cac66 100644 --- a/tests/acl/custom_acl_table/test_custom_acl_table.py +++ b/tests/acl/custom_acl_table/test_custom_acl_table.py @@ -11,6 +11,7 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer, LogAnalyzerError from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa: F401 logger = logging.getLogger(__name__) @@ -250,7 +251,7 @@ def build_exp_pkt(input_pkt): def test_custom_acl(rand_selected_dut, rand_unselected_dut, tbinfo, ptfadapter, setup_acl_rules, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 - setup_counterpoll_interval, remove_dataacl_table): + setup_counterpoll_interval, remove_dataacl_table, skip_traffic_test): # noqa F811 """ The test case is to verify the functionality of custom ACL table Test steps @@ -287,14 +288,15 @@ def test_custom_acl(rand_selected_dut, rand_unselected_dut, tbinfo, ptfadapter, exp_pkt = build_exp_pkt(pkt) # Send and verify packet clear_acl_counter(rand_selected_dut) - if "dualtor-aa" in tbinfo["topo"]["name"]: - clear_acl_counter(rand_unselected_dut) - ptfadapter.dataplane.flush() - testutils.send(ptfadapter, pkt=pkt, port_id=src_port_indice) - testutils.verify_packet_any_port(ptfadapter, exp_pkt, ports=dst_port_indices, timeout=5) - acl_counter = read_acl_counter(rand_selected_dut, rule) - if "dualtor-aa" in tbinfo["topo"]["name"]: - acl_counter_unselected_dut = read_acl_counter(rand_unselected_dut, rule) - acl_counter += acl_counter_unselected_dut - # Verify acl counter - pytest_assert(acl_counter == 1, "ACL counter for {} didn't increase as expected".format(rule)) + if not skip_traffic_test: + if "dualtor-aa" in tbinfo["topo"]["name"]: + clear_acl_counter(rand_unselected_dut) + ptfadapter.dataplane.flush() + testutils.send(ptfadapter, pkt=pkt, port_id=src_port_indice) + testutils.verify_packet_any_port(ptfadapter, exp_pkt, ports=dst_port_indices, timeout=5) + acl_counter = read_acl_counter(rand_selected_dut, rule) + if "dualtor-aa" in tbinfo["topo"]["name"]: + acl_counter_unselected_dut = read_acl_counter(rand_unselected_dut, rule) + acl_counter += acl_counter_unselected_dut + # Verify acl counter + pytest_assert(acl_counter == 1, "ACL counter for {} didn't increase as expected".format(rule)) diff --git a/tests/acl/null_route/test_null_route_helper.py b/tests/acl/null_route/test_null_route_helper.py index 41f81f5b95..e2f5da0a55 100644 --- a/tests/acl/null_route/test_null_route_helper.py +++ b/tests/acl/null_route/test_null_route_helper.py @@ -9,7 +9,7 @@ from ptf.mask import Mask import ptf.packet as scapy -from tests.common.fixtures.ptfhost_utils import remove_ip_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import remove_ip_addresses, skip_traffic_test # noqa F401 import ptf.testutils as testutils from tests.common.helpers.assertions import pytest_require from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer, LogAnalyzerError @@ -229,10 +229,12 @@ def generate_packet(src_ip, dst_ip, dst_mac): return pkt, exp_pkt -def send_and_verify_packet(ptfadapter, pkt, exp_pkt, tx_port, rx_port, expected_action): +def send_and_verify_packet(ptfadapter, pkt, exp_pkt, tx_port, rx_port, expected_action, skip_traffic_test): # noqa F811 """ Send packet with ptfadapter and verify if packet is forwarded or dropped as expected. """ + if skip_traffic_test: + return ptfadapter.dataplane.flush() testutils.send(ptfadapter, pkt=pkt, port_id=tx_port) if expected_action == FORWARD: @@ -241,7 +243,8 @@ def send_and_verify_packet(ptfadapter, pkt, exp_pkt, tx_port, rx_port, expected_ testutils.verify_no_packet(ptfadapter, pkt=exp_pkt, port_id=rx_port, timeout=5) -def test_null_route_helper(rand_selected_dut, tbinfo, ptfadapter, apply_pre_defined_rules, setup_ptf): +def test_null_route_helper(rand_selected_dut, tbinfo, ptfadapter, + apply_pre_defined_rules, setup_ptf, skip_traffic_test): # noqa F811 """ Test case to verify script null_route_helper. Some packets are generated as defined in TEST_DATA and sent to DUT, @@ -276,4 +279,5 @@ def test_null_route_helper(rand_selected_dut, tbinfo, ptfadapter, apply_pre_defi rand_selected_dut.shell(NULL_ROUTE_HELPER + " " + action) time.sleep(1) - send_and_verify_packet(ptfadapter, pkt, exp_pkt, random.choice(ptf_interfaces), rx_port, expected_result) + send_and_verify_packet(ptfadapter, pkt, exp_pkt, random.choice(ptf_interfaces), + rx_port, expected_result, skip_traffic_test) diff --git a/tests/acl/test_acl.py b/tests/acl/test_acl.py index d9547f5a8d..1894045cc8 100644 --- a/tests/acl/test_acl.py +++ b/tests/acl/test_acl.py @@ -15,17 +15,18 @@ from tests.common import reboot, port_toggle from tests.common.helpers.assertions import pytest_require, pytest_assert +from tests.common.helpers.sonic_db import AsicDbCli +from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer, LogAnalyzerError from tests.common.config_reload import config_reload from tests.common.fixtures.ptfhost_utils import copy_arp_responder_py, run_garp_service, change_mac_addresses # noqa F401 -from tests.common.utilities import wait_until +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.dualtor.dual_tor_mock import mock_server_base_ip_addr # noqa F401 from tests.common.helpers.constants import DEFAULT_NAMESPACE -from tests.common.utilities import get_upstream_neigh_type, get_downstream_neigh_type +from tests.common.utilities import wait_until, get_upstream_neigh_type, get_downstream_neigh_type, check_msg_in_syslog from tests.common.fixtures.conn_graph_facts import conn_graph_facts # noqa F401 from tests.common.platform.processes_utils import wait_critical_processes from tests.common.platform.interface_utils import check_all_interface_information -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) @@ -150,25 +151,31 @@ def remove_dataacl_table(duthosts): The change is written to configdb as we don't want DATAACL recovered after reboot """ TABLE_NAME = "DATAACL" - for duthost in duthosts: - lines = duthost.shell(cmd="show acl table {}".format(TABLE_NAME))['stdout_lines'] - data_acl_existing = False - for line in lines: - if TABLE_NAME in line: - data_acl_existing = True - break - if data_acl_existing: - # Remove DATAACL - logger.info("Removing ACL table {}".format(TABLE_NAME)) - cmds = [ - "config acl remove table {}".format(TABLE_NAME), - "config save -y" - ] - duthost.shell_cmds(cmds=cmds) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts: + executor.submit(remove_dataacl_table_single_dut, TABLE_NAME, duthost) yield - # Recover DUT by reloading minigraph - for duthost in duthosts: - config_reload(duthost, config_source="minigraph") + with SafeThreadPoolExecutor(max_workers=8) as executor: + # Recover DUT by reloading minigraph + for duthost in duthosts: + executor.submit(config_reload, duthost, config_source="minigraph", safe_reload=True) + + +def remove_dataacl_table_single_dut(table_name, duthost): + lines = duthost.shell(cmd="show acl table {}".format(table_name))['stdout_lines'] + data_acl_existing = False + for line in lines: + if table_name in line: + data_acl_existing = True + break + if data_acl_existing: + # Remove DATAACL + logger.info("{} Removing ACL table {}".format(duthost.hostname, table_name)) + cmds = [ + "config acl remove table {}".format(table_name), + "config save -y" + ] + duthost.shell_cmds(cmds=cmds) def get_t2_info(duthosts, tbinfo): @@ -429,7 +436,6 @@ def populate_vlan_arp_entries(setup, ptfhost, duthosts, rand_one_dut_hostname, i global DOWNSTREAM_IP_PORT_MAP # For m0 topo, need to refresh this constant for two different scenario DOWNSTREAM_IP_PORT_MAP = {} - duthost = duthosts[rand_one_dut_hostname] if setup["topo"] not in ["t0", "mx", "m0_vlan"]: def noop(): pass @@ -468,7 +474,8 @@ def populate_arp_table(): dut.command("sonic-clear arp") dut.command("sonic-clear ndp") # Wait some time to ensure the async call of clear is completed - time.sleep(20) + time.sleep(20) + for dut in duthosts: for addr in addr_list: dut.command("ping {} -c 3".format(addr), module_ignore_errors=True) @@ -479,9 +486,10 @@ def populate_arp_table(): logging.info("Stopping ARP responder") ptfhost.shell("supervisorctl stop arp_responder", module_ignore_errors=True) - duthost.command("sonic-clear fdb all") - duthost.command("sonic-clear arp") - duthost.command("sonic-clear ndp") + for dut in duthosts: + dut.command("sonic-clear fdb all") + dut.command("sonic-clear arp") + dut.command("sonic-clear ndp") @pytest.fixture(scope="module", params=["ingress", "egress"]) @@ -532,12 +540,13 @@ def create_or_remove_acl_table(duthost, acl_table_config, setup, op, topo): logger.info("Removing ACL table \"{}\" in namespace {} on device {}" .format(acl_table_config["table_name"], namespace, duthost)) sonic_host_or_asic_inst.command("config acl remove table {}".format(acl_table_config["table_name"])) - # Give the dut some time for the ACL to be applied and LOG message generated - time.sleep(30) @pytest.fixture(scope="module") -def acl_table(duthosts, rand_one_dut_hostname, setup, stage, ip_version, tbinfo): +def acl_table(duthosts, rand_one_dut_hostname, setup, stage, ip_version, tbinfo, + # make sure the tear down of core_dump_and_config_check happened after acl_table + core_dump_and_config_check + ): """Apply ACL table configuration and remove after tests. Args: @@ -564,32 +573,44 @@ def acl_table(duthosts, rand_one_dut_hostname, setup, stage, ip_version, tbinfo) dut_to_analyzer_map = {} - for duthost in duthosts: - if duthost.is_supervisor_node(): - continue - loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl") - loganalyzer.load_common_config() - dut_to_analyzer_map[duthost] = loganalyzer - - try: - loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE] - # Ignore any other errors to reduce noise - loganalyzer.ignore_regex = [r".*"] - with loganalyzer: - create_or_remove_acl_table(duthost, acl_table_config, setup, "add", topo) - except LogAnalyzerError as err: - # Cleanup Config DB if table creation failed - logger.error("ACL table creation failed, attempting to clean-up...") - create_or_remove_acl_table(duthost, acl_table_config, setup, "remove", topo) - raise err - + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts: + executor.submit(set_up_acl_table_single_dut, acl_table_config, dut_to_analyzer_map, duthost, setup, topo) try: yield acl_table_config finally: - for duthost, loganalyzer in list(dut_to_analyzer_map.items()): - loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE] - with loganalyzer: - create_or_remove_acl_table(duthost, acl_table_config, setup, "remove", topo) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost, loganalyzer in list(dut_to_analyzer_map.items()): + executor.submit(tear_down_acl_table_single_dut, acl_table_config, duthost, loganalyzer, setup, topo) + + +def tear_down_acl_table_single_dut(acl_table_config, duthost, loganalyzer, setup, topo): + loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE] + with loganalyzer: + create_or_remove_acl_table(duthost, acl_table_config, setup, "remove", topo) + wait_until(60, 10, 0, check_msg_in_syslog, + duthost, LOG_EXPECT_ACL_TABLE_REMOVE_RE) + + +def set_up_acl_table_single_dut(acl_table_config, dut_to_analyzer_map, duthost, setup, topo): + if duthost.is_supervisor_node(): + return + loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl") + loganalyzer.load_common_config() + dut_to_analyzer_map[duthost] = loganalyzer + try: + loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE] + # Ignore any other errors to reduce noise + loganalyzer.ignore_regex = [r".*"] + with loganalyzer: + create_or_remove_acl_table(duthost, acl_table_config, setup, "add", topo) + wait_until(300, 20, 0, check_msg_in_syslog, + duthost, LOG_EXPECT_ACL_TABLE_CREATE_RE) + except LogAnalyzerError as err: + # Cleanup Config DB if table creation failed + logger.error("ACL table creation failed, attempting to clean-up...") + create_or_remove_acl_table(duthost, acl_table_config, setup, "remove", topo) + raise err class BaseAclTest(six.with_metaclass(ABCMeta, object)): @@ -643,7 +664,7 @@ def teardown_rules(self, dut): dut.command("config acl update full {}".format(remove_rules_dut_path)) @pytest.fixture(scope="class", autouse=True) - def acl_rules(self, duthosts, localhost, setup, acl_table, populate_vlan_arp_entries, tbinfo, + def acl_rules(self, request, duthosts, localhost, setup, acl_table, populate_vlan_arp_entries, tbinfo, ip_version, conn_graph_facts): # noqa F811 """Setup/teardown ACL rules for the current set of tests. @@ -657,42 +678,66 @@ def acl_rules(self, duthosts, localhost, setup, acl_table, populate_vlan_arp_ent """ dut_to_analyzer_map = {} - for duthost in duthosts: - if duthost.is_supervisor_node(): - continue - loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl_rules") - loganalyzer.load_common_config() - dut_to_analyzer_map[duthost] = loganalyzer - - try: - loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE] - # Ignore any other errors to reduce noise - loganalyzer.ignore_regex = [r".*"] - with loganalyzer: - self.setup_rules(duthost, acl_table, ip_version) - # Give the dut some time for the ACL rules to be applied and LOG message generated - time.sleep(30) - - self.post_setup_hook(duthost, localhost, populate_vlan_arp_entries, tbinfo, conn_graph_facts) - - assert self.check_rule_counters(duthost), "Rule counters should be ready!" - except LogAnalyzerError as err: - # Cleanup Config DB if rule creation failed - logger.error("ACL rule application failed, attempting to clean-up...") - self.teardown_rules(duthost) - raise err + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts: + executor.submit(self.set_up_acl_rules_single_dut, request, acl_table, conn_graph_facts, + dut_to_analyzer_map, duthost, ip_version, localhost, + populate_vlan_arp_entries, tbinfo) + logger.info("Set up acl_rules finished") try: yield finally: - for duthost, loganalyzer in list(dut_to_analyzer_map.items()): - if duthost.is_supervisor_node(): - continue - loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE] - with loganalyzer: - logger.info("Removing ACL rules") - self.teardown_rules(duthost) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost, loganalyzer in list(dut_to_analyzer_map.items()): + executor.submit(self.tear_down_acl_rule_single_dut, duthost, loganalyzer) + logger.info("Tear down acl_rules finished") + + def tear_down_acl_rule_single_dut(self, duthost, loganalyzer): + if duthost.is_supervisor_node(): + return + loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE] + with loganalyzer: + logger.info("Removing ACL rules") + self.teardown_rules(duthost) + wait_until(60, 10, 0, check_msg_in_syslog, + duthost, LOG_EXPECT_ACL_RULE_REMOVE_RE) + + def set_up_acl_rules_single_dut(self, request, acl_table, + conn_graph_facts, dut_to_analyzer_map, duthost, # noqa F811 + ip_version, localhost, + populate_vlan_arp_entries, tbinfo): + logger.info("{}: ACL rule application started".format(duthost.hostname)) + if duthost.is_supervisor_node(): + return + loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl_rules") + loganalyzer.load_common_config() + dut_to_analyzer_map[duthost] = loganalyzer + try: + loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE] + # Ignore any other errors to reduce noise + loganalyzer.ignore_regex = [r".*"] + with loganalyzer: + self.setup_rules(duthost, acl_table, ip_version) + # Give the dut some time for the ACL rules to be applied and LOG message generated + wait_until(300, 20, 0, check_msg_in_syslog, + duthost, LOG_EXPECT_ACL_RULE_CREATE_RE) + + self.post_setup_hook(duthost, localhost, populate_vlan_arp_entries, tbinfo, conn_graph_facts) + + assert self.check_rule_counters(duthost), "Rule counters should be ready!" + asic_db = AsicDbCli(duthost) + asic_db.get_acl_entries(refresh=True) + asic_db.get_acl_range_entries(refresh=True) + request.config.asic_db[duthost.hostname] = asic_db + + except LogAnalyzerError as err: + # Cleanup Config DB if rule creation failed + logger.error("ACL rule application failed, attempting to clean-up...") + self.teardown_rules(duthost) + raise err + logger.info("{}: ACL rule application finished".format(duthost.hostname)) @pytest.yield_fixture(scope="class", autouse=True) def counters_sanity_check(self, duthosts, acl_rules, acl_table): @@ -730,6 +775,9 @@ def counters_sanity_check(self, duthosts, acl_rules, acl_table): time.sleep(self.ACL_COUNTERS_UPDATE_INTERVAL_SECS) for duthost in duthosts: + if duthost.facts["asic_type"] == 'vs': + logger.info('Skip checking rule counters for vs platform') + return if duthost.is_supervisor_node(): continue acl_facts[duthost]['after'] = \ @@ -785,8 +833,11 @@ def direction(self, request): return request.param def check_rule_counters(self, duthost): - logger.info('Wait all rule counters are ready') + if duthost.facts['asic_type'] == 'vs': + logger.info('Skip checking rule counters for vs platform') + return True + logger.info('Wait all rule counters are ready') return wait_until(60, 2, 0, self.check_rule_counters_internal, duthost) def check_rule_counters_internal(self, duthost): @@ -921,53 +972,81 @@ def expected_mask_routed_packet(self, pkt, ip_version): return exp_pkt - def test_ingress_unmatched_blocked(self, setup, direction, ptfadapter, ip_version, stage): + def test_ingress_unmatched_blocked(self, setup, direction, ptfadapter, ip_version, stage, skip_traffic_test): # noqa F811 """Verify that unmatched packets are dropped for ingress.""" if stage == "egress": pytest.skip("Only run for ingress") pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) - def test_egress_unmatched_forwarded(self, setup, direction, ptfadapter, ip_version, stage): + def test_egress_unmatched_forwarded(self, setup, direction, ptfadapter, ip_version, stage, skip_traffic_test): # noqa F811 """Verify that default egress rule allow all traffics""" if stage == "ingress": pytest.skip("Only run for egress") pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) - def test_source_ip_match_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_source_ip_match_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and forward a packet on source IP.""" src_ip = "20.0.0.2" if ip_version == "ipv4" else "60c0:a800::6" pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, src_ip=src_ip) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + # verify ASIC DB has entry that matches source ip with forward action set + asic_db = next(iter(request.config.asic_db.values())) + acl_entry = asic_db.find_acl_by(src_ip=src_ip, packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) counters_sanity_check.append(1) - def test_rules_priority_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_rules_priority_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we respect rule priorites in the forwarding case.""" src_ip = "20.0.0.7" if ip_version == "ipv4" else "60c0:a800::7" pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, src_ip=src_ip) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + # verify ASIC DB has a higher priority entry with src_ip and FORWARD than src_ip and DROP action + asic_db = next(iter(request.config.asic_db.values())) + acl_fwd_entry = asic_db.find_acl_by(src_ip=src_ip, packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_fwd_entry != [] + acl_drop_entry = asic_db.find_acl_by(src_ip=src_ip, packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] + assert int(acl_fwd_entry[0]['SAI_ACL_ENTRY_ATTR_PRIORITY']) > \ + int(acl_drop_entry[0]['SAI_ACL_ENTRY_ATTR_PRIORITY']) + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) counters_sanity_check.append(20) - def test_rules_priority_dropped(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_rules_priority_dropped(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we respect rule priorites in the drop case.""" src_ip = "20.0.0.3" if ip_version == "ipv4" else "60c0:a800::4" pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, src_ip=src_ip) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + # verify ASIC DB has the DROP rule for the src_ip + asic_db = next(iter(request.config.asic_db.values())) + acl_drop_entry = asic_db.find_acl_by(src_ip=src_ip, packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) counters_sanity_check.append(7) - def test_dest_ip_match_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version, vlan_name): + def test_dest_ip_match_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, vlan_name, skip_traffic_test): # noqa F811 """Verify that we can match and forward a packet on destination IP.""" dst_ip = DOWNSTREAM_IP_TO_ALLOW[ip_version] \ if direction == "uplink->downlink" else UPSTREAM_IP_TO_ALLOW[ip_version] pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, dst_ip=dst_ip) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + # verify ASIC DB has the FORWARD rule for the dst_ip + asic_db = next(iter(request.config.asic_db.values())) + acl_fwd_entry = asic_db.find_acl_by(dst_ip=dst_ip, packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_fwd_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) # Because m0_l3_scenario use differnet IPs, so need to verify different acl rules. if direction == "uplink->downlink": if setup["topo"] == "m0_l3": @@ -986,13 +1065,19 @@ def test_dest_ip_match_forwarded(self, setup, direction, ptfadapter, counters_sa rule_id = 3 counters_sanity_check.append(rule_id) - def test_dest_ip_match_dropped(self, setup, direction, ptfadapter, counters_sanity_check, ip_version, vlan_name): + def test_dest_ip_match_dropped(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, vlan_name, skip_traffic_test): # noqa F811 """Verify that we can match and drop a packet on destination IP.""" dst_ip = DOWNSTREAM_IP_TO_BLOCK[ip_version] \ if direction == "uplink->downlink" else UPSTREAM_IP_TO_BLOCK[ip_version] pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, dst_ip=dst_ip) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + # verify ASIC DB has dst_ip DROP rule + asic_db = next(iter(request.config.asic_db.values())) + acl_drop_entry = asic_db.find_acl_by(dst_ip=dst_ip, packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) # Because m0_l3_scenario use differnet IPs, so need to verify different acl rules. if direction == "uplink->downlink": if setup["topo"] == "m0_l3": @@ -1011,145 +1096,296 @@ def test_dest_ip_match_dropped(self, setup, direction, ptfadapter, counters_sani rule_id = 16 counters_sanity_check.append(rule_id) - def test_source_ip_match_dropped(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_source_ip_match_dropped(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and drop a packet on source IP.""" src_ip = "20.0.0.6" if ip_version == "ipv4" else "60c0:a800::3" pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, src_ip=src_ip) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + # verify ASIC DB has src_ip DROP rule + asic_db = next(iter(request.config.asic_db.values())) + acl_drop_entry = asic_db.find_acl_by(src_ip=src_ip, packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) counters_sanity_check.append(14) - def test_udp_source_ip_match_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_udp_source_ip_match_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and forward a UDP packet on source IP.""" src_ip = "20.0.0.4" if ip_version == "ipv4" else "60c0:a800::8" pkt = self.udp_packet(setup, direction, ptfadapter, ip_version, src_ip=src_ip) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + # verify ASIC DB has src_ip FORWARD action ACL rule + asic_db = next(iter(request.config.asic_db.values())) + acl_fwd_entry = asic_db.find_acl_by(src_ip=src_ip, packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_fwd_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) counters_sanity_check.append(13) - def test_udp_source_ip_match_dropped(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_udp_source_ip_match_dropped(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and drop a UDP packet on source IP.""" src_ip = "20.0.0.8" if ip_version == "ipv4" else "60c0:a800::2" pkt = self.udp_packet(setup, direction, ptfadapter, ip_version, src_ip=src_ip) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + # verify ASIC DB has src_ip DROP action rule + asic_db = next(iter(request.config.asic_db.values())) + acl_drop_entry = asic_db.find_acl_by(src_ip=src_ip, packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) counters_sanity_check.append(26) - def test_icmp_source_ip_match_dropped(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_icmp_source_ip_match_dropped(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and drop an ICMP packet on source IP.""" src_ip = "20.0.0.8" if ip_version == "ipv4" else "60c0:a800::2" pkt = self.icmp_packet(setup, direction, ptfadapter, ip_version, src_ip=src_ip) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + # verify ASIC DB has src_ip DROP action ACL rule + asic_db = next(iter(request.config.asic_db.values())) + acl_drop_entry = asic_db.find_acl_by(src_ip=src_ip, packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) counters_sanity_check.append(25) - def test_icmp_source_ip_match_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_icmp_source_ip_match_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and forward an ICMP packet on source IP.""" src_ip = "20.0.0.4" if ip_version == "ipv4" else "60c0:a800::8" pkt = self.icmp_packet(setup, direction, ptfadapter, ip_version, src_ip=src_ip) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + # verify ASIC DB has src_ip FORWARD action ACL rule + asic_db = next(iter(request.config.asic_db.values())) + acl_fwd_entry = asic_db.find_acl_by(src_ip=src_ip, packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_fwd_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) counters_sanity_check.append(12) - def test_l4_dport_match_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_l4_dport_match_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and forward on L4 destination port.""" - pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, dport=0x1217) + dst_port = 0x1217 + pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, dport=dst_port) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + # verify ASIC DB has dst_port FORWARD action ACL rule + asic_db = next(iter(request.config.asic_db.values())) + acl_fwd_entry = asic_db.find_acl_by(l4_dst_port=str(dst_port), packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_fwd_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) counters_sanity_check.append(9) - def test_l4_sport_match_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_l4_sport_match_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and forward on L4 source port.""" - pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, sport=0x120D) + src_port = 0x120D + pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, sport=src_port) + + # verify ASIC DB has src_port FORWARD action ACL rule + asic_db = next(iter(request.config.asic_db.values())) + acl_fwd_entry = asic_db.find_acl_by(l4_src_port=str(src_port), packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_fwd_entry != [] - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) counters_sanity_check.append(4) - def test_l4_dport_range_match_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_l4_dport_range_match_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and forward on a range of L4 destination ports.""" - pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, dport=0x123B) + dport = 0x123B + pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, dport=dport) + + # verify ASIC DB has FORWARD rule for port in configured range + asic_db = next(iter(request.config.asic_db.values())) + acl_fwd_entry = asic_db.find_acl_by(range_type='l4_dst_port', l4_dst_port=str(dport), + packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_fwd_entry != [] - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) counters_sanity_check.append(11) - def test_l4_sport_range_match_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_l4_sport_range_match_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and forward on a range of L4 source ports.""" - pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, sport=0x123A) + sport = 0x123A + pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, sport=sport) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + # verify ASIC DB has FORWARD rule for port in configured range + asic_db = next(iter(request.config.asic_db.values())) + acl_fwd_entry = asic_db.find_acl_by(range_type='l4_src_port', l4_src_port=str(sport), + packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_fwd_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) counters_sanity_check.append(10) - def test_l4_dport_range_match_dropped(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_l4_dport_range_match_dropped(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and drop on a range of L4 destination ports.""" - pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, dport=0x127B) + dport = 0x1285 + pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, dport=dport) + + # verify ASIC DB has DROP rule for port in configured range + asic_db = next(iter(request.config.asic_db.values())) + acl_drop_entry = asic_db.find_acl_by(range_type='l4_dst_port', l4_dst_port=str(dport), + packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) counters_sanity_check.append(22) - def test_l4_sport_range_match_dropped(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_l4_sport_range_match_dropped(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and drop on a range of L4 source ports.""" - pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, sport=0x1271) + sport = 0x1298 + pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, sport=sport) + + # verify ASIC DB has DROP rule for port in configured range + asic_db = next(iter(request.config.asic_db.values())) + acl_drop_entry = asic_db.find_acl_by(range_type='l4_src_port', l4_src_port=str(sport), + packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) counters_sanity_check.append(17) - def test_ip_proto_match_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_ip_proto_match_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and forward on the IP protocol.""" + ip_protocol = 0x7E pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, proto=0x7E) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + # verify ASIC DB has FORWARD rule for IP protocol number + asic_db = next(iter(request.config.asic_db.values())) + acl_fwd_entry = [] + if ip_version == "ipv4": + acl_fwd_entry = asic_db.find_acl_by(ip_protocol=str(ip_protocol), + packet_action='SAI_PACKET_ACTION_FORWARD') + else: + acl_fwd_entry = asic_db.find_acl_by(ipv6_next_header=str(ip_protocol), + packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_fwd_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) counters_sanity_check.append(5) - def test_tcp_flags_match_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_tcp_flags_match_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and forward on the TCP flags.""" - pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, flags=0x1B) + tcp_flags = 0x1B + pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, flags=tcp_flags) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + # verify ASIC DB has FORWARD rule for TCP Flags + asic_db = next(iter(request.config.asic_db.values())) + acl_fwd_entry = asic_db.find_acl_by(tcp_flags=str(tcp_flags), packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_fwd_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) counters_sanity_check.append(6) - def test_l4_dport_match_dropped(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_l4_dport_match_dropped(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and drop on L4 destination port.""" - pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, dport=0x127B) + dst_port = 0x127B + pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, dport=dst_port) + + # verify ASIC DB has DROP rule for dst_port + asic_db = next(iter(request.config.asic_db.values())) + acl_drop_entry = asic_db.find_acl_by(l4_dst_port=str(dst_port), packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) counters_sanity_check.append(22) - def test_l4_sport_match_dropped(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_l4_sport_match_dropped(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and drop on L4 source port.""" - pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, sport=0x1271) + src_port = 0x1271 + pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, sport=src_port) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + # verify ASIC DB has DROP rule for src_port + asic_db = next(iter(request.config.asic_db.values())) + acl_drop_entry = asic_db.find_acl_by(l4_src_port=str(src_port), packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) counters_sanity_check.append(17) - def test_ip_proto_match_dropped(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_ip_proto_match_dropped(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and drop on the IP protocol.""" - pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, proto=0x7F) + ip_protocol = 0x7F + pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, proto=ip_protocol) - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + # verify ASIC DB has DROP rule for IP protocol number + asic_db = next(iter(request.config.asic_db.values())) + acl_drop_entry = [] + if ip_version == "ipv4": + acl_drop_entry = asic_db.find_acl_by(ip_protocol=str(ip_protocol), + packet_action='SAI_PACKET_ACTION_DROP') + else: + acl_drop_entry = asic_db.find_acl_by(ipv6_next_header=str(ip_protocol), + packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] + + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) counters_sanity_check.append(18) - def test_tcp_flags_match_dropped(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_tcp_flags_match_dropped(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and drop on the TCP flags.""" - pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, flags=0x24) + tcp_flags = 0x24 + pkt = self.tcp_packet(setup, direction, ptfadapter, ip_version, flags=tcp_flags) + + # verify ASIC DB has DROP rule for matched TCP flags + asic_db = next(iter(request.config.asic_db.values())) + acl_drop_entry = asic_db.find_acl_by(tcp_flags=str(tcp_flags), packet_action='SAI_PACKET_ACTION_DROP') + assert acl_drop_entry != [] - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version) + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, True, ip_version, skip_traffic_test) counters_sanity_check.append(19) - def test_icmp_match_forwarded(self, setup, direction, ptfadapter, counters_sanity_check, ip_version): + def test_icmp_match_forwarded(self, request, setup, direction, ptfadapter, + counters_sanity_check, ip_version, skip_traffic_test): # noqa F811 """Verify that we can match and drop on the TCP flags.""" src_ip = "20.0.0.10" if ip_version == "ipv4" else "60c0:a800::10" - pkt = self.icmp_packet(setup, direction, ptfadapter, ip_version, src_ip=src_ip, icmp_type=3, icmp_code=1) + icmp_type = 3 + icmp_code = 1 + pkt = self.icmp_packet(setup, direction, ptfadapter, ip_version, src_ip=src_ip, + icmp_type=icmp_type, icmp_code=icmp_code) + + # verify ASIC DB has FORWARD rule for ICMP packet type and code + asic_db = next(iter(request.config.asic_db.values())) + acl_fwd_entry = [] + if ip_version == "ipv4": + acl_fwd_entry = asic_db.find_acl_by(src_ip=src_ip, icmp_type=str(icmp_type), + icmp_code=str(icmp_code), + packet_action='SAI_PACKET_ACTION_FORWARD') + else: + acl_fwd_entry = asic_db.find_acl_by(src_ip=src_ip, icmpv6_type=str(icmp_type), + icmpv6_code=str(icmp_code), + packet_action='SAI_PACKET_ACTION_FORWARD') + assert acl_fwd_entry != [] - self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version) + self._verify_acl_traffic(setup, direction, ptfadapter, pkt, False, ip_version, skip_traffic_test) counters_sanity_check.append(29) - def _verify_acl_traffic(self, setup, direction, ptfadapter, pkt, dropped, ip_version): + def _verify_acl_traffic(self, setup, direction, ptfadapter, pkt, dropped, ip_version, skip_traffic_test): # noqa F811 exp_pkt = self.expected_mask_routed_packet(pkt, ip_version) if ip_version == "ipv4": downstream_dst_port = DOWNSTREAM_IP_PORT_MAP.get(pkt[packet.IP].dst) else: downstream_dst_port = DOWNSTREAM_IP_PORT_MAP.get(pkt[packet.IPv6].dst) + + if skip_traffic_test: + return + ptfadapter.dataplane.flush() testutils.send(ptfadapter, self.src_port, pkt) if direction == "uplink->downlink" and downstream_dst_port: @@ -1236,7 +1472,7 @@ def post_setup_hook(self, dut, localhost, populate_vlan_arp_entries, tbinfo, con """ dut.command("config save -y") - reboot(dut, localhost, wait=240) + reboot(dut, localhost, safe_reboot=True, check_intf_up_ports=True) # We need some additional delay on e1031 if dut.facts["platform"] == "x86_64-cel_e1031-r0": time.sleep(240) diff --git a/tests/acl/test_acl_outer_vlan.py b/tests/acl/test_acl_outer_vlan.py index b5fbb6e57e..8c674ef461 100644 --- a/tests/acl/test_acl_outer_vlan.py +++ b/tests/acl/test_acl_outer_vlan.py @@ -14,14 +14,13 @@ from tests.common.utilities import wait_until from tests.common.config_reload import config_reload from tests.common.helpers.assertions import pytest_assert, pytest_require -from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import change_mac_addresses, skip_traffic_test # noqa F401 from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer, LogAnalyzerError from abc import abstractmethod from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F401 from tests.common.utilities import check_skip_release from tests.common.utilities import get_neighbor_ptf_port_list from tests.common.helpers.constants import UPSTREAM_NEIGHBOR_MAP -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) @@ -476,7 +475,8 @@ def _setup_acl_rules(self, duthost, stage, ip_ver, vlan_id, action): logger.info("Creating ACL rule matching vlan {} action {}".format(vlan_id, action)) duthost.shell("config load -y {}".format(dest_path)) - pytest_assert(wait_until(60, 2, 0, check_rule_counters, duthost), "Acl rule counters are not ready") + if duthost.facts['asic_type'] != 'vs': + pytest_assert(wait_until(60, 2, 0, check_rule_counters, duthost), "Acl rule counters are not ready") def _remove_acl_rules(self, duthost, stage, ip_ver): table_name = ACL_TABLE_NAME_TEMPLATE.format(stage, ip_ver) @@ -514,7 +514,8 @@ def setup(self, rand_selected_dut, ptfhost, ip_version, vlan_setup_info): finally: self.post_running_hook(rand_selected_dut, ptfhost, ip_version) - def _do_verification(self, ptfadapter, duthost, tbinfo, vlan_setup_info, ip_version, tagged_mode, action): + def _do_verification(self, ptfadapter, duthost, tbinfo, vlan_setup_info, + ip_version, tagged_mode, action, skip_traffic_test): # noqa F811 vlan_setup, _, _, _ = vlan_setup_info test_setup_config = self.setup_cfg(duthost, tbinfo, vlan_setup, tagged_mode, ip_version) @@ -557,14 +558,15 @@ def _do_verification(self, ptfadapter, duthost, tbinfo, vlan_setup_info, ip_vers table_name = ACL_TABLE_NAME_TEMPLATE.format(stage, ip_version) try: self._setup_acl_rules(duthost, stage, ip_version, outer_vlan_id, action) - count_before = get_acl_counter(duthost, table_name, RULE_1, timeout=0) - - send_and_verify_traffic(ptfadapter, pkt, exp_pkt, src_port, dst_port, pkt_action=action) - count_after = get_acl_counter(duthost, table_name, RULE_1) - - logger.info("Verify Acl counter incremented {} > {}".format(count_after, count_before)) - pytest_assert(count_after >= count_before + 1, - "Unexpected results, counter_after {} > counter_before {}".format(count_after, count_before)) + if not skip_traffic_test: + count_before = get_acl_counter(duthost, table_name, RULE_1, timeout=0) + send_and_verify_traffic(ptfadapter, pkt, exp_pkt, src_port, dst_port, pkt_action=action) + count_after = get_acl_counter(duthost, table_name, RULE_1) + + logger.info("Verify Acl counter incremented {} > {}".format(count_after, count_before)) + pytest_assert(count_after >= count_before + 1, + "Unexpected results, counter_after {} > counter_before {}" + .format(count_after, count_before)) except Exception as e: raise (e) finally: @@ -572,83 +574,83 @@ def _do_verification(self, ptfadapter, duthost, tbinfo, vlan_setup_info, ip_vers @pytest.mark.po2vlan def test_tagged_forwarded(self, ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F811 - ): + ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 + skip_traffic_test): # noqa F811 """ Verify packet is forwarded by ACL rule on tagged interface """ self._do_verification(ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, TYPE_TAGGED, ACTION_FORWARD) + ip_version, TYPE_TAGGED, ACTION_FORWARD, skip_traffic_test) @pytest.mark.po2vlan def test_tagged_dropped(self, ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F811 - ): + ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 + skip_traffic_test): # noqa F811 """ Verify packet is dropped by ACL rule on tagged interface """ self._do_verification(ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, TYPE_TAGGED, ACTION_DROP) + ip_version, TYPE_TAGGED, ACTION_DROP, skip_traffic_test) @pytest.mark.po2vlan def test_untagged_forwarded(self, ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F811 - ): + ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 + skip_traffic_test): # noqa F811 """ Verify packet is forwarded by ACL rule on untagged interface """ self._do_verification(ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, TYPE_UNTAGGED, ACTION_FORWARD) + ip_version, TYPE_UNTAGGED, ACTION_FORWARD, skip_traffic_test) @pytest.mark.po2vlan def test_untagged_dropped(self, ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F811 - ): + ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 + skip_traffic_test): # noqa F811 """ Verify packet is dropped by ACL rule on untagged interface """ self._do_verification(ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, TYPE_UNTAGGED, ACTION_DROP) + ip_version, TYPE_UNTAGGED, ACTION_DROP, skip_traffic_test) @pytest.mark.po2vlan def test_combined_tagged_forwarded(self, ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F811 - ): + ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 + skip_traffic_test): # noqa F811 """ Verify packet is forwarded by ACL rule on tagged interface, and the interface belongs to two vlans """ self._do_verification(ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, TYPE_COMBINE_TAGGED, ACTION_FORWARD) + ip_version, TYPE_COMBINE_TAGGED, ACTION_FORWARD, skip_traffic_test) @pytest.mark.po2vlan def test_combined_tagged_dropped(self, ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F811 - ): + ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 + skip_traffic_test): # noqa F811 """ Verify packet is dropped by ACL rule on tagged interface, and the interface belongs to two vlans """ self._do_verification(ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, TYPE_COMBINE_TAGGED, ACTION_DROP) + ip_version, TYPE_COMBINE_TAGGED, ACTION_DROP, skip_traffic_test) @pytest.mark.po2vlan def test_combined_untagged_forwarded(self, ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F811 - ): + ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 + skip_traffic_test): # noqa F811 """ Verify packet is forwarded by ACL rule on untagged interface, and the interface belongs to two vlans """ self._do_verification(ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, TYPE_COMBINE_UNTAGGED, ACTION_FORWARD) + ip_version, TYPE_COMBINE_UNTAGGED, ACTION_FORWARD, skip_traffic_test) @pytest.mark.po2vlan def test_combined_untagged_dropped(self, ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F811 - ): + ip_version, toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 + skip_traffic_test): # noqa F811 """ Verify packet is dropped by ACL rule on untagged interface, and the interface belongs to two vlans """ self._do_verification(ptfadapter, rand_selected_dut, tbinfo, vlan_setup_info, - ip_version, TYPE_COMBINE_UNTAGGED, ACTION_DROP) + ip_version, TYPE_COMBINE_UNTAGGED, ACTION_DROP, skip_traffic_test) @pytest.fixture(scope='module', autouse=True) diff --git a/tests/acl/test_stress_acl.py b/tests/acl/test_stress_acl.py index 881f5ae31a..c507564456 100644 --- a/tests/acl/test_stress_acl.py +++ b/tests/acl/test_stress_acl.py @@ -7,7 +7,7 @@ from collections import defaultdict from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor # noqa F401 from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 pytestmark = [ pytest.mark.topology("t0", "t1", "m0", "mx"), @@ -118,8 +118,8 @@ def prepare_test_port(rand_selected_dut, tbinfo): return ptf_src_port, upstream_port_ids, dut_port -def verify_acl_rules(rand_selected_dut, ptfadapter, ptf_src_port, - ptf_dst_ports, acl_rule_list, del_rule_id, verity_status): +def verify_acl_rules(rand_selected_dut, ptfadapter, ptf_src_port, ptf_dst_ports, + acl_rule_list, del_rule_id, verity_status, skip_traffic_test): # noqa F811 for acl_id in acl_rule_list: ip_addr1 = acl_id % 256 @@ -146,12 +146,13 @@ def verify_acl_rules(rand_selected_dut, ptfadapter, ptf_src_port, exp_pkt.set_do_not_care_scapy(packet.Ether, 'src') exp_pkt.set_do_not_care_scapy(packet.IP, "chksum") - ptfadapter.dataplane.flush() - testutils.send(test=ptfadapter, port_id=ptf_src_port, pkt=pkt) - if verity_status == "forward" or acl_id == del_rule_id: - testutils.verify_packet_any_port(test=ptfadapter, pkt=exp_pkt, ports=ptf_dst_ports) - elif verity_status == "drop" and acl_id != del_rule_id: - testutils.verify_no_packet_any(test=ptfadapter, pkt=exp_pkt, ports=ptf_dst_ports) + if not skip_traffic_test: + ptfadapter.dataplane.flush() + testutils.send(test=ptfadapter, port_id=ptf_src_port, pkt=pkt) + if verity_status == "forward" or acl_id == del_rule_id: + testutils.verify_packet_any_port(test=ptfadapter, pkt=exp_pkt, ports=ptf_dst_ports) + elif verity_status == "drop" and acl_id != del_rule_id: + testutils.verify_no_packet_any(test=ptfadapter, pkt=exp_pkt, ports=ptf_dst_ports) def acl_rule_loaded(rand_selected_dut, acl_rule_list): @@ -166,8 +167,8 @@ def acl_rule_loaded(rand_selected_dut, acl_rule_list): def test_acl_add_del_stress(rand_selected_dut, tbinfo, ptfadapter, prepare_test_file, - prepare_test_port, get_function_conpleteness_level, - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + prepare_test_port, get_function_completeness_level, + toggle_all_simulator_ports_to_rand_selected_tor, skip_traffic_test): # noqa F811 ptf_src_port, ptf_dst_ports, dut_port = prepare_test_port @@ -176,15 +177,16 @@ def test_acl_add_del_stress(rand_selected_dut, tbinfo, ptfadapter, prepare_test_ cmd_add_rules = "sonic-cfggen -j {} -w".format(STRESS_ACL_RULE_JSON_FILE) cmd_rm_all_rules = "acl-loader delete STRESS_ACL" - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: - normalized_level = 'basic' + normalized_level = 'debug' loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level] wait_timeout = 15 rand_selected_dut.shell(cmd_create_table) acl_rule_list = list(range(1, ACL_RULE_NUMS + 1)) - verify_acl_rules(rand_selected_dut, ptfadapter, ptf_src_port, ptf_dst_ports, acl_rule_list, 0, "forward") + verify_acl_rules(rand_selected_dut, ptfadapter, ptf_src_port, ptf_dst_ports, + acl_rule_list, 0, "forward", skip_traffic_test) try: loops = 0 while loops <= loop_times: @@ -201,7 +203,8 @@ def test_acl_add_del_stress(rand_selected_dut, tbinfo, ptfadapter, prepare_test_ acl_rule_list.append(readd_id) wait_until(wait_timeout, 2, 0, acl_rule_loaded, rand_selected_dut, acl_rule_list) - verify_acl_rules(rand_selected_dut, ptfadapter, ptf_src_port, ptf_dst_ports, acl_rule_list, 0, "drop") + verify_acl_rules(rand_selected_dut, ptfadapter, ptf_src_port, ptf_dst_ports, + acl_rule_list, 0, "drop", skip_traffic_test) del_rule_id = random.choice(acl_rule_list) rand_selected_dut.shell('sonic-db-cli CONFIG_DB del "ACL_RULE|STRESS_ACL| RULE_{}"'.format(del_rule_id)) @@ -209,7 +212,7 @@ def test_acl_add_del_stress(rand_selected_dut, tbinfo, ptfadapter, prepare_test_ wait_until(wait_timeout, 2, 0, acl_rule_loaded, rand_selected_dut, acl_rule_list) verify_acl_rules(rand_selected_dut, ptfadapter, ptf_src_port, ptf_dst_ports, - acl_rule_list, del_rule_id, "drop") + acl_rule_list, del_rule_id, "drop", skip_traffic_test) loops += 1 finally: diff --git a/tests/arp/arp_utils.py b/tests/arp/arp_utils.py index 891d26845f..3b4c73270a 100644 --- a/tests/arp/arp_utils.py +++ b/tests/arp/arp_utils.py @@ -34,26 +34,6 @@ def collect_info(duthost): duthost.shell('grep . /sys/class/net/PortChannel*/address', module_ignore_errors=True) -def increment_ipv4_addr(ipv4_addr, incr=1): - octets = str(ipv4_addr).split('.') - last_octet = int(octets[-1]) - last_octet += incr - octets[-1] = str(last_octet) - - return '.'.join(octets) - - -def increment_ipv6_addr(ipv6_addr, incr=1): - octets = str(ipv6_addr).split(':') - last_octet = octets[-1] - if last_octet == '': - last_octet = '0' - incremented_octet = int(last_octet, 16) + incr - new_octet_str = '{:x}'.format(incremented_octet) - - return ':'.join(octets[:-1]) + ':' + new_octet_str - - def MacToInt(mac): mac = mac.replace(":", "") return int(mac, 16) diff --git a/tests/arp/conftest.py b/tests/arp/conftest.py index be8354a596..5ef03ecf06 100644 --- a/tests/arp/conftest.py +++ b/tests/arp/conftest.py @@ -10,9 +10,8 @@ from tests.common import constants from tests.common.config_reload import config_reload from ipaddress import ip_network, IPv6Network, IPv4Network -from tests.arp.arp_utils import increment_ipv6_addr, increment_ipv4_addr from tests.common.helpers.assertions import pytest_require as pt_require -from tests.common.utilities import wait +from tests.common.utilities import wait, increment_ipv6_addr, increment_ipv4_addr from scapy.all import Ether, IPv6, ICMPv6ND_NS, ICMPv6ND_NA, \ ICMPv6NDOptSrcLLAddr, in6_getnsmac, \ in6_getnsma, inet_pton, inet_ntop, socket @@ -51,7 +50,7 @@ def pytest_addoption(parser): @pytest.fixture(scope='module') -def get_function_conpleteness_level(pytestconfig): +def get_function_completeness_level(pytestconfig): return pytestconfig.getoption("--completeness_level") diff --git a/tests/arp/test_arp_dualtor.py b/tests/arp/test_arp_dualtor.py index f262248fce..b2abc94d5f 100644 --- a/tests/arp/test_arp_dualtor.py +++ b/tests/arp/test_arp_dualtor.py @@ -15,7 +15,6 @@ from tests.common.dualtor.dual_tor_common import mux_config # noqa F401 from tests.common.fixtures.ptfhost_utils import run_garp_service, \ change_mac_addresses, run_icmp_responder, pause_garp_service # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 from tests.common.utilities import wait_until diff --git a/tests/arp/test_arp_extended.py b/tests/arp/test_arp_extended.py index 987f32dc31..c3f432e695 100644 --- a/tests/arp/test_arp_extended.py +++ b/tests/arp/test_arp_extended.py @@ -5,9 +5,9 @@ import ptf.testutils as testutils import pytest -from tests.arp.arp_utils import clear_dut_arp_cache, increment_ipv4_addr +from tests.arp.arp_utils import clear_dut_arp_cache +from tests.common.utilities import increment_ipv4_addr from tests.common.helpers.assertions import pytest_assert, pytest_require -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('t0', 'dualtor') diff --git a/tests/arp/test_arpall.py b/tests/arp/test_arpall.py index 00aca289ed..b885023b68 100644 --- a/tests/arp/test_arpall.py +++ b/tests/arp/test_arpall.py @@ -8,7 +8,6 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 from tests.common.fixtures.ptfhost_utils import set_ptf_port_mapping_mode # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ diff --git a/tests/arp/test_neighbor_mac.py b/tests/arp/test_neighbor_mac.py index 7721ebfb83..e0347bdd77 100644 --- a/tests/arp/test_neighbor_mac.py +++ b/tests/arp/test_neighbor_mac.py @@ -1,9 +1,10 @@ +import contextlib import logging import pytest import time from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from tests.common.utilities import wait_until logger = logging.getLogger(__name__) @@ -36,15 +37,43 @@ def interfaceConfig(self, duthosts, rand_one_dut_hostname): None """ duthost = duthosts[rand_one_dut_hostname] - logger.info("Configure the DUT interface, start interface, add IP address") - self.__startInterface(duthost) - self.__configureInterfaceIp(duthost, action="add") - yield - - logger.info("Restore the DUT interface config, remove IP address") - self.__configureInterfaceIp(duthost, action="remove") - self.__shutdownInterface(duthost) + intfStatus = duthost.show_interface(command="status")["ansible_facts"]["int_status"] + if self.DUT_ETH_IF not in intfStatus: + pytest.skip('{} not found'.format(self.DUT_ETH_IF)) + + status = intfStatus[self.DUT_ETH_IF] + if "up" not in status["oper_state"]: + pytest.skip('{} is down'.format(self.DUT_ETH_IF)) + + portchannel = status["vlan"] if "PortChannel" in status["vlan"] else None + + @contextlib.contextmanager + def removeFromPortChannel(duthost, portchannel, intf): + try: + if portchannel: + duthost.command("sudo config portchannel member del {} {}".format(portchannel, intf)) + pytest_assert(wait_until( + 10, 1, 0, + lambda: 'routed' in duthost.show_interface(command="status") + ["ansible_facts"]["int_status"][intf]["vlan"]), + '{} is not in routed status'.format(intf) + ) + yield + finally: + if portchannel: + duthost.command("sudo config portchannel member add {} {}".format(portchannel, intf)) + + with removeFromPortChannel(duthost, portchannel, self.DUT_ETH_IF): + logger.info("Configure the DUT interface, start interface, add IP address") + self.__startInterface(duthost) + self.__configureInterfaceIp(duthost, action="add") + + yield + + logger.info("Restore the DUT interface config, remove IP address") + self.__configureInterfaceIp(duthost, action="remove") + self.__shutdownInterface(duthost) @pytest.fixture(params=[0, 1]) def macIndex(self, request): diff --git a/tests/arp/test_neighbor_mac_noptf.py b/tests/arp/test_neighbor_mac_noptf.py index 18be64a12f..34c76b9ec2 100644 --- a/tests/arp/test_neighbor_mac_noptf.py +++ b/tests/arp/test_neighbor_mac_noptf.py @@ -5,7 +5,6 @@ from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert from tests.common.config_reload import config_reload -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/arp/test_stress_arp.py b/tests/arp/test_stress_arp.py index a335d8fcfc..9461448a4c 100644 --- a/tests/arp/test_stress_arp.py +++ b/tests/arp/test_stress_arp.py @@ -2,14 +2,15 @@ import time import pytest from .arp_utils import MacToInt, IntToMac, get_crm_resources, fdb_cleanup, \ - clear_dut_arp_cache, increment_ipv6_addr, get_fdb_dynamic_mac_count + clear_dut_arp_cache, get_fdb_dynamic_mac_count import ptf.testutils as testutils from tests.common.helpers.assertions import pytest_assert, pytest_require from scapy.all import Ether, IPv6, ICMPv6ND_NS, ICMPv6NDOptSrcLLAddr, in6_getnsmac, \ in6_getnsma, inet_pton, inet_ntop, socket from ipaddress import ip_address, ip_network -from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from tests.common.utilities import wait_until, increment_ipv6_addr +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 +from tests.common.errors import RunAnsibleModuleFail ARP_BASE_IP = "172.16.0.1/16" ARP_SRC_MAC = "00:00:01:02:03:04" @@ -30,6 +31,34 @@ } +@pytest.fixture(autouse=True) +def arp_cache_fdb_cleanup(duthost): + try: + clear_dut_arp_cache(duthost) + fdb_cleanup(duthost) + except RunAnsibleModuleFail as e: + if 'Failed to send flush request: No such file or directory' in str(e): + logger.warning("Failed to clear arp cache or cleanup fdb table, file may not exist yet") + else: + raise e + + time.sleep(5) + + yield + + # Ensure clean test environment even after failing + try: + clear_dut_arp_cache(duthost) + fdb_cleanup(duthost) + except RunAnsibleModuleFail as e: + if 'Failed to send flush request: No such file or directory' in str(e): + logger.warning("Failed to clear arp cache or cleanup fdb table, file may not exist yet") + else: + raise e + + time.sleep(10) + + def add_arp(ptf_intf_ipv4_addr, intf1_index, ptfadapter): ip_num = 0 for arp_request_ip in ptf_intf_ipv4_addr: @@ -57,20 +86,18 @@ def genrate_ipv4_ip(): def test_ipv4_arp(duthost, garp_enabled, ip_and_intf_info, intfs_for_test, - ptfadapter, get_function_conpleteness_level): + ptfadapter, get_function_completeness_level, skip_traffic_test): # noqa F811 """ Send gratuitous ARP (GARP) packet sfrom the PTF to the DUT The DUT should learn the (previously unseen) ARP info from the packet """ - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: normalized_level = "debug" - ipv4_avaliable = get_crm_resources(duthost, "ipv4_neighbor", "available") - \ - get_crm_resources(duthost, "ipv4_neighbor", "used") - fdb_avaliable = get_crm_resources(duthost, "fdb_entry", "available") - \ - get_crm_resources(duthost, "fdb_entry", "used") + ipv4_avaliable = get_crm_resources(duthost, "ipv4_neighbor", "available") + fdb_avaliable = get_crm_resources(duthost, "fdb_entry", "available") pytest_assert(ipv4_avaliable > 0 and fdb_avaliable > 0, "Entries have been filled") arp_avaliable = min(min(ipv4_avaliable, fdb_avaliable), ENTRIES_NUMBERS) @@ -86,12 +113,23 @@ def test_ipv4_arp(duthost, garp_enabled, ip_and_intf_info, intfs_for_test, loop_times -= 1 try: add_arp(ptf_intf_ipv4_hosts, intf1_index, ptfadapter) - - pytest_assert(wait_until(20, 1, 0, lambda: get_fdb_dynamic_mac_count(duthost) >= arp_avaliable), - "ARP Table Add failed") + if not skip_traffic_test: + # There is a certain probability of hash collision, we set the percentage as 1% here + # The entries we add will not exceed 10000, so the number we tolerate is 100 + logger.debug("Expected route number: {}, real route number {}" + .format(arp_avaliable, get_fdb_dynamic_mac_count(duthost))) + pytest_assert(wait_until(20, 1, 0, + lambda: abs(arp_avaliable - get_fdb_dynamic_mac_count(duthost)) < 250), + "ARP Table Add failed") finally: - clear_dut_arp_cache(duthost) - fdb_cleanup(duthost) + try: + clear_dut_arp_cache(duthost) + fdb_cleanup(duthost) + except RunAnsibleModuleFail as e: + if 'Failed to send flush request: No such file or directory' in str(e): + logger.warning("Failed to clear arp cache, file may not exist yet") + else: + raise e time.sleep(5) @@ -137,21 +175,19 @@ def add_nd(ptfadapter, ip_and_intf_info, ptf_intf_index, nd_avaliable): def test_ipv6_nd(duthost, ptfhost, config_facts, tbinfo, ip_and_intf_info, - ptfadapter, get_function_conpleteness_level, proxy_arp_enabled): + ptfadapter, get_function_completeness_level, proxy_arp_enabled, skip_traffic_test): # noqa F811 _, _, ptf_intf_ipv6_addr, _, ptf_intf_index = ip_and_intf_info ptf_intf_ipv6_addr = increment_ipv6_addr(ptf_intf_ipv6_addr) pytest_require(proxy_arp_enabled, 'Proxy ARP not enabled for all VLANs') pytest_require(ptf_intf_ipv6_addr is not None, 'No IPv6 VLAN address configured on device') - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: normalized_level = "debug" loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level] - ipv6_avaliable = get_crm_resources(duthost, "ipv6_neighbor", "available") - \ - get_crm_resources(duthost, "ipv6_neighbor", "used") - fdb_avaliable = get_crm_resources(duthost, "fdb_entry", "available") - \ - get_crm_resources(duthost, "fdb_entry", "used") + ipv6_avaliable = get_crm_resources(duthost, "ipv6_neighbor", "available") + fdb_avaliable = get_crm_resources(duthost, "fdb_entry", "available") pytest_assert(ipv6_avaliable > 0 and fdb_avaliable > 0, "Entries have been filled") nd_avaliable = min(min(ipv6_avaliable, fdb_avaliable), ENTRIES_NUMBERS) @@ -160,11 +196,22 @@ def test_ipv6_nd(duthost, ptfhost, config_facts, tbinfo, ip_and_intf_info, loop_times -= 1 try: add_nd(ptfadapter, ip_and_intf_info, ptf_intf_index, nd_avaliable) - - pytest_assert(wait_until(20, 1, 0, lambda: get_fdb_dynamic_mac_count(duthost) >= nd_avaliable), - "Neighbor Table Add failed") + if not skip_traffic_test: + # There is a certain probability of hash collision, we set the percentage as 1% here + # The entries we add will not exceed 10000, so the number we tolerate is 100 + logger.debug("Expected route number: {}, real route number {}" + .format(nd_avaliable, get_fdb_dynamic_mac_count(duthost))) + pytest_assert(wait_until(20, 1, 0, + lambda: abs(nd_avaliable - get_fdb_dynamic_mac_count(duthost)) < 250), + "Neighbor Table Add failed") finally: - clear_dut_arp_cache(duthost) - fdb_cleanup(duthost) + try: + clear_dut_arp_cache(duthost) + fdb_cleanup(duthost) + except RunAnsibleModuleFail as e: + if 'Failed to send flush request: No such file or directory' in str(e): + logger.warning("Failed to clear arp cache, file may not exist yet") + else: + raise e # Wait for 10 seconds before starting next loop time.sleep(10) diff --git a/tests/arp/test_tagged_arp.py b/tests/arp/test_tagged_arp.py index 1424c0f20f..a537e1e225 100644 --- a/tests/arp/test_tagged_arp.py +++ b/tests/arp/test_tagged_arp.py @@ -17,7 +17,6 @@ from tests.common.helpers.portchannel_to_vlan import setup_po2vlan # noqa F401 from tests.common.fixtures.ptfhost_utils import remove_ip_addresses # noqa F401 from tests.common.helpers.portchannel_to_vlan import running_vlan_ports_list -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/arp/test_unknown_mac.py b/tests/arp/test_unknown_mac.py index b0f63b778b..e1233b1f73 100644 --- a/tests/arp/test_unknown_mac.py +++ b/tests/arp/test_unknown_mac.py @@ -14,11 +14,11 @@ from tests.common import constants from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 from tests.common.fixtures.ptfhost_utils import copy_arp_responder_py # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.helpers.assertions import pytest_assert, pytest_require from tests.common.dualtor.dual_tor_utils import mux_cable_server_ip from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F401 from tests.common.utilities import get_intf_by_sub_intf, wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) @@ -42,6 +42,25 @@ def wrapper(self, *args): return wrapper +@pytest.fixture(autouse=True, scope="module") +def dut_disable_arp_update(rand_selected_dut): + """ + Fixture to disable arp update before the test and re-enable it afterwards + + Args: + rand_selected_dut(AnsibleHost) : dut instance + """ + duthost = rand_selected_dut + if duthost.shell("docker exec -t swss supervisorctl stop arp_update")['stdout_lines'][0] \ + == 'arp_update: ERROR (not running)': + logger.warning("arp_update not running, already disabled") + + yield + + assert duthost.shell("docker exec -t swss supervisorctl start arp_update")['stdout_lines'][0] \ + == 'arp_update: started' + + @pytest.fixture(autouse=True, scope="module") def unknownMacSetup(duthosts, rand_one_dut_hostname, tbinfo): """ @@ -69,6 +88,8 @@ def unknownMacSetup(duthosts, rand_one_dut_hostname, tbinfo): servers = mux_cable_server_ip(duthost) for ips in list(servers.values()): server_ips.append(ips['server_ipv4'].split('/')[0]) + if 'soc_ipv4' in ips: + server_ips.append(ips['soc_ipv4'].split('/')[0]) # populate vlan info vlan = dict() @@ -153,7 +174,8 @@ def flushArpFdb(duthosts, rand_one_dut_hostname): @pytest.fixture(autouse=True) def populateArp(unknownMacSetup, flushArpFdb, ptfhost, duthosts, rand_one_dut_hostname, - toggle_all_simulator_ports_to_rand_selected_tor_m): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 + setup_standby_ports_on_rand_unselected_tor_unconditionally): # noqa F811 """ Fixture to populate ARP entry on the DUT for the traffic destination @@ -237,7 +259,7 @@ class TrafficSendVerify(object): """ Send traffic and check interface counters and ptf ports """ @initClassVars def __init__(self, duthost, ptfadapter, dst_ip, ptf_dst_port, ptf_vlan_ports, - intfs, ptf_ports, arp_entry, dscp): + intfs, ptf_ports, arp_entry, dscp, skip_traffic_test): # noqa F811 """ Args: duthost(AnsibleHost) : dut instance @@ -255,6 +277,7 @@ def __init__(self, duthost, ptfadapter, dst_ip, ptf_dst_port, ptf_vlan_ports, self.pkt_map = dict() self.pre_rx_drops = dict() self.dut_mac = duthost.facts['router_mac'] + self.skip_traffic_test = skip_traffic_test def _constructPacket(self): """ @@ -337,23 +360,24 @@ def runTest(self): self._constructPacket() logger.info("Clear all counters before test run") self.duthost.command("sonic-clear counters") - time.sleep(1) - logger.info("Collect drop counters before test run") - self._verifyIntfCounters(pretest=True) - for pkt, exp_pkt in zip(self.pkts, self.exp_pkts): - self.ptfadapter.dataplane.flush() - out_intf = self.pkt_map[str(pkt)][0] - src_port = self.ptf_ports[out_intf][0] - logger.info("Sending traffic on intf {}".format(out_intf)) - testutils.send(self.ptfadapter, src_port, pkt, count=TEST_PKT_CNT) - testutils.verify_no_packet_any(self.ptfadapter, exp_pkt, ports=self.ptf_vlan_ports) - logger.info("Collect and verify drop counters after test run") - self.verifyIntfCounters() + if not self.skip_traffic_test: + time.sleep(1) + logger.info("Collect drop counters before test run") + self._verifyIntfCounters(pretest=True) + for pkt, exp_pkt in zip(self.pkts, self.exp_pkts): + self.ptfadapter.dataplane.flush() + out_intf = self.pkt_map[str(pkt)][0] + src_port = self.ptf_ports[out_intf][0] + logger.info("Sending traffic on intf {}".format(out_intf)) + testutils.send(self.ptfadapter, src_port, pkt, count=TEST_PKT_CNT) + testutils.verify_no_packet_any(self.ptfadapter, exp_pkt, ports=self.ptf_vlan_ports) + logger.info("Collect and verify drop counters after test run") + self.verifyIntfCounters() class TestUnknownMac(object): @pytest.mark.parametrize("dscp", ["dscp-3", "dscp-4", "dscp-8"]) - def test_unknown_mac(self, unknownMacSetup, dscp, duthosts, rand_one_dut_hostname, ptfadapter): + def test_unknown_mac(self, unknownMacSetup, dscp, duthosts, rand_one_dut_hostname, ptfadapter, skip_traffic_test): # noqa F811 """ Verify unknown mac behavior for lossless and lossy priority @@ -379,6 +403,7 @@ def test_unknown_mac(self, unknownMacSetup, dscp, duthosts, rand_one_dut_hostnam self.ptf_vlan_ports = setup['ptf_vlan_ports'] self.intfs = setup['intfs'] self.ptf_ports = setup['ptf_ports'] + self.skip_traffic_test = skip_traffic_test self.validateEntries() self.run() @@ -396,5 +421,5 @@ def run(self): thandle = TrafficSendVerify(self.duthost, self.ptfadapter, self.dst_ip, self.ptf_dst_port, self.ptf_vlan_ports, self.intfs, self.ptf_ports, - self.arp_entry, self.dscp) + self.arp_entry, self.dscp, self.skip_traffic_test) thandle.runTest() diff --git a/tests/arp/test_wr_arp.py b/tests/arp/test_wr_arp.py index 70b2d4a11c..92dd027220 100644 --- a/tests/arp/test_wr_arp.py +++ b/tests/arp/test_wr_arp.py @@ -1,16 +1,15 @@ -import json import logging import pytest -from tests.common.helpers.assertions import pytest_assert from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 from tests.common.fixtures.ptfhost_utils import remove_ip_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.storage_backend.backend_utils import skip_test_module_over_backend_topologies # noqa F401 from tests.ptf_runner import ptf_runner from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 - +from tests.common.arp_utils import setupFerret, teardownRouteToPtfhost, setupRouteToPtfhost, \ + PTFRUNNER_QLEN, VXLAN_CONFIG_FILE, DEFAULT_TEST_DURATION, testWrArp logger = logging.getLogger(__name__) @@ -19,291 +18,103 @@ pytest.mark.disable_loganalyzer ] -# Globals -PTFRUNNER_QLEN = 1000 -VXLAN_CONFIG_FILE = '/tmp/vxlan_decap.json' -DEFAULT_TEST_DURATION = 370 - - -class TestWrArp: - ''' - TestWrArp Performs control plane assisted warm-reboot - ''' - def __prepareVxlanConfigData(self, duthost, ptfhost, tbinfo): - ''' - Prepares Vxlan Configuration data for Ferret service running on PTF host - - Args: - duthost (AnsibleHost): Device Under Test (DUT) - ptfhost (AnsibleHost): Packet Test Framework (PTF) - - Returns: - None - ''' - mgFacts = duthost.get_extended_minigraph_facts(tbinfo) - vlan_facts = duthost.vlan_facts()['ansible_facts']['ansible_vlan_facts'] - vxlanConfigData = { - 'minigraph_port_indices': mgFacts['minigraph_ptf_indices'], - 'minigraph_portchannel_interfaces': mgFacts['minigraph_portchannel_interfaces'], - 'minigraph_portchannels': mgFacts['minigraph_portchannels'], - 'minigraph_lo_interfaces': mgFacts['minigraph_lo_interfaces'], - 'vlan_facts': vlan_facts, - 'dut_mac': duthost.facts['router_mac'] - } - with open(VXLAN_CONFIG_FILE, 'w') as file: - file.write(json.dumps(vxlanConfigData, indent=4)) - - logger.info('Copying ferret config file to {0}'.format(ptfhost.hostname)) - ptfhost.copy(src=VXLAN_CONFIG_FILE, dest='/tmp/') - - def setupFerret(self, duthost, ptfhost, tbinfo): - ''' - Sets Ferret service on PTF host. This class-scope fixture runs once before test start - - Args: - duthost (AnsibleHost): Device Under Test (DUT) - ptfhost (AnsibleHost): Packet Test Framework (PTF) - - Returns: - None - ''' - ptfhost.copy(src="arp/files/ferret.py", dest="/opt") - - ''' - Get the IP which will be used by ferret script from the "ip route show type unicast" - command output. The output looks as follows: - - default proto 186 src 10.1.0.32 metric 20 - nexthop via 10.0.0.57 dev PortChannel0001 weight 1 - nexthop via 10.0.0.59 dev PortChannel0002 weight 1 - nexthop via 10.0.0.61 dev PortChannel0003 weight 1 - nexthop via 10.0.0.63 dev PortChannel0004 weight 1 - 10.0.0.56/31 dev PortChannel0001 proto kernel scope link src 10.0.0.56 - 10.232.24.0/23 dev eth0 proto kernel scope link src 10.232.24.122 - 100.1.0.29 via 10.0.0.57 dev PortChannel0001 proto 186 src 10.1.0.32 metric 20 - 192.168.0.0/21 dev Vlan1000 proto kernel scope link src 192.168.0.1 - 192.168.8.0/25 proto 186 src 10.1.0.32 metric 20 - nexthop via 10.0.0.57 dev PortChannel0001 weight 1 - nexthop via 10.0.0.59 dev PortChannel0002 weight 1 - nexthop via 10.0.0.61 dev PortChannel0003 weight 1 - nexthop via 10.0.0.63 dev PortChannel0004 weight 1 - 192.168.16.0/25 proto 186 src 10.1.0.32 metric 20 - ... - - We'll use the first subnet IP taken from zebra protocol as the base for the host IP. - As in the new SONiC image the proto will look as '186'(201911) or bgp (master) - instead of 'zebra' (like it looks in 201811)the filtering output command below will pick - the first line containing either 'proto zebra' (or 'proto 186' or 'proto bgp') - (except the one for the deafult route) and take host IP from the subnet IP. For the output - above 192.168.8.0/25 subnet will be taken and host IP given to ferret script will be 192.168.8.1 - ''' - result = duthost.shell( - cmd=r'''ip route show type unicast | - sed -e '/proto 186\|proto zebra\|proto bgp/!d' -e '/default/d' -ne '/0\//p' | - head -n 1 | - sed -ne 's/0\/.*$/1/p' - ''' - ) - - pytest_assert(len(result['stdout'].strip()) > 0, 'Empty DIP returned') - - dip = result['stdout'] - logger.info('VxLan Sender {0}'.format(dip)) - vxlan_port_out = duthost.shell('redis-cli -n 0 hget "SWITCH_TABLE:switch" "vxlan_port"') - if 'stdout' in vxlan_port_out and vxlan_port_out['stdout'].isdigit(): - vxlan_port = int(vxlan_port_out['stdout']) - ferret_args = '-f /tmp/vxlan_decap.json -s {0} -a {1} -p {2}'.format( - dip, duthost.facts["asic_type"], vxlan_port) - else: - ferret_args = '-f /tmp/vxlan_decap.json -s {0} -a {1}'.format(dip, duthost.facts["asic_type"]) +@pytest.fixture(scope='class', autouse=True) +def setupFerretFixture(duthosts, rand_one_dut_hostname, ptfhost, tbinfo): + duthost = duthosts[rand_one_dut_hostname] + setupFerret(duthost, ptfhost, tbinfo) - ptfhost.host.options['variable_manager'].extra_vars.update({'ferret_args': ferret_args}) - logger.info('Copying ferret config file to {0}'.format(ptfhost.hostname)) - ptfhost.template(src='arp/files/ferret.conf.j2', dest='/etc/supervisor/conf.d/ferret.conf') +@pytest.fixture(scope='class', autouse=True) +def clean_dut(duthosts, rand_one_dut_hostname): + duthost = duthosts[rand_one_dut_hostname] + yield + logger.info("Clear ARP cache on DUT") + duthost.command('sonic-clear arp') - logger.info('Generate pem and key files for ssl') - ptfhost.command( - cmd='''openssl req -new -x509 -keyout test.key -out test.pem -days 365 -nodes - -subj "/C=10/ST=Test/L=Test/O=Test/OU=Test/CN=test.com"''', - chdir='/opt' - ) - self.__prepareVxlanConfigData(duthost, ptfhost, tbinfo) +@pytest.fixture(scope='class', autouse=True) +def setupRouteToPtfhostFixture(duthosts, rand_one_dut_hostname, ptfhost): + duthost = duthosts[rand_one_dut_hostname] + route, ptfIp, gwIp = setupRouteToPtfhost(duthost, ptfhost) + yield + teardownRouteToPtfhost(duthost, route, ptfIp, gwIp) - logger.info('Refreshing supervisor control with ferret configuration') - ptfhost.shell('supervisorctl reread && supervisorctl update') - @pytest.fixture(scope='class', autouse=True) - def setupFerretFixture(self, duthosts, rand_one_dut_hostname, ptfhost, tbinfo): - duthost = duthosts[rand_one_dut_hostname] - self.setupFerret(duthost, ptfhost, tbinfo) +def checkWarmbootFlag(duthost): + """ + Checks if warm-reboot system flag is set to false. + """ + warmbootFlag = duthost.shell( + cmd='sonic-db-cli STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable')['stdout'] + logger.info("warmbootFlag: " + warmbootFlag) + return warmbootFlag != 'true' - @pytest.fixture(scope='class', autouse=True) - def clean_dut(self, duthosts, rand_one_dut_hostname): - duthost = duthosts[rand_one_dut_hostname] - yield - logger.info("Clear ARP cache on DUT") - duthost.command('sonic-clear arp') - def setupRouteToPtfhost(self, duthost, ptfhost): - ''' - Sets routes up on DUT to PTF host. This class-scope fixture runs once before test start +@pytest.fixture(scope='class', autouse=True) +def warmRebootSystemFlag(duthost): + """ + Sets warm-reboot system flag to false after test. This class-scope fixture runs once before test start - Args: - duthost (AnsibleHost): Device Under Test (DUT) - ptfhost (AnsibleHost): Packet Test Framework (PTF) + Args: + duthost (AnsibleHost): Device Under Test (DUT) - Returns: - None - ''' - result = duthost.shell(cmd="ip route show table default | sed -n 's/default //p'") - assert len(result['stderr_lines']) == 0, 'Could not find the gateway for management port' + Returns: + None + """ + yield + if not wait_until(300, 10, 0, checkWarmbootFlag, duthost): + logger.info('Setting warm-reboot system flag to false') + duthost.shell(cmd='sonic-db-cli STATE_DB hset "WARM_RESTART_ENABLE_TABLE|system" enable false') - gwIp = result['stdout'] - ptfIp = ptfhost.host.options['inventory_manager'].get_host(ptfhost.hostname).vars['ansible_host'] - route = duthost.shell(cmd='ip route get {0}'.format(ptfIp))['stdout'] - if 'PortChannel' in route: - logger.info( - "Add explicit route for PTF host ({0}) through eth0 (mgmt) interface ({1})".format(ptfIp, gwIp) - ) - duthost.shell(cmd='ip route add {0}/32 {1}'.format(ptfIp, gwIp)) - - return route, ptfIp, gwIp - - def teardownRouteToPtfhost(self, duthost, route, ptfIp, gwIp): - """ - Teardown the routes added by setupRouteToPtfhost - """ - if 'PortChannel' in route: - logger.info( - "Delete explicit route for PTF host ({0}) through eth0 (mgmt) interface ({1})".format(ptfIp, gwIp) - ) - result = duthost.shell(cmd='ip route delete {0}/32 {1}'.format(ptfIp, gwIp), module_ignore_errors=True) - assert result["rc"] == 0 or "No such process" in result["stderr"], \ - "Failed to delete route with error '{0}'".format(result["stderr"]) - - @pytest.fixture(scope='class', autouse=True) - def setupRouteToPtfhostFixture(self, duthosts, rand_one_dut_hostname, ptfhost): - duthost = duthosts[rand_one_dut_hostname] - route, ptfIp, gwIp = self.setupRouteToPtfhost(duthost, ptfhost) - yield - self.teardownRouteToPtfhost(duthost, route, ptfIp, gwIp) - - @pytest.fixture(scope='class', autouse=True) - def warmRebootSystemFlag(self, duthost): - """ - Sets warm-reboot system flag to false after test. This class-scope fixture runs once before test start - - Args: - duthost (AnsibleHost): Device Under Test (DUT) - - Returns: - None - """ - yield - if not wait_until(300, 10, 0, self.checkWarmbootFlag, duthost): - logger.info('Setting warm-reboot system flag to false') - duthost.shell(cmd='sonic-db-cli STATE_DB hset "WARM_RESTART_ENABLE_TABLE|system" enable false') - - def checkWarmbootFlag(self, duthost): - """ - Checks if warm-reboot system flag is set to false. - """ - warmbootFlag = duthost.shell( - cmd='sonic-db-cli STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable')['stdout'] - logger.info("warmbootFlag: " + warmbootFlag) - return warmbootFlag != 'true' - - def Setup(self, duthost, ptfhost, tbinfo): - """ - A setup function that do the exactly same thing as the autoused fixtures do - Will be called in vnet_vxlan test - """ - self.setupFerret(duthost, ptfhost, tbinfo) - self.route, self.ptfIp, self.gwIp = self.setupRouteToPtfhost(duthost, ptfhost) - - def Teardown(self, duthost): - """ - A teardown function that do some cleanup after test - Will be called in vnet_vxlan test - """ - logger.info("Clear ARP cache on DUT") - duthost.command('sonic-clear arp') - self.teardownRouteToPtfhost(duthost, self.route, self.ptfIp, self.gwIp) - - def testWrArp(self, request, duthost, ptfhost, creds): - ''' - Control Plane Assistant test for Warm-Reboot. - - The test first start Ferret server, implemented in Python. Then initiate Warm-Reboot procedure. - While the host in Warm-Reboot test continuously sending ARP request to the Vlan member ports and - expect to receive ARP replies. The test will fail as soon as there is no replies for - more than 25 seconds for one of the Vlan member ports. - - Args: - request: pytest request object - duthost (AnsibleHost): Device Under Test (DUT) - ptfhost (AnsibleHost): Packet Test Framework (PTF) - - Returns: - None - ''' - testDuration = request.config.getoption('--test_duration', default=DEFAULT_TEST_DURATION) - ptfIp = ptfhost.host.options['inventory_manager'].get_host(ptfhost.hostname).vars['ansible_host'] - dutIp = duthost.host.options['inventory_manager'].get_host(duthost.hostname).vars['ansible_host'] +def test_wr_arp(request, duthost, ptfhost, creds, skip_traffic_test): # noqa F811 + ''' + Control Plane Assistant test for Warm-Reboot. - logger.info('Warm-Reboot Control-Plane assist feature') - sonicadmin_alt_password = duthost.host.options['variable_manager'].\ - _hostvars[duthost.hostname]['sonic_default_passwords'] - ptf_runner( - ptfhost, - 'ptftests', - 'wr_arp.ArpTest', - qlen=PTFRUNNER_QLEN, - platform_dir='ptftests', - platform='remote', - params={ - 'ferret_ip': ptfIp, - 'dut_ssh': dutIp, - 'dut_username': creds['sonicadmin_user'], - 'dut_password': creds['sonicadmin_password'], - "alt_password": sonicadmin_alt_password, - 'config_file': VXLAN_CONFIG_FILE, - 'how_long': testDuration, - 'advance': False, - }, - log_file='/tmp/wr_arp.ArpTest.log', - is_python3=True - ) + The test first start Ferret server, implemented in Python. Then initiate Warm-Reboot procedure. + While the host in Warm-Reboot test continuously sending ARP request to the Vlan member ports and + expect to receive ARP replies. The test will fail as soon as there is no replies for + more than 25 seconds for one of the Vlan member ports. - def testWrArpAdvance(self, request, duthost, ptfhost, creds): - testDuration = request.config.getoption('--test_duration', default=DEFAULT_TEST_DURATION) - ptfIp = ptfhost.host.options['inventory_manager'].get_host(ptfhost.hostname).vars['ansible_host'] - dutIp = duthost.host.options['inventory_manager'].get_host(duthost.hostname).vars['ansible_host'] + Args: + request: pytest request object + duthost (AnsibleHost): Device Under Test (DUT) + ptfhost (AnsibleHost): Packet Test Framework (PTF) - logger.info('Warm-Reboot Control-Plane assist feature') - sonicadmin_alt_password = duthost.host.options['variable_manager'].\ - _hostvars[duthost.hostname]['sonic_default_passwords'] - ptf_runner( - ptfhost, - 'ptftests', - 'wr_arp.ArpTest', - qlen=PTFRUNNER_QLEN, - platform_dir='ptftests', - platform='remote', - params={ - 'ferret_ip': ptfIp, - 'dut_ssh': dutIp, - 'dut_username': creds['sonicadmin_user'], - 'dut_password': creds['sonicadmin_password'], - "alt_password": sonicadmin_alt_password, - 'config_file': VXLAN_CONFIG_FILE, - 'how_long': testDuration, - 'advance': True, - }, - log_file='/tmp/wr_arp.ArpTest.Advance.log', - is_python3=True - ) + Returns: + None + ''' + testWrArp(request, duthost, ptfhost, creds, skip_traffic_test) + + +def test_wr_arp_advance(request, duthost, ptfhost, creds, skip_traffic_test): # noqa F811 + testDuration = request.config.getoption('--test_duration', default=DEFAULT_TEST_DURATION) + ptfIp = ptfhost.host.options['inventory_manager'].get_host(ptfhost.hostname).vars['ansible_host'] + dutIp = duthost.host.options['inventory_manager'].get_host(duthost.hostname).vars['ansible_host'] + + logger.info('Warm-Reboot Control-Plane assist feature') + sonicadmin_alt_password = duthost.host.options['variable_manager'].\ + _hostvars[duthost.hostname]['sonic_default_passwords'] + if skip_traffic_test is True: + return + ptf_runner( + ptfhost, + 'ptftests', + 'wr_arp.ArpTest', + qlen=PTFRUNNER_QLEN, + platform_dir='ptftests', + platform='remote', + params={ + 'ferret_ip': ptfIp, + 'dut_ssh': dutIp, + 'dut_username': creds['sonicadmin_user'], + 'dut_password': creds['sonicadmin_password'], + "alt_password": sonicadmin_alt_password, + 'config_file': VXLAN_CONFIG_FILE, + 'how_long': testDuration, + 'advance': True, + }, + log_file='/tmp/wr_arp.ArpTest.Advance.log', + is_python3=True + ) diff --git a/tests/autorestart/test_container_autorestart.py b/tests/autorestart/test_container_autorestart.py index 4ea88ded76..591b2e8a75 100644 --- a/tests/autorestart/test_container_autorestart.py +++ b/tests/autorestart/test_container_autorestart.py @@ -12,7 +12,6 @@ from tests.common.helpers.assertions import pytest_require from tests.common import config_reload from tests.common.helpers.dut_utils import get_disabled_container_list -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) @@ -47,6 +46,7 @@ def config_reload_after_tests(duthosts, selected_rand_one_per_hwsku_hostname, tb and "enabled" not in feature_list.get(DHCP_SERVER, ""): dhcp_server_hosts.append(hostname) duthost.shell("config feature state %s enabled" % DHCP_SERVER) + duthost.shell("sudo config feature autorestart %s enabled" % DHCP_SERVER) duthost.shell("sudo systemctl restart %s.service" % DHCP_RELAY) pytest_require( wait_until(120, 1, 1, diff --git a/tests/bfd/bfd_base.py b/tests/bfd/bfd_base.py index be87686c0d..08b8a39b9f 100644 --- a/tests/bfd/bfd_base.py +++ b/tests/bfd/bfd_base.py @@ -1,237 +1,78 @@ +import logging import random -import sys import pytest -import re -import time -import logging -from tests.platform_tests.cli import util -from tests.common.plugins.sanity_check.checks import _parse_bfd_output + +from tests.bfd.bfd_helpers import modify_all_bfd_sessions, find_bfd_peers_with_given_state +from tests.common import config_reload +from tests.common.platform.processes_utils import wait_critical_processes +from tests.common.utilities import wait_until logger = logging.getLogger(__name__) class BfdBase: - def list_to_dict(self, sample_list): - data_rows = sample_list[3:] - for data in data_rows: - data_dict = {} - if sys.version_info.major < 3: - data = data.encode("utf-8").split() - else: - data = data.split() - - data_dict["Peer Addr"] = data[0] - data_dict["Interface"] = data[1] - data_dict["Vrf"] = data[2] - data_dict["State"] = data[3] - data_dict["Type"] = data[4] - data_dict["Local Addr"] = data[5] - data_dict["TX Interval"] = data[6] - data_dict["RX Interval"] = data[7] - data_dict["Multiplier"] = data[8] - data_dict["Multihop"] = data[9] - data_dict["Local Discriminator"] = data[10] - return data_dict - - def selecting_route_to_delete(self, asic_routes, nexthops): - for asic in asic_routes: - for prefix in asic_routes[asic]: - nexthops_in_static_route_output = asic_routes[asic][prefix] - # If nexthops on source dut are same destination dut's interfaces, we are picking that static route - if sorted(nexthops_in_static_route_output) == sorted(nexthops): - time.sleep(2) - logger.info("Nexthops from static route output") - logger.info(sorted(nexthops_in_static_route_output)) - logger.info("Given Nexthops") - logger.info(sorted(nexthops)) - logger.info("Prefix") - logger.info(prefix) - return prefix - - def modify_all_bfd_sessions(self, dut, flag): - # Extracting asic count - cmd = "show platform summary" - logging.info("Verifying output of '{}' on '{}'...".format(cmd, dut.hostname)) - summary_output_lines = dut.command(cmd)["stdout_lines"] - summary_dict = util.parse_colon_speparated_lines(summary_output_lines) - asic_count = int(summary_dict["ASIC Count"]) - - # Creating bfd.json, bfd0.json, bfd1.json, bfd2.json ... - for i in range(asic_count): - file_name = "config_db{}.json".format(i) - dut.shell("cp /etc/sonic/{} /etc/sonic/{}.bak".format(file_name, file_name)) - if flag == "false": - command = """sed -i 's/"bfd": "true"/"bfd": "false"/' {}""".format( - "/etc/sonic/" + file_name - ) - elif flag == "true": - command = """sed -i 's/"bfd": "false"/"bfd": "true"/' {}""".format( - "/etc/sonic/" + file_name - ) - dut.shell(command) - - def extract_backend_portchannels(self, dut): - output = dut.show_and_parse("show int port -d all") - port_channel_dict = {} - - for item in output: - if "BP" in item.get("ports", ""): - port_channel = item.get("team dev", "") - ports_with_status = [ - port.strip() - for port in item.get("ports", "").split() - if "BP" in port + @pytest.fixture(autouse=True, scope="class") + def modify_bfd_sessions(self, duthosts): + """ + 1. Gather all front end nodes + 2. Modify BFD state to required state & issue config reload. + 3. Wait for Critical processes + 4. Gather all ASICs for each dut + 5. Calls find_bfd_peers_with_given_state using wait_until + a. Runs ip netns exec asic{} show bfd sum + b. If expected state is "Total number of BFD sessions: 0" and it is in result, output is True + c. If expected state is "Up" and no. of down peers is 0, output is True + d. If expected state is "Down" and no. of up peers is 0, output is True + """ + try: + duts = duthosts.frontend_nodes + for dut in duts: + modify_all_bfd_sessions(dut, "false") + for dut in duts: + # config reload + config_reload(dut) + wait_critical_processes(dut) + # Verification that all BFD sessions are deleted + for dut in duts: + asics = [ + asic.split("asic")[1] for asic in dut.get_asic_namespace_list() ] - ports = [ - ( - re.match(r"^([\w-]+)\([A-Za-z]\)", port).group(1) - if re.match(r"^([\w-]+)\([A-Za-z]\)", port) - else None + for asic in asics: + assert wait_until( + 600, + 10, + 0, + lambda: find_bfd_peers_with_given_state( + dut, asic, "No BFD sessions found" + ), ) - for port in ports_with_status - ] - status_match = re.search( - r"LACP\(A\)\((\w+)\)", item.get("protocol", "") - ) - status = status_match.group(1) if status_match else "" - if ports: - port_channel_dict[port_channel] = { - "members": ports, - "status": status, - } - - return port_channel_dict - - def extract_ip_addresses_for_backend_portchannels(self, dut, dut_asic, version): - backend_port_channels = self.extract_backend_portchannels(dut) - if version == "ipv4": - command = "show ip int -d all" - elif version == "ipv6": - command = "show ipv6 int -d all" - data = dut.show_and_parse("{} -n asic{}".format(command, dut_asic.asic_index)) - result_dict = {} - for item in data: - if version == "ipv4": - ip_address = item.get("ipv4 address/mask", "").split("/")[0] - elif version == "ipv6": - ip_address = item.get("ipv6 address/mask", "").split("/")[0] - interface = item.get("interface", "") - - if interface in backend_port_channels: - result_dict[interface] = ip_address - return result_dict - - def delete_bfd(self, asic_number, prefix, dut): - command = "sonic-db-cli -n asic{} CONFIG_DB HSET \"STATIC_ROUTE|{}\" bfd 'false'".format( - asic_number, prefix - ).replace( - "\\", "" - ) - logger.info(command) - dut.shell(command) - time.sleep(15) - - def add_bfd(self, asic_number, prefix, dut): - command = "sonic-db-cli -n asic{} CONFIG_DB HSET \"STATIC_ROUTE|{}\" bfd 'true'".format( - asic_number, prefix - ).replace( - "\\", "" - ) - logger.info(command) - dut.shell(command) - time.sleep(15) - - def extract_current_bfd_state(self, nexthop, asic_number, dut): - bfd_peer_command = "ip netns exec asic{} show bfd peer {}".format( - asic_number, nexthop - ) - logger.info("Verifying BFD status on {}".format(dut)) - logger.info(bfd_peer_command) - bfd_peer_status = dut.shell(bfd_peer_command, module_ignore_errors=True)["stdout"] - if sys.version_info.major < 3: - bfd_peer_output = bfd_peer_status.encode("utf-8").strip().split("\n") - else: - bfd_peer_output = bfd_peer_status.strip().split("\n") - - if "No BFD sessions found" in bfd_peer_output[0]: - return "No BFD sessions found" - else: - entry = self.list_to_dict(bfd_peer_output) - return entry["State"] - - def find_bfd_peers_with_given_state(self, dut, dut_asic, expected_bfd_state): - # Expected BFD states: Up, Down, No BFD sessions found - bfd_cmd = "ip netns exec asic{} show bfd sum" - result = True - asic_bfd_sum = dut.shell(bfd_cmd.format(dut_asic))["stdout"] - if sys.version_info.major < 3: - bfd_peer_output = asic_bfd_sum.encode("utf-8").strip().split("\n") - else: - bfd_peer_output = asic_bfd_sum.strip().split("\n") - - invalid_peers = [] - if any( - keyword in bfd_peer_output[0] - for keyword in ("Total number of BFD sessions: 0", "No BFD sessions found") - ): - return result - else: - bfd_output = _parse_bfd_output(bfd_peer_output) - for peer in bfd_output: - if bfd_output[peer]["State"] != expected_bfd_state: - invalid_peers.append(peer) - - if len(invalid_peers) > 0: - result = False - return result - - def verify_bfd_state(self, dut, dut_nexthops, dut_asic, expected_bfd_state): - logger.info("Verifying BFD state on {} ".format(dut)) - for nexthop in dut_nexthops: - current_bfd_state = self.extract_current_bfd_state( - nexthop, dut_asic.asic_index, dut - ) - logger.info("current_bfd_state: {}".format(current_bfd_state)) - logger.info("expected_bfd_state: {}".format(expected_bfd_state)) - if current_bfd_state != expected_bfd_state: - return False - return True - - def extract_routes(self, static_route_output, version): - asic_routes = {} - asic = None - - for line in static_route_output: - if line.startswith("asic"): - asic = line.split(":")[0] - asic_routes[asic] = {} - elif line.startswith("S>*") or line.startswith(" *"): - parts = line.split(",") - if line.startswith("S>*"): - if version == "ipv4": - prefix_match = re.search(r"(\d+\.\d+\.\d+\.\d+/\d+)", parts[0]) - elif version == "ipv6": - prefix_match = re.search(r"([0-9a-fA-F:.\/]+)", parts[0]) - if prefix_match: - prefix = prefix_match.group(1) - else: - continue - if version == "ipv4": - next_hop_match = re.search(r"via\s+(\d+\.\d+\.\d+\.\d+)", parts[0]) - elif version == "ipv6": - next_hop_match = re.search(r"via\s+([0-9a-fA-F:.\/]+)", parts[0]) - if next_hop_match: - next_hop = next_hop_match.group(1) - else: - continue - asic_routes[asic].setdefault(prefix, []).append(next_hop) - return asic_routes + yield + + finally: + duts = duthosts.frontend_nodes + for dut in duts: + modify_all_bfd_sessions(dut, "true") + for dut in duts: + config_reload(dut) + wait_critical_processes(dut) + # Verification that all BFD sessions are added + for dut in duts: + asics = [ + asic.split("asic")[1] for asic in dut.get_asic_namespace_list() + ] + for asic in asics: + assert wait_until( + 600, + 10, + 0, + lambda: find_bfd_peers_with_given_state( + dut, asic, "Up" + ), + ) - @pytest.fixture( - scope="class", name="select_src_dst_dut_and_asic", params=(["multi_dut"]) - ) + @pytest.fixture(scope="class", name="select_src_dst_dut_and_asic", params=(["multi_dut"])) def select_src_dst_dut_and_asic(self, duthosts, request, tbinfo): if (len(duthosts.frontend_nodes)) < 2: pytest.skip("Don't have 2 frontend nodes - so can't run multi_dut tests") @@ -242,13 +83,13 @@ def select_src_dst_dut_and_asic(self, duthosts, request, tbinfo): # Random selection of source asic based on number of asics available on source dut src_asic_index_selection = random.choice( - duthosts[src_dut_index].get_asic_namespace_list() + duthosts.frontend_nodes[src_dut_index].get_asic_namespace_list() ) src_asic_index = src_asic_index_selection.split("asic")[1] # Random selection of destination asic based on number of asics available on destination dut dst_asic_index_selection = random.choice( - duthosts[dst_dut_index].get_asic_namespace_list() + duthosts.frontend_nodes[dst_dut_index].get_asic_namespace_list() ) dst_asic_index = dst_asic_index_selection.split("asic")[1] @@ -289,3 +130,52 @@ def get_src_dst_asic_and_duts(self, duthosts, select_src_dst_dut_and_asic): } rtn_dict.update(select_src_dst_dut_and_asic) yield rtn_dict + + @pytest.fixture(scope="class") + def select_dut_and_src_dst_asic_index(self, duthosts): + if not duthosts.frontend_nodes: + pytest.skip("DUT does not have any frontend nodes") + + dut_index = random.choice(list(range(len(duthosts.frontend_nodes)))) + asic_namespace_list = duthosts.frontend_nodes[dut_index].get_asic_namespace_list() + if len(asic_namespace_list) < 2: + pytest.skip("DUT does not have more than one ASICs") + + # Random selection of src asic & dst asic on DUT + src_asic_namespace, dst_asic_namespace = random.sample(asic_namespace_list, 2) + src_asic_index = src_asic_namespace.split("asic")[1] + dst_asic_index = dst_asic_namespace.split("asic")[1] + + yield { + "dut_index": dut_index, + "src_asic_index": int(src_asic_index), + "dst_asic_index": int(dst_asic_index), + } + + @pytest.fixture(scope="class") + def get_src_dst_asic(self, request, duthosts, select_dut_and_src_dst_asic_index): + logger.info("Printing select_dut_and_src_dst_asic_index") + logger.info(select_dut_and_src_dst_asic_index) + + logger.info("Printing duthosts.frontend_nodes") + logger.info(duthosts.frontend_nodes) + dut = duthosts.frontend_nodes[select_dut_and_src_dst_asic_index["dut_index"]] + + logger.info("Printing dut asics") + logger.info(dut.asics) + + src_asic = dut.asics[select_dut_and_src_dst_asic_index["src_asic_index"]] + dst_asic = dut.asics[select_dut_and_src_dst_asic_index["dst_asic_index"]] + + request.config.src_asic = src_asic + request.config.dst_asic = dst_asic + request.config.dut = dut + + rtn_dict = { + "src_asic": src_asic, + "dst_asic": dst_asic, + "dut": dut, + } + + rtn_dict.update(select_dut_and_src_dst_asic_index) + yield rtn_dict diff --git a/tests/bfd/bfd_helpers.py b/tests/bfd/bfd_helpers.py index db1e3381d1..154eb01a13 100644 --- a/tests/bfd/bfd_helpers.py +++ b/tests/bfd/bfd_helpers.py @@ -1,13 +1,39 @@ import logging +import random +import re import sys +import time + +import pytest +from ptf import testutils from tests.common.utilities import wait_until logger = logging.getLogger(__name__) +def get_dut_asic_static_routes(version, dut): + if version == "ipv4": + static_route_command = "show ip route static" + elif version == "ipv6": + static_route_command = "show ipv6 route static" + else: + assert False, "Invalid version" + + stdout = dut.shell(static_route_command, module_ignore_errors=True)["stdout"] + if sys.version_info.major < 3: + static_routes_output = stdout.encode("utf-8").strip().split("\n") + else: + static_routes_output = stdout.strip().split("\n") + + asic_static_routes = extract_routes(static_routes_output, version) + logger.info("asic routes, {}".format(asic_static_routes)) + assert len(asic_static_routes) > 0, "static routes on dut are empty" + return asic_static_routes + + def select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version + request, get_src_dst_asic_and_duts, version ): logger.debug("Selecting source and destination DUTs with ASICs...") # Random selection of dut & asic. @@ -26,41 +52,12 @@ def select_src_dst_dut_with_asic( request.config.src_dut = src_dut request.config.dst_dut = dst_dut - # Extracting static routes - if version == "ipv4": - static_route_command = "show ip route static" - elif version == "ipv6": - static_route_command = "show ipv6 route static" - else: - assert False, "Invalid version" - - src_dut_static_route = src_dut.shell(static_route_command, module_ignore_errors=True)["stdout"] - if sys.version_info.major < 3: - src_dut_static_route_output = src_dut_static_route.encode("utf-8").strip().split("\n") - else: - src_dut_static_route_output = src_dut_static_route.strip().split("\n") - - src_asic_routes = bfd_base_instance.extract_routes( - src_dut_static_route_output, version - ) - logger.info("Source asic routes, {}".format(src_asic_routes)) - assert len(src_asic_routes) > 0, "static routes on source dut are empty" - - dst_dut_static_route = dst_dut.shell(static_route_command, module_ignore_errors=True)["stdout"] - if sys.version_info.major < 3: - dst_dut_static_route_output = dst_dut_static_route.encode("utf-8").strip().split("\n") - else: - dst_dut_static_route_output = dst_dut_static_route.strip().split("\n") - - dst_asic_routes = bfd_base_instance.extract_routes( - dst_dut_static_route_output, version - ) - logger.info("Destination asic routes, {}".format(dst_asic_routes)) - assert len(dst_asic_routes) > 0, "static routes on destination dut are empty" + src_asic_routes = get_dut_asic_static_routes(version, src_dut) + dst_asic_routes = get_dut_asic_static_routes(version, dst_dut) # Extracting nexthops dst_dut_nexthops = ( - bfd_base_instance.extract_ip_addresses_for_backend_portchannels( + extract_ip_addresses_for_backend_portchannels( src_dut, src_asic, version ) ) @@ -68,7 +65,7 @@ def select_src_dst_dut_with_asic( assert len(dst_dut_nexthops) != 0, "Destination Nexthops are empty" src_dut_nexthops = ( - bfd_base_instance.extract_ip_addresses_for_backend_portchannels( + extract_ip_addresses_for_backend_portchannels( dst_dut, dst_asic, version ) ) @@ -76,14 +73,14 @@ def select_src_dst_dut_with_asic( assert len(src_dut_nexthops) != 0, "Source Nexthops are empty" # Picking a static route to delete correspinding BFD session - src_prefix = bfd_base_instance.selecting_route_to_delete( + src_prefix = selecting_route_to_delete( src_asic_routes, src_dut_nexthops.values() ) logger.info("Source prefix: %s", src_prefix) request.config.src_prefix = src_prefix assert src_prefix is not None and src_prefix != "", "Source prefix not found" - dst_prefix = bfd_base_instance.selecting_route_to_delete( + dst_prefix = selecting_route_to_delete( dst_asic_routes, dst_dut_nexthops.values() ) logger.info("Destination prefix: %s", dst_prefix) @@ -104,13 +101,25 @@ def select_src_dst_dut_with_asic( ) +def verify_bfd_state(dut, dut_nexthops, dut_asic, expected_bfd_state): + logger.info("Verifying BFD state on {} ".format(dut)) + for nexthop in dut_nexthops: + current_bfd_state = extract_current_bfd_state( + nexthop, dut_asic.asic_index, dut + ) + logger.info("current_bfd_state: {}".format(current_bfd_state)) + logger.info("expected_bfd_state: {}".format(expected_bfd_state)) + if current_bfd_state != expected_bfd_state: + return False + return True + + def verify_static_route( request, asic, prefix, dut, expected_prefix_state, - bfd_base_instance, version, ): # Verification of static route @@ -127,7 +136,7 @@ def verify_static_route( else: static_route_output = static_route.strip().split("\n") - asic_routes = bfd_base_instance.extract_routes(static_route_output, version) + asic_routes = extract_routes(static_route_output, version) logger.info("Here are asic routes, {}".format(asic_routes)) if expected_prefix_state == "Route Removal": @@ -189,13 +198,676 @@ def control_interface_state(dut, asic, interface, action): def check_bgp_status(request): check_bgp = request.getfixturevalue("check_bgp") results = check_bgp() - bgp_failures = [] - for result in results: - if "failed" in result and result["failed"]: - bgp_failures.append(result) - - if bgp_failures: - logger.info("BGP check failed: {}".format(bgp_failures)) - return False + failed = [ + result for result in results if "failed" in result and result["failed"] + ] + if failed: + pytest.fail( + "BGP check failed, not all BGP sessions are up. Failed: {}".format(failed) + ) + + +def selecting_route_to_delete(asic_routes, nexthops): + for asic in asic_routes: + for prefix in asic_routes[asic]: + nexthops_in_static_route_output = asic_routes[asic][prefix] + # If nexthops on source dut are same destination dut's interfaces, we are picking that static route + if sorted(nexthops_in_static_route_output) == sorted(nexthops): + time.sleep(2) + logger.info("Nexthops from static route output") + logger.info(sorted(nexthops_in_static_route_output)) + logger.info("Given Nexthops") + logger.info(sorted(nexthops)) + logger.info("Prefix") + logger.info(prefix) + return prefix + + +def parse_colon_speparated_lines(lines): + """ + @summary: Helper function for parsing lines which consist of key-value pairs + formatted like ": ", where the colon can be surrounded + by 0 or more whitespace characters + @return: A dictionary containing key-value pairs of the output + """ + res = {} + for line in lines: + fields = line.split(":") + if len(fields) != 2: + continue + res[fields[0].strip()] = fields[1].strip() + return res + + +def modify_all_bfd_sessions(dut, flag): + # Extracting asic count + cmd = "show platform summary" + logging.info("Verifying output of '{}' on '{}'...".format(cmd, dut.hostname)) + summary_output_lines = dut.command(cmd)["stdout_lines"] + summary_dict = parse_colon_speparated_lines(summary_output_lines) + asic_count = int(summary_dict["ASIC Count"]) + + # Creating bfd.json, bfd0.json, bfd1.json, bfd2.json ... + for i in range(asic_count): + file_name = "config_db{}.json".format(i) + dut.shell("cp /etc/sonic/{} /etc/sonic/{}.bak".format(file_name, file_name)) + if flag == "false": + command = """sed -i 's/"bfd": "true"/"bfd": "false"/' {}""".format( + "/etc/sonic/" + file_name + ) + elif flag == "true": + command = """sed -i 's/"bfd": "false"/"bfd": "true"/' {}""".format( + "/etc/sonic/" + file_name + ) + dut.shell(command) + + +def extract_backend_portchannels(dut): + output = dut.show_and_parse("show int port -d all") + port_channel_dict = {} + + for item in output: + if "BP" in item.get("ports", ""): + port_channel = item.get("team dev", "") + ports_with_status = [ + port.strip() + for port in item.get("ports", "").split() + if "BP" in port + ] + ports = [ + ( + re.match(r"^([\w-]+)\([A-Za-z]\)", port).group(1) + if re.match(r"^([\w-]+)\([A-Za-z]\)", port) + else None + ) + for port in ports_with_status + ] + status_match = re.search( + r"LACP\(A\)\((\w+)\)", item.get("protocol", "") + ) + status = status_match.group(1) if status_match else "" + if ports: + port_channel_dict[port_channel] = { + "members": ports, + "status": status, + } + + return port_channel_dict + + +def extract_ip_addresses_for_backend_portchannels(dut, dut_asic, version, backend_port_channels=None): + if not backend_port_channels: + backend_port_channels = extract_backend_portchannels(dut) + + if version == "ipv4": + command = "show ip int -d all" + elif version == "ipv6": + command = "show ipv6 int -d all" + output = dut.show_and_parse("{} -n asic{}".format(command, dut_asic.asic_index)) + result_dict = {} + for item in output: + if version == "ipv4": + ip_address = item.get("ipv4 address/mask", "").split("/")[0] + elif version == "ipv6": + ip_address = item.get("ipv6 address/mask", "").split("/")[0] + interface = item.get("interface", "") + + if interface in backend_port_channels: + result_dict[interface] = ip_address + return result_dict + + +def delete_bfd(asic_number, prefix, dut): + command = "sonic-db-cli -n asic{} CONFIG_DB HSET \"STATIC_ROUTE|{}\" bfd 'false'".format( + asic_number, prefix + ).replace( + "\\", "" + ) + logger.info(command) + dut.shell(command) + time.sleep(15) + + +def add_bfd(asic_number, prefix, dut): + command = "sonic-db-cli -n asic{} CONFIG_DB HSET \"STATIC_ROUTE|{}\" bfd 'true'".format( + asic_number, prefix + ).replace( + "\\", "" + ) + logger.info(command) + dut.shell(command) + time.sleep(15) + + +def list_to_dict(sample_list): + data_rows = sample_list[3:] + for data in data_rows: + data_dict = {} + if sys.version_info.major < 3: + data = data.encode("utf-8").split() + else: + data = data.split() + + data_dict["Peer Addr"] = data[0] + data_dict["Interface"] = data[1] + data_dict["Vrf"] = data[2] + data_dict["State"] = data[3] + data_dict["Type"] = data[4] + data_dict["Local Addr"] = data[5] + data_dict["TX Interval"] = data[6] + data_dict["RX Interval"] = data[7] + data_dict["Multiplier"] = data[8] + data_dict["Multihop"] = data[9] + data_dict["Local Discriminator"] = data[10] + return data_dict + + +def extract_current_bfd_state(nexthop, asic_number, dut): + bfd_peer_command = "ip netns exec asic{} show bfd peer {}".format( + asic_number, nexthop + ) + logger.info("Verifying BFD status on {}".format(dut)) + logger.info(bfd_peer_command) + bfd_peer_status = dut.shell(bfd_peer_command, module_ignore_errors=True)["stdout"] + if sys.version_info.major < 3: + bfd_peer_output = bfd_peer_status.encode("utf-8").strip().split("\n") + else: + bfd_peer_output = bfd_peer_status.strip().split("\n") + + if "No BFD sessions found" in bfd_peer_output[0]: + return "No BFD sessions found" + else: + entry = list_to_dict(bfd_peer_output) + return entry["State"] + + +def parse_bfd_output(output): + data_rows = output[3:] + data_dict = {} + for data in data_rows: + data = data.split() + data_dict[data[0]] = {} + data_dict[data[0]]['Interface'] = data[1] + data_dict[data[0]]['Vrf'] = data[2] + data_dict[data[0]]['State'] = data[3] + data_dict[data[0]]['Type'] = data[4] + data_dict[data[0]]['Local Addr'] = data[5] + data_dict[data[0]]['TX Interval'] = data[6] + data_dict[data[0]]['RX Interval'] = data[7] + data_dict[data[0]]['Multiplier'] = data[8] + data_dict[data[0]]['Multihop'] = data[9] + data_dict[data[0]]['Local Discriminator'] = data[10] + return data_dict + + +def find_bfd_peers_with_given_state(dut, dut_asic, expected_bfd_state): + # Expected BFD states: Up, Down, No BFD sessions found + bfd_cmd = "ip netns exec asic{} show bfd sum" + result = True + asic_bfd_sum = dut.shell(bfd_cmd.format(dut_asic))["stdout"] + if sys.version_info.major < 3: + bfd_peer_output = asic_bfd_sum.encode("utf-8").strip().split("\n") + else: + bfd_peer_output = asic_bfd_sum.strip().split("\n") + + invalid_peers = [] + if any( + keyword in bfd_peer_output[0] + for keyword in ("Total number of BFD sessions: 0", "No BFD sessions found") + ): + return result + else: + bfd_output = parse_bfd_output(bfd_peer_output) + for peer in bfd_output: + if bfd_output[peer]["State"] != expected_bfd_state: + invalid_peers.append(peer) + + if len(invalid_peers) > 0: + result = False + return result + + +def extract_routes(static_route_output, version): + asic_routes = {} + asic = None + + for line in static_route_output: + if line.startswith("asic"): + asic = line.split(":")[0] + asic_routes[asic] = {} + elif line.startswith("S>*") or line.startswith(" *"): + parts = line.split(",") + if line.startswith("S>*"): + if version == "ipv4": + prefix_match = re.search(r"(\d+\.\d+\.\d+\.\d+/\d+)", parts[0]) + elif version == "ipv6": + prefix_match = re.search(r"([0-9a-fA-F:.\/]+)", parts[0]) + if prefix_match: + prefix = prefix_match.group(1) + else: + continue + if version == "ipv4": + next_hop_match = re.search(r"via\s+(\d+\.\d+\.\d+\.\d+)", parts[0]) + elif version == "ipv6": + next_hop_match = re.search(r"via\s+([0-9a-fA-F:.\/]+)", parts[0]) + if next_hop_match: + next_hop = next_hop_match.group(1) + else: + continue + + asic_routes[asic].setdefault(prefix, []).append(next_hop) + return asic_routes + + +def ensure_interface_is_up(dut, asic, interface): + int_oper_status = dut.show_interface( + command="status", include_internal_intfs=True, asic_index=asic.asic_index + )["ansible_facts"]["int_status"][interface]["oper_state"] + if int_oper_status == "down": + logger.info( + "Starting downed interface {} on {} asic{}".format(interface, dut, asic.asic_index) + ) + exec_cmd = ( + "sudo ip netns exec asic{} config interface -n asic{} startup {}".format( + asic.asic_index, asic.asic_index, interface + ) + ) + + logger.info("Command: {}".format(exec_cmd)) + dut.shell(exec_cmd) + assert wait_until( + 180, + 10, + 0, + lambda: dut.show_interface( + command="status", + include_internal_intfs=True, + asic_index=asic.asic_index, + )["ansible_facts"]["int_status"][interface]["oper_state"] == "up", + ) + + +def prepare_traffic_test_variables(get_src_dst_asic, request, version): + dut = get_src_dst_asic["dut"] + src_asic = get_src_dst_asic["src_asic"] + src_asic_index = get_src_dst_asic["src_asic_index"] + dst_asic = get_src_dst_asic["dst_asic"] + dst_asic_index = get_src_dst_asic["dst_asic_index"] + logger.info( + "DUT: {}, src_asic_index: {}, dst_asic_index: {}".format(dut.hostname, src_asic_index, dst_asic_index) + ) + + backend_port_channels = extract_backend_portchannels(dut) + src_asic_next_hops, dst_asic_next_hops, src_prefix, dst_prefix = get_src_dst_asic_next_hops( + version, + dut, + src_asic, + dst_asic, + request, + backend_port_channels, + ) + + add_bfd(src_asic_index, src_prefix, dut) + add_bfd(dst_asic_index, dst_prefix, dut) + assert wait_until( + 180, + 10, + 0, + lambda: verify_bfd_state(dut, src_asic_next_hops.values(), src_asic, "Up"), + ) + assert wait_until( + 180, + 10, + 0, + lambda: verify_bfd_state(dut, dst_asic_next_hops.values(), dst_asic, "Up"), + ) + + src_asic_router_mac = src_asic.get_router_mac() + + return ( + dut, + src_asic, + src_asic_index, + dst_asic, + dst_asic_index, + src_asic_next_hops, + dst_asic_next_hops, + src_asic_router_mac, + backend_port_channels, + ) + + +def clear_bfd_configs(dut, asic_index, prefix): + logger.info("Clearing BFD configs on {}".format(dut)) + command = ( + "sonic-db-cli -n asic{} CONFIG_DB HSET \"STATIC_ROUTE|{}\" bfd 'false'".format( + asic_index, + prefix, + ).replace("\\", "") + ) + + dut.shell(command) + + +def get_random_bgp_neighbor_ip_of_asic(dut, asic_index, version): + command = "show ip bgp sum" + if version == "ipv4": + command = "show ip bgp sum" + elif version == "ipv6": + command = "show ipv6 bgp sum" + + output = dut.show_and_parse("{} -n asic{}".format(command, asic_index)) + if not output: + return None + + random_neighbor_bgp_info = random.choice(output) + if "neighbhor" in random_neighbor_bgp_info: + return random_neighbor_bgp_info["neighbhor"] + elif "neighbor" in random_neighbor_bgp_info: + return random_neighbor_bgp_info["neighbor"] else: - return True + return None + + +def get_bgp_neighbor_ip_of_port_channel(dut, asic_index, port_channel, version): + command = None + if version == "ipv4": + command = "show ip int" + elif version == "ipv6": + command = "show ipv6 int" + + output = dut.show_and_parse("{} -n asic{}".format(command, asic_index)) + for item in output: + if item.get("interface", "") == port_channel: + return item.get("neighbor ip", "") + + return None + + +def get_asic_frontend_port_channels(dut, dst_asic_index): + output = dut.show_and_parse("show int port -n asic{}".format(dst_asic_index)) + asic_port_channel_info = {} + + for item in output: + port_channel = item.get("team dev", "") + ports_with_status = [port.strip() for port in item.get("ports", "").split()] + ports = [ + ( + re.match(r"^([\w-]+)\([A-Za-z]\)", port).group(1) + if re.match(r"^([\w-]+)\([A-Za-z]\)", port) + else None + ) + for port in ports_with_status + ] + status_match = re.search( + r"LACP\(A\)\((\w+)\)", item.get("protocol", "") + ) + status = status_match.group(1) if status_match else "" + if ports: + asic_port_channel_info[port_channel] = { + "members": ports, + "status": status, + } + + return asic_port_channel_info + + +def get_ptf_src_port(src_asic, tbinfo): + src_asic_mg_facts = src_asic.get_extended_minigraph_facts(tbinfo) + ptf_src_ports = list(src_asic_mg_facts["minigraph_ptf_indices"].values()) + if not ptf_src_ports: + pytest.skip("No PTF ports found for asic{}".format(src_asic.asic_index)) + + return random.choice(ptf_src_ports) + + +def clear_interface_counters(dut): + dut.shell("sonic-clear counters") + + +def send_packets_batch_from_ptf( + packet_count, + version, + src_asic_router_mac, + ptfadapter, + ptf_src_port, + dst_neighbor_ip, +): + for _ in range(packet_count): + if version == "ipv4": + pkt = testutils.simple_ip_packet( + eth_dst=src_asic_router_mac, + eth_src=ptfadapter.dataplane.get_mac(0, ptf_src_port), + # ip_src=src_ip_addr, + ip_dst=dst_neighbor_ip, + ip_proto=47, + ip_tos=0x84, + ip_id=0, + ip_ihl=5, + ip_ttl=121 + ) + else: + pkt = testutils.simple_ipv6ip_packet( + eth_dst=src_asic_router_mac, + eth_src=ptfadapter.dataplane.get_mac(0, ptf_src_port), + # ip_src=src_ip_addr, + ipv6_dst=dst_neighbor_ip, + ) + + ptfadapter.dataplane.flush() + testutils.send(test=ptfadapter, port_id=ptf_src_port, pkt=pkt) + + +def get_backend_interface_in_use_by_counter( + dut, + packet_count, + version, + src_asic_router_mac, + ptfadapter, + ptf_src_port, + dst_neighbor_ip, + src_asic_index, + dst_asic_index, +): + clear_interface_counters(dut) + send_packets_batch_from_ptf(packet_count, version, src_asic_router_mac, ptfadapter, ptf_src_port, dst_neighbor_ip) + src_output = dut.show_and_parse("show int counters -n asic{} -d all".format(src_asic_index)) + dst_output = dut.show_and_parse("show int counters -n asic{} -d all".format(dst_asic_index)) + src_bp_iface = None + for item in src_output: + if "BP" in item.get("iface", "") and int(item.get("tx_ok", "0").replace(',', '')) >= packet_count: + src_bp_iface = item.get("iface", "") + + dst_bp_iface = None + for item in dst_output: + if "BP" in item.get("iface", "") and int(item.get("rx_ok", "0").replace(',', '')) >= packet_count: + dst_bp_iface = item.get("iface", "") + + if not src_bp_iface or not dst_bp_iface: + pytest.fail("Backend interface in use not found") + + return src_bp_iface, dst_bp_iface + + +def get_src_dst_asic_next_hops(version, dut, src_asic, dst_asic, request, backend_port_channels): + src_asic_next_hops = extract_ip_addresses_for_backend_portchannels(dut, dst_asic, version, backend_port_channels) + assert len(src_asic_next_hops) != 0, "Source next hops are empty" + dst_asic_next_hops = extract_ip_addresses_for_backend_portchannels(dut, src_asic, version, backend_port_channels) + assert len(dst_asic_next_hops) != 0, "Destination next hops are empty" + + dut_asic_static_routes = get_dut_asic_static_routes(version, dut) + + # Picking a static route to delete its BFD session + src_prefix = selecting_route_to_delete(dut_asic_static_routes, src_asic_next_hops.values()) + request.config.src_prefix = src_prefix + assert src_prefix is not None and src_prefix != "", "Source prefix not found" + + dst_prefix = selecting_route_to_delete(dut_asic_static_routes, dst_asic_next_hops.values()) + request.config.dst_prefix = dst_prefix + assert dst_prefix is not None and dst_prefix != "", "Destination prefix not found" + + return src_asic_next_hops, dst_asic_next_hops, src_prefix, dst_prefix + + +def get_port_channel_by_member(backend_port_channels, member): + for port_channel in backend_port_channels: + if member in backend_port_channels[port_channel]["members"]: + return port_channel + + return None + + +def toggle_port_channel_or_member( + target_to_toggle, + dut, + asic, + request, + action, +): + request.config.portchannels_on_dut = "dut" + request.config.selected_portchannels = [target_to_toggle] + request.config.asic = asic + + control_interface_state(dut, asic, target_to_toggle, action) + if action == "shutdown": + time.sleep(120) + + +def assert_bp_iface_after_shutdown( + src_bp_iface_before_shutdown, + dst_bp_iface_before_shutdown, + src_bp_iface_after_shutdown, + dst_bp_iface_after_shutdown, + src_asic_index, + dst_asic_index, + dut_hostname, +): + if src_bp_iface_before_shutdown == src_bp_iface_after_shutdown: + pytest.fail( + "Source backend interface in use on asic{} of dut {} does not change after shutdown".format( + src_asic_index, + dut_hostname, + ) + ) + + if dst_bp_iface_before_shutdown == dst_bp_iface_after_shutdown: + pytest.fail( + "Destination backend interface in use on asic{} of dut {} does not change after shutdown".format( + dst_asic_index, + dut_hostname, + ) + ) + + +def assert_port_channel_after_shutdown( + src_port_channel_before_shutdown, + dst_port_channel_before_shutdown, + src_port_channel_after_shutdown, + dst_port_channel_after_shutdown, + src_asic_index, + dst_asic_index, + dut_hostname, +): + if src_port_channel_before_shutdown == src_port_channel_after_shutdown: + pytest.fail( + "Source port channel in use on asic{} of dut {} does not change after shutdown".format( + src_asic_index, + dut_hostname, + ) + ) + + if dst_port_channel_before_shutdown == dst_port_channel_after_shutdown: + pytest.fail( + "Destination port channel in use on asic{} of dut {} does not change after shutdown".format( + dst_asic_index, + dut_hostname, + ) + ) + + +def verify_given_bfd_state(asic_next_hops, port_channel, asic_index, dut, expected_state): + current_state = extract_current_bfd_state(asic_next_hops[port_channel], asic_index, dut) + return current_state == expected_state + + +def wait_until_given_bfd_down( + src_asic_next_hops, + src_port_channel, + src_asic_index, + dst_asic_next_hops, + dst_port_channel, + dst_asic_index, + dut, +): + assert wait_until( + 180, + 10, + 0, + lambda: verify_given_bfd_state(src_asic_next_hops, dst_port_channel, src_asic_index, dut, "Down"), + ) + + assert wait_until( + 180, + 10, + 0, + lambda: verify_given_bfd_state(dst_asic_next_hops, src_port_channel, dst_asic_index, dut, "Down"), + ) + + +def assert_traffic_switching( + dut, + backend_port_channels, + src_asic_index, + src_bp_iface_before_shutdown, + src_bp_iface_after_shutdown, + src_port_channel_before_shutdown, + dst_asic_index, + dst_bp_iface_after_shutdown, + dst_bp_iface_before_shutdown, + dst_port_channel_before_shutdown, +): + assert_bp_iface_after_shutdown( + src_bp_iface_before_shutdown, + dst_bp_iface_before_shutdown, + src_bp_iface_after_shutdown, + dst_bp_iface_after_shutdown, + src_asic_index, + dst_asic_index, + dut.hostname, + ) + + src_port_channel_after_shutdown = get_port_channel_by_member( + backend_port_channels, + src_bp_iface_after_shutdown, + ) + + dst_port_channel_after_shutdown = get_port_channel_by_member( + backend_port_channels, + dst_bp_iface_after_shutdown, + ) + + assert_port_channel_after_shutdown( + src_port_channel_before_shutdown, + dst_port_channel_before_shutdown, + src_port_channel_after_shutdown, + dst_port_channel_after_shutdown, + src_asic_index, + dst_asic_index, + dut.hostname, + ) + + +def wait_until_bfd_up(dut, src_asic_next_hops, src_asic, dst_asic_next_hops, dst_asic): + assert wait_until( + 180, + 10, + 0, + lambda: verify_bfd_state(dut, src_asic_next_hops.values(), src_asic, "Up"), + ) + + assert wait_until( + 180, + 10, + 0, + lambda: verify_bfd_state(dut, dst_asic_next_hops.values(), dst_asic, "Up"), + ) diff --git a/tests/bfd/conftest.py b/tests/bfd/conftest.py index 84dcdd3547..667c484714 100644 --- a/tests/bfd/conftest.py +++ b/tests/bfd/conftest.py @@ -1,60 +1,52 @@ -import pytest -from bfd_base import BfdBase import logging -from tests.platform_tests.link_flap.link_flap_utils import check_orch_cpu_utilization -from tests.common.utilities import wait_until +import pytest + +from tests.bfd.bfd_helpers import ensure_interface_is_up, clear_bfd_configs from tests.common.config_reload import config_reload +# from tests.common.utilities import wait_until +# from tests.platform_tests.link_flap.link_flap_utils import check_orch_cpu_utilization logger = logging.getLogger(__name__) -@pytest.fixture(scope="class") -def bfd_base_instance(): - return BfdBase() - - def pytest_addoption(parser): parser.addoption("--num_sessions", action="store", default=5) parser.addoption("--num_sessions_scale", action="store", default=128) +@pytest.fixture(scope='module') +def get_function_completeness_level(pytestconfig): + return pytestconfig.getoption("--completeness_level") + + @pytest.fixture(scope="function") -def bfd_cleanup_db( - request, duthosts, enum_supervisor_dut_hostname, bfd_base_instance, autouse=True -): - orch_cpu_threshold = 10 - # Make Sure Orch CPU < orch_cpu_threshold before starting test. - logger.info( - "Make Sure orchagent CPU utilization is less that %d before starting the test", - orch_cpu_threshold, - ) - duts = duthosts.frontend_nodes - for dut in duts: - assert wait_until( - 100, 2, 0, check_orch_cpu_utilization, dut, orch_cpu_threshold - ), "Orch CPU utilization {} > orch cpu threshold {} before starting the test".format( - dut.shell("show processes cpu | grep orchagent | awk '{print $9}'")[ - "stdout" - ], - orch_cpu_threshold, - ) +def bfd_cleanup_db(request, duthosts, enum_supervisor_dut_hostname): + # Temporarily disable orchagent CPU check before starting test as it is not stable + # orch_cpu_threshold = 10 + # # Make Sure Orch CPU < orch_cpu_threshold before starting test. + # logger.info( + # "Make Sure orchagent CPU utilization is less that %d before starting the test", + # orch_cpu_threshold, + # ) + # duts = duthosts.frontend_nodes + # for dut in duts: + # assert wait_until( + # 100, 2, 0, check_orch_cpu_utilization, dut, orch_cpu_threshold + # ), "Orch CPU utilization exceeds orch cpu threshold {} before starting the test".format(orch_cpu_threshold) yield - orch_cpu_threshold = 10 - # Orchagent CPU should consume < orch_cpu_threshold at last. - logger.info( - "watch orchagent CPU utilization when it goes below %d", orch_cpu_threshold - ) - for dut in duts: - assert wait_until( - 120, 4, 0, check_orch_cpu_utilization, dut, orch_cpu_threshold - ), "Orch CPU utilization {} > orch cpu threshold {} after the test".format( - dut.shell("show processes cpu | grep orchagent | awk '{print $9}'")[ - "stdout" - ], - orch_cpu_threshold, - ) + + # Temporarily disable orchagent CPU check after finishing test as it is not stable + # orch_cpu_threshold = 10 + # # Orchagent CPU should consume < orch_cpu_threshold at last. + # logger.info( + # "watch orchagent CPU utilization when it goes below %d", orch_cpu_threshold + # ) + # for dut in duts: + # assert wait_until( + # 120, 4, 0, check_orch_cpu_utilization, dut, orch_cpu_threshold + # ), "Orch CPU utilization exceeds orch cpu threshold {} after finishing the test".format(orch_cpu_threshold) logger.info("Verifying swss container status on RP") rp = duthosts[enum_supervisor_dut_hostname] @@ -69,23 +61,12 @@ def bfd_cleanup_db( if not container_status: config_reload(rp) - logger.info( - "Clearing BFD configs on {}, {}".format( - request.config.src_dut, request.config.dst_dut - ) - ) - command = ( - "sonic-db-cli -n asic{} CONFIG_DB HSET \"STATIC_ROUTE|{}\" bfd 'false'".format( - request.config.src_asic.asic_index, request.config.src_prefix - ).replace("\\", "") - ) - request.config.src_dut.shell(command) - command = ( - "sonic-db-cli -n asic{} CONFIG_DB HSET \"STATIC_ROUTE|{}\" bfd 'false'".format( - request.config.dst_asic.asic_index, request.config.dst_prefix - ).replace("\\", "") - ) - request.config.dst_dut.shell(command) + if hasattr(request.config, "src_dut") and hasattr(request.config, "dst_dut"): + clear_bfd_configs(request.config.src_dut, request.config.src_asic.asic_index, request.config.src_prefix) + clear_bfd_configs(request.config.dst_dut, request.config.dst_asic.asic_index, request.config.dst_prefix) + elif hasattr(request.config, "dut"): + clear_bfd_configs(request.config.dut, request.config.src_asic.asic_index, request.config.src_prefix) + clear_bfd_configs(request.config.dut, request.config.dst_asic.asic_index, request.config.dst_prefix) logger.info("Bringing up portchannels or respective members") portchannels_on_dut = None @@ -102,43 +83,19 @@ def bfd_cleanup_db( selected_interfaces = [] if selected_interfaces: - dut = ( - request.config.src_dut - if portchannels_on_dut == "src" - else request.config.dst_dut - ) - asic = ( - request.config.src_asic - if portchannels_on_dut == "src" - else request.config.dst_asic - ) + if portchannels_on_dut == "src": + dut = request.config.src_dut + elif portchannels_on_dut == "dst": + dut = request.config.dst_dut + else: + dut = request.config.dut + + if portchannels_on_dut == "src": + asic = request.config.src_asic + elif portchannels_on_dut == "dst": + asic = request.config.dst_asic + else: + asic = request.config.asic + for interface in selected_interfaces: ensure_interface_is_up(dut, asic, interface) - - -def ensure_interface_is_up(dut, asic, interface): - int_oper_status = dut.show_interface( - command="status", include_internal_intfs=True, asic_index=asic.asic_index - )["ansible_facts"]["int_status"][interface]["oper_state"] - if int_oper_status == "down": - logger.info( - "Starting downed interface {} on {} asic{}".format(interface, dut, asic.asic_index) - ) - exec_cmd = ( - "sudo ip netns exec asic{} config interface -n asic{} startup {}".format( - asic.asic_index, asic.asic_index, interface - ) - ) - - logger.info("Command: {}".format(exec_cmd)) - dut.shell(exec_cmd) - assert wait_until( - 180, - 10, - 0, - lambda: dut.show_interface( - command="status", - include_internal_intfs=True, - asic_index=asic.asic_index, - )["ansible_facts"]["int_status"][interface]["oper_state"] == "up", - ) diff --git a/tests/bfd/test_bfd.py b/tests/bfd/test_bfd.py index 13d72fea1a..43d95e629a 100644 --- a/tests/bfd/test_bfd.py +++ b/tests/bfd/test_bfd.py @@ -6,10 +6,10 @@ from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F401 from tests.common.snappi_tests.common_helpers import get_egress_queue_count -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ - pytest.mark.topology('t1') + pytest.mark.topology('t1'), + pytest.mark.device_type('physical') ] BFD_RESPONDER_SCRIPT_SRC_PATH = '../ansible/roles/test/files/helpers/bfd_responder.py' diff --git a/tests/bfd/test_bfd_static_route.py b/tests/bfd/test_bfd_static_route.py index d102db46a4..64ae9304c4 100644 --- a/tests/bfd/test_bfd_static_route.py +++ b/tests/bfd/test_bfd_static_route.py @@ -1,234 +1,48 @@ -import pytest -from bfd_base import BfdBase import logging import time +import pytest + +from tests.bfd.bfd_base import BfdBase from tests.bfd.bfd_helpers import verify_static_route, select_src_dst_dut_with_asic, control_interface_state, \ - check_bgp_status -from tests.common.utilities import wait_until + check_bgp_status, add_bfd, verify_bfd_state, delete_bfd, extract_backend_portchannels from tests.common.config_reload import config_reload from tests.common.platform.processes_utils import wait_critical_processes from tests.common.reboot import reboot -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from tests.common.utilities import wait_until -pytestmark = [pytest.mark.topology("t2")] +pytestmark = [ + pytest.mark.topology("t2"), + pytest.mark.device_type('physical') +] logger = logging.getLogger(__name__) class TestBfdStaticRoute(BfdBase): - TOTAL_ITERATIONS = 100 - - @pytest.fixture(autouse=True, scope="class") - def modify_bfd_sessions(self, duthosts, bfd_base_instance): - """ - 1. Gather all front end nodes - 2. Modify BFD state to required state & issue config reload. - 3. Wait for Critical processes - 4. Gather all ASICs for each dut - 5. Calls find_bfd_peers_with_given_state using wait_until - a. Runs ip netns exec asic{} show bfd sum - b. If expected state is "Total number of BFD sessions: 0" and it is in result, output is True - c. If expected state is "Up" and no. of down peers is 0, output is True - d. If expected state is "Down" and no. of up peers is 0, output is True - """ - try: - duts = duthosts.frontend_nodes - for dut in duts: - bfd_base_instance.modify_all_bfd_sessions(dut, "false") - for dut in duts: - # config reload - config_reload(dut) - wait_critical_processes(dut) - # Verification that all BFD sessions are deleted - for dut in duts: - asics = [ - asic.split("asic")[1] for asic in dut.get_asic_namespace_list() - ] - for asic in asics: - assert wait_until( - 600, - 10, - 0, - lambda: bfd_base_instance.find_bfd_peers_with_given_state( - dut, asic, "No BFD sessions found" - ), - ) - - yield - - finally: - duts = duthosts.frontend_nodes - for dut in duts: - bfd_base_instance.modify_all_bfd_sessions(dut, "true") - for dut in duts: - config_reload(dut) - wait_critical_processes(dut) - # Verification that all BFD sessions are added - for dut in duts: - asics = [ - asic.split("asic")[1] for asic in dut.get_asic_namespace_list() - ] - for asic in asics: - assert wait_until( - 600, - 10, - 0, - lambda: bfd_base_instance.find_bfd_peers_with_given_state( - dut, asic, "Up" - ), - ) - - def test_bfd_with_lc_reboot_ipv4( - self, - localhost, - duthost, - request, - duthosts, - tbinfo, - get_src_dst_asic_and_duts, - bfd_base_instance, - bfd_cleanup_db, - ): - """ - Author: Harsha Golla - Email : harsgoll@cisco.com - """ - - version = "ipv4" - - # Selecting source, destination dut & prefix & BFD status verification for all nexthops - logger.info( - "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" - ) - ( - src_asic, - dst_asic, - src_dut, - dst_dut, - src_dut_nexthops, - dst_dut_nexthops, - src_prefix, - dst_prefix, - ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version - ) - - # Creation of BFD - logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - - logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Verification of BFD session state. - assert wait_until( - 300, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 300, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - # Savings the configs - src_dut.shell("sudo config save -y") - - # Perform a cold reboot on source dut - reboot(src_dut, localhost) - - # Waiting for all processes on Source dut - wait_critical_processes(src_dut) - - assert wait_until( - 300, - 10, - 0, - lambda: check_bgp_status(request), - ) - - # Verification of BFD session state. - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - logger.info("BFD deletion on source & destination dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Savings the configs - src_dut.shell("sudo config save -y") - - # Config reload of Source dut - reboot(src_dut, localhost) - - # Waiting for all processes on Source dut - wait_critical_processes(src_dut) - - assert wait_until( - 300, - 10, - 0, - lambda: check_bgp_status(request), - ) - - # Verification of BFD session state. - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" - ), - ) - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" - ), - ) - - def test_bfd_with_lc_reboot_ipv6( + COMPLETENESS_TO_ITERATIONS = { + 'debug': 1, + 'basic': 10, + 'confident': 50, + 'thorough': 100, + 'diagnose': 200, + } + + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_with_lc_reboot( self, localhost, duthost, request, - duthosts, tbinfo, get_src_dst_asic_and_duts, - bfd_base_instance, bfd_cleanup_db, + version, ): """ Author: Harsha Golla Email : harsgoll@cisco.com """ - - version = "ipv6" - # Selecting source, destination dut & prefix & BFD status verification for all nexthops logger.info( "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" @@ -243,22 +57,22 @@ def test_bfd_with_lc_reboot_ipv6( src_prefix, dst_prefix, ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version + request, get_src_dst_asic_and_duts, version ) # Creation of BFD logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) + add_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Verification of BFD session state. assert wait_until( 300, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -266,7 +80,7 @@ def test_bfd_with_lc_reboot_ipv6( 300, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) @@ -280,19 +94,14 @@ def test_bfd_with_lc_reboot_ipv6( # Waiting for all processes on Source dut wait_critical_processes(src_dut) - assert wait_until( - 300, - 10, - 0, - lambda: check_bgp_status(request), - ) + check_bgp_status(request) # Verification of BFD session state. assert wait_until( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -300,16 +109,16 @@ def test_bfd_with_lc_reboot_ipv6( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) logger.info("BFD deletion on source & destination dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + delete_bfd(src_asic.asic_index, src_prefix, src_dut) + delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - # Savings the configs + # Save the configs src_dut.shell("sudo config save -y") # Config reload of Source dut @@ -318,19 +127,14 @@ def test_bfd_with_lc_reboot_ipv6( # Waiting for all processes on Source dut wait_critical_processes(src_dut) - assert wait_until( - 300, - 10, - 0, - lambda: check_bgp_status(request), - ) + check_bgp_status(request) # Verification of BFD session state. assert wait_until( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" ), ) @@ -338,20 +142,20 @@ def test_bfd_with_lc_reboot_ipv6( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" ), ) - def test_bfd_static_route_deletion_ipv4( + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_static_route_deletion( self, duthost, request, - duthosts, tbinfo, get_src_dst_asic_and_duts, - bfd_base_instance, bfd_cleanup_db, + version, ): """ Author: Harsha Golla @@ -365,8 +169,6 @@ def test_bfd_static_route_deletion_ipv4( 4. Delete BFD on Destination dut. 5. Verify that on Destination dut BFD gets cleaned up and static route will be added back. """ - version = "ipv4" - logger.info( "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" ) @@ -380,21 +182,21 @@ def test_bfd_static_route_deletion_ipv4( src_prefix, dst_prefix, ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version + request, get_src_dst_asic_and_duts, version ) # Creation of BFD logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) + add_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Verification of BFD session state. assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -402,20 +204,20 @@ def test_bfd_static_route_deletion_ipv4( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) logger.info("BFD deletion on source dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) + delete_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD & Static route verifications") assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" ), ) @@ -423,7 +225,7 @@ def test_bfd_static_route_deletion_ipv4( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" ), ) @@ -433,7 +235,6 @@ def test_bfd_static_route_deletion_ipv4( dst_prefix, dst_dut, "Route Removal", - bfd_base_instance, version, ) verify_static_route( @@ -442,19 +243,18 @@ def test_bfd_static_route_deletion_ipv4( src_prefix, src_dut, "Route Addition", - bfd_base_instance, version, ) logger.info("BFD deletion on destination dut") - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) logger.info("BFD & Static route verifications") assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" ), ) @@ -462,7 +262,7 @@ def test_bfd_static_route_deletion_ipv4( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" ), ) @@ -472,7 +272,6 @@ def test_bfd_static_route_deletion_ipv4( dst_prefix, dst_dut, "Route Addition", - bfd_base_instance, version, ) verify_static_route( @@ -481,29 +280,37 @@ def test_bfd_static_route_deletion_ipv4( src_prefix, src_dut, "Route Addition", - bfd_base_instance, version, ) logger.info("BFD deletion did not influence static routes and test completed successfully") - def test_bfd_static_route_deletion_ipv6( + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_flap( self, duthost, request, duthosts, tbinfo, get_src_dst_asic_and_duts, - bfd_base_instance, bfd_cleanup_db, + get_function_completeness_level, + version, ): """ Author: Harsha Golla Email : harsgoll@cisco.com - """ - - version = "ipv6" + To flap the BFD session ( Up <--> Down <---> Up) between linecards for 100 times. + Test Steps: + 1. Delete BFD on Source dut + 2. Verify that on Source dut BFD gets cleaned up and static route exists. + 3. Verify that on Destination dut BFD goes down and static route will be removed. + 4. Add BFD on Source dut. + 5. Verify that on Source dut BFD is up + 6. Verify that on destination dut BFD is up and static route is added back. + 7. Repeat above steps 100 times. + """ logger.info( "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" ) @@ -517,21 +324,21 @@ def test_bfd_static_route_deletion_ipv6( src_prefix, dst_prefix, ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version + request, get_src_dst_asic_and_duts, version ) # Creation of BFD logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) + add_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Verification of BFD session state. assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -539,384 +346,72 @@ def test_bfd_static_route_deletion_ipv6( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) - logger.info("BFD deletion on source dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) + completeness_level = get_function_completeness_level + if completeness_level is None: + completeness_level = "debug" - logger.info("BFD & Static route verifications") - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" - ), - ) - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Removal", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Addition", - bfd_base_instance, - version, - ) + total_iterations = self.COMPLETENESS_TO_ITERATIONS[completeness_level] + successful_iterations = 0 # Counter for successful iterations + for i in range(total_iterations): + logger.info("Iteration {}".format(i)) - logger.info("BFD deletion on destination dut") - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + logger.info("BFD deletion on source dut") + delete_bfd(src_asic.asic_index, src_prefix, src_dut) - logger.info("BFD & Static route verifications") - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" - ), - ) - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Addition", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Addition", - bfd_base_instance, - version, - ) + logger.info("Waiting for 5s post BFD shutdown") + time.sleep(5) - logger.info("BFD deletion did not influence static routes and test completed successfully") - - def test_bfd_flap_ipv4( - self, - duthost, - request, - duthosts, - tbinfo, - get_src_dst_asic_and_duts, - bfd_base_instance, - bfd_cleanup_db, - ): - """ - Author: Harsha Golla - Email : harsgoll@cisco.com - - To flap the BFD session ( Up <--> Down <---> Up) between linecards for 100 times. - Test Steps: - 1. Delete BFD on Source dut - 2. Verify that on Source dut BFD gets cleaned up and static route exists. - 3. Verify that on Destination dut BFD goes down and static route will be removed. - 4. Add BFD on Source dut. - 5. Verify that on Source dut BFD is up - 6. Verify that on destination dut BFD is up and static route is added back. - 7. Repeat above steps 100 times. - """ - version = "ipv4" - - logger.info( - "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" - ) - ( - src_asic, - dst_asic, - src_dut, - dst_dut, - src_dut_nexthops, - dst_dut_nexthops, - src_prefix, - dst_prefix, - ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version - ) - - # Creation of BFD - logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - successful_iterations = 0 # Counter for successful iterations - for i in range(self.TOTAL_ITERATIONS): - logger.info("Iteration {}".format(i)) - - logger.info("BFD deletion on source dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - - logger.info("Waiting for 5s post BFD shutdown") - time.sleep(5) - - logger.info("BFD & Static route verifications") - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, - src_dut_nexthops.values(), - src_asic, - "No BFD sessions found", - ), - ) - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Removal", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Addition", - bfd_base_instance, - version, - ) - - logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - - logger.info("BFD & Static route verifications") - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Addition", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Addition", - bfd_base_instance, - version, - ) - - # Check if both iterations were successful and increment the counter - successful_iterations += 1 - - # Determine the success rate - logger.info("successful_iterations: %d", successful_iterations) - success_rate = (successful_iterations / self.TOTAL_ITERATIONS) * 100 - - logger.info("Current success rate: %.2f%%", success_rate) - # Check if the success rate is above the threshold (e.g., 98%) - assert ( - success_rate >= 98 - ), "BFD flap verification success rate is below 98% ({}%)".format(success_rate) - - logger.info("test_bfd_flap completed") - - def test_bfd_flap_ipv6( - self, - duthost, - request, - duthosts, - tbinfo, - get_src_dst_asic_and_duts, - bfd_base_instance, - bfd_cleanup_db, - ): - """ - Author: Harsha Golla - Email : harsgoll@cisco.com - - To flap the BFD session ( Up <--> Down <---> Up) between linecards for 100 times. - Test Steps: - 1. Delete BFD on Source dut - 2. Verify that on Source dut BFD gets cleaned up and static route exists. - 3. Verify that on Destination dut BFD goes down and static route will be removed. - 4. Add BFD on Source dut. - 5. Verify that on Source dut BFD is up - 6. Verify that on destination dut BFD is up and static route is added back. - 7. Repeat above steps 100 times. - """ - version = "ipv6" - - logger.info( - "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" - ) - ( - src_asic, - dst_asic, - src_dut, - dst_dut, - src_dut_nexthops, - dst_dut_nexthops, - src_prefix, - dst_prefix, - ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version - ) - - # Creation of BFD - logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - successful_iterations = 0 # Counter for successful iterations - for i in range(self.TOTAL_ITERATIONS): - logger.info("Iteration {}".format(i)) - - logger.info("BFD deletion on source dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - - logger.info("Waiting for 5s post BFD shutdown") - time.sleep(5) - - logger.info("BFD & Static route verifications") - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, - src_dut_nexthops.values(), - src_asic, - "No BFD sessions found", - ), - ) - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Removal", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Addition", - bfd_base_instance, - version, - ) + logger.info("BFD & Static route verifications") + assert wait_until( + 180, + 10, + 0, + lambda: verify_bfd_state( + dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" + ), + ) + assert wait_until( + 180, + 10, + 0, + lambda: verify_bfd_state( + src_dut, + src_dut_nexthops.values(), + src_asic, + "No BFD sessions found", + ), + ) + verify_static_route( + request, + dst_asic, + dst_prefix, + dst_dut, + "Route Removal", + version, + ) + verify_static_route( + request, + src_asic, + src_prefix, + src_dut, + "Route Addition", + version, + ) logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) + add_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD & Static route verifications") assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -924,7 +419,7 @@ def test_bfd_flap_ipv6( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) @@ -934,7 +429,6 @@ def test_bfd_flap_ipv6( dst_prefix, dst_dut, "Route Addition", - bfd_base_instance, version, ) verify_static_route( @@ -943,7 +437,6 @@ def test_bfd_flap_ipv6( src_prefix, src_dut, "Route Addition", - bfd_base_instance, version, ) @@ -952,7 +445,7 @@ def test_bfd_flap_ipv6( # Determine the success rate logger.info("successful_iterations: %d", successful_iterations) - success_rate = (successful_iterations / self.TOTAL_ITERATIONS) * 100 + success_rate = (successful_iterations / total_iterations) * 100 logger.info("Current success rate: %.2f%%", success_rate) # Check if the success rate is above the threshold (e.g., 98%) @@ -962,7 +455,8 @@ def test_bfd_flap_ipv6( logger.info("test_bfd_flap completed") - def test_bfd_with_rp_reboot_ipv4( + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_with_rp_reboot( self, localhost, duthost, @@ -970,17 +464,14 @@ def test_bfd_with_rp_reboot_ipv4( duthosts, tbinfo, get_src_dst_asic_and_duts, - bfd_base_instance, enum_supervisor_dut_hostname, bfd_cleanup_db, + version, ): """ Author: Harsha Golla Email : harsgoll@cisco.com """ - - version = "ipv4" - rp = duthosts[enum_supervisor_dut_hostname] # Selecting source, destination dut & prefix & BFD status verification for all nexthops @@ -997,22 +488,22 @@ def test_bfd_with_rp_reboot_ipv4( src_prefix, dst_prefix, ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version + request, get_src_dst_asic_and_duts, version ) # Creation of BFD logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) + add_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Verification of BFD session state. assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -1020,7 +511,7 @@ def test_bfd_with_rp_reboot_ipv4( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) @@ -1036,989 +527,75 @@ def test_bfd_with_rp_reboot_ipv4( wait_critical_processes(src_dut) wait_critical_processes(dst_dut) - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) - - # Verification of BFD session state. - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - logger.info("BFD deletion on source & destination dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Savings the configs - src_dut.shell("sudo config save -y") - dst_dut.shell("sudo config save -y") - - # Config reload of Source dut - reboot(rp, localhost) - - # Waiting for all processes on Source & destination dut - wait_critical_processes(src_dut) - wait_critical_processes(dst_dut) - - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) - - # Verification of BFD session state. - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" - ), - ) - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" - ), - ) - - def test_bfd_with_rp_reboot_ipv6( - self, - localhost, - duthost, - request, - duthosts, - tbinfo, - get_src_dst_asic_and_duts, - bfd_base_instance, - enum_supervisor_dut_hostname, - bfd_cleanup_db, - ): - """ - Author: Harsha Golla - Email : harsgoll@cisco.com - """ - - version = "ipv6" - - rp = duthosts[enum_supervisor_dut_hostname] - - # Selecting source, destination dut & prefix & BFD status verification for all nexthops - logger.info( - "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" - ) - ( - src_asic, - dst_asic, - src_dut, - dst_dut, - src_dut_nexthops, - dst_dut_nexthops, - src_prefix, - dst_prefix, - ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version - ) - - # Creation of BFD - logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - - logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - # Savings the configs - src_dut.shell("sudo config save -y") - dst_dut.shell("sudo config save -y") - - # Perform a cold reboot on source dut - reboot(rp, localhost) - - # Waiting for all processes on Source & destination dut - wait_critical_processes(src_dut) - wait_critical_processes(dst_dut) - - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) - - # Verification of BFD session state. - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - logger.info("BFD deletion on source & destination dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Savings the configs - src_dut.shell("sudo config save -y") - dst_dut.shell("sudo config save -y") - - # Config reload of Source dut - reboot(rp, localhost) - - # Waiting for all processes on Source & destination dut - wait_critical_processes(src_dut) - wait_critical_processes(dst_dut) - - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) - - # Verification of BFD session state. - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" - ), - ) - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" - ), - ) - - def test_bfd_remote_link_flap_ipv4( - self, - duthost, - request, - duthosts, - tbinfo, - get_src_dst_asic_and_duts, - bfd_base_instance, - bfd_cleanup_db, - ): - """ - Author: Harsha Golla - Email : harsgoll@cisco.com - """ - - version = "ipv4" - - request.config.interface_shutdown = True - - # Selecting source, destination dut & prefix & BFD status verification for all nexthops - logger.info( - "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" - ) - ( - src_asic, - dst_asic, - src_dut, - dst_dut, - src_dut_nexthops, - dst_dut_nexthops, - src_prefix, - dst_prefix, - ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version - ) - - # Creation of BFD - logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - # Extract portchannel interfaces on dst - list_of_portchannels_on_dst = src_dut_nexthops.keys() - request.config.portchannels_on_dut = "dst" - request.config.selected_portchannels = list_of_portchannels_on_dst - - # Shutdown PortChannels on destination dut - for interface in list_of_portchannels_on_dst: - action = "shutdown" - control_interface_state( - dst_dut, dst_asic, interface, action - ) - - # Verification of BFD session state on src dut - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Down" - ), - ) - - # Verify that corresponding static route has been removed on both duts - logger.info("BFD & Static route verifications") - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Removal", - bfd_base_instance, - version, - ) - - for interface in list_of_portchannels_on_dst: - action = "startup" - control_interface_state( - dst_dut, dst_asic, interface, action - ) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - # Verify that corresponding static route has been added on both duts - logger.info("BFD & Static route verifications") - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Addition", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Addition", - bfd_base_instance, - version, - ) - - def test_bfd_remote_link_flap_ipv6( - self, - duthost, - request, - duthosts, - tbinfo, - get_src_dst_asic_and_duts, - bfd_base_instance, - bfd_cleanup_db, - ): - """ - Author: Harsha Golla - Email : harsgoll@cisco.com - """ - - version = "ipv6" - - request.config.interface_shutdown = True - - # Selecting source, destination dut & prefix & BFD status verification for all nexthops - logger.info( - "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" - ) - ( - src_asic, - dst_asic, - src_dut, - dst_dut, - src_dut_nexthops, - dst_dut_nexthops, - src_prefix, - dst_prefix, - ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version - ) - - # Creation of BFD - logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - # Extract portchannel interfaces on dst - list_of_portchannels_on_dst = src_dut_nexthops.keys() - request.config.portchannels_on_dut = "dst" - request.config.selected_portchannels = list_of_portchannels_on_dst - - # Shutdown PortChannels on destination dut - for interface in list_of_portchannels_on_dst: - action = "shutdown" - control_interface_state( - dst_dut, dst_asic, interface, action - ) - - # Verification of BFD session state on src dut - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Down" - ), - ) - - # Verify that corresponding static route has been removed on both duts - logger.info("BFD & Static route verifications") - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Removal", - bfd_base_instance, - version, - ) - - for interface in list_of_portchannels_on_dst: - action = "startup" - control_interface_state( - dst_dut, dst_asic, interface, action - ) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - # Verify that corresponding static route has been added on both duts - logger.info("BFD & Static route verifications") - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Addition", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Addition", - bfd_base_instance, - version, - ) - - def test_bfd_lc_asic_shutdown_ipv4( - self, - duthost, - request, - duthosts, - tbinfo, - get_src_dst_asic_and_duts, - bfd_base_instance, - bfd_cleanup_db, - ): - """ - Author: Harsha Golla - Email : harsgoll@cisco.com - """ - - version = "ipv4" - - request.config.interface_shutdown = True - - # Selecting source, destination dut & prefix & BFD status verification for all nexthops - logger.info( - "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" - ) - ( - src_asic, - dst_asic, - src_dut, - dst_dut, - src_dut_nexthops, - dst_dut_nexthops, - src_prefix, - dst_prefix, - ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version - ) - - # Creation of BFD - logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - # Extract portchannel interfaces on src - list_of_portchannels_on_src = dst_dut_nexthops.keys() - request.config.portchannels_on_dut = "src" - request.config.selected_portchannels = list_of_portchannels_on_src - - # Shutdown PortChannels - for interface in list_of_portchannels_on_src: - action = "shutdown" - control_interface_state( - src_dut, src_asic, interface, action - ) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Down" - ), - ) - - # Verify that corresponding static route has been removed on both duts - logger.info("BFD & Static route verifications") - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Removal", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Removal", - bfd_base_instance, - version, - ) - - for interface in list_of_portchannels_on_src: - action = "startup" - control_interface_state( - src_dut, src_asic, interface, action - ) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - # Verify that corresponding static route has been added on both duts - logger.info("BFD & Static route verifications") - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Addition", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Addition", - bfd_base_instance, - version, - ) - - def test_bfd_lc_asic_shutdown_ipv6( - self, - duthost, - request, - duthosts, - tbinfo, - get_src_dst_asic_and_duts, - bfd_base_instance, - bfd_cleanup_db, - ): - """ - Author: Harsha Golla - Email : harsgoll@cisco.com - """ - - version = "ipv6" - - request.config.interface_shutdown = True - - # Selecting source, destination dut & prefix & BFD status verification for all nexthops - logger.info( - "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" - ) - ( - src_asic, - dst_asic, - src_dut, - dst_dut, - src_dut_nexthops, - dst_dut_nexthops, - src_prefix, - dst_prefix, - ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version - ) - - # Creation of BFD - logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - # Extract portchannel interfaces on src - list_of_portchannels_on_src = dst_dut_nexthops.keys() - request.config.portchannels_on_dut = "src" - request.config.selected_portchannels = list_of_portchannels_on_src - - # Shutdown PortChannels - for interface in list_of_portchannels_on_src: - action = "shutdown" - control_interface_state( - src_dut, src_asic, interface, action - ) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Down" - ), - ) - - # Verify that corresponding static route has been removed on both duts - logger.info("BFD & Static route verifications") - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Removal", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Removal", - bfd_base_instance, - version, - ) - - for interface in list_of_portchannels_on_src: - action = "startup" - control_interface_state( - src_dut, src_asic, interface, action - ) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - # Verify that corresponding static route has been added on both duts - logger.info("BFD & Static route verifications") - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Addition", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Addition", - bfd_base_instance, - version, - ) - - def test_bfd_portchannel_member_flap_ipv4( - self, - duthost, - request, - duthosts, - tbinfo, - get_src_dst_asic_and_duts, - bfd_base_instance, - bfd_cleanup_db, - ): - """ - Author: Harsha Golla - Email : harsgoll@cisco.com - """ - - version = "ipv4" - - request.config.interface_shutdown = True - - # Selecting source, destination dut & prefix & BFD status verification for all nexthops - logger.info( - "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" - ) - ( - src_asic, - dst_asic, - src_dut, - dst_dut, - src_dut_nexthops, - dst_dut_nexthops, - src_prefix, - dst_prefix, - ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version - ) - - # Creation of BFD - logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - # Extract portchannel interfaces on src - list_of_portchannels_on_src = dst_dut_nexthops.keys() - request.config.portchannels_on_dut = "src" - request.config.selected_portchannels = list_of_portchannels_on_src - - # Shutdown PortChannel members - for portchannel_interface in list_of_portchannels_on_src: - action = "shutdown" - list_of_portchannel_members_on_src = ( - bfd_base_instance.extract_backend_portchannels(src_dut)[ - portchannel_interface - ]["members"] - ) - request.config.selected_portchannel_members = ( - list_of_portchannel_members_on_src - ) - for each_member in list_of_portchannel_members_on_src: - control_interface_state( - src_dut, src_asic, each_member, action - ) - - # Verification of BFD session state. - assert wait_until( - 300, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" - ), - ) - assert wait_until( - 300, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Down" - ), - ) - - # Verify that corresponding static route has been removed on both duts - logger.info("BFD & Static route verifications") - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Removal", - bfd_base_instance, - version, - ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Removal", - bfd_base_instance, - version, - ) - - # Bring up of PortChannel members - for portchannel_interface in list_of_portchannels_on_src: - action = "startup" - list_of_portchannel_members_on_src = ( - bfd_base_instance.extract_backend_portchannels(src_dut)[ - portchannel_interface - ]["members"] - ) - for each_member in list_of_portchannel_members_on_src: - control_interface_state( - src_dut, src_asic, each_member, action - ) + check_bgp_status(request) # Verification of BFD session state. assert wait_until( 300, - 10, + 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) assert wait_until( 300, - 10, + 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) - # Verify that corresponding static route has been added on both duts - logger.info("Static route verifications") - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Addition", - bfd_base_instance, - version, + logger.info("BFD deletion on source & destination dut") + delete_bfd(src_asic.asic_index, src_prefix, src_dut) + delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + + # Savings the configs + src_dut.shell("sudo config save -y") + dst_dut.shell("sudo config save -y") + + # Config reload of Source dut + reboot(rp, localhost) + + # Waiting for all processes on Source & destination dut + wait_critical_processes(src_dut) + wait_critical_processes(dst_dut) + + check_bgp_status(request) + + # Verification of BFD session state. + assert wait_until( + 300, + 20, + 0, + lambda: verify_bfd_state( + dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" + ), ) - verify_static_route( - request, - src_asic, - src_prefix, - src_dut, - "Route Addition", - bfd_base_instance, - version, + assert wait_until( + 300, + 20, + 0, + lambda: verify_bfd_state( + src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" + ), ) - def test_bfd_portchannel_member_flap_ipv6( + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_remote_link_flap( self, duthost, request, - duthosts, tbinfo, get_src_dst_asic_and_duts, - bfd_base_instance, bfd_cleanup_db, + version, ): """ Author: Harsha Golla Email : harsgoll@cisco.com """ - - version = "ipv6" - request.config.interface_shutdown = True # Selecting source, destination dut & prefix & BFD status verification for all nexthops @@ -2035,21 +612,21 @@ def test_bfd_portchannel_member_flap_ipv6( src_prefix, dst_prefix, ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version + request, get_src_dst_asic_and_duts, version ) # Creation of BFD logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) + add_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Verification of BFD session state. assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -2057,111 +634,76 @@ def test_bfd_portchannel_member_flap_ipv6( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) - # Extract portchannel interfaces on src - list_of_portchannels_on_src = dst_dut_nexthops.keys() - request.config.portchannels_on_dut = "src" - request.config.selected_portchannels = list_of_portchannels_on_src + # Extract portchannel interfaces on dst + list_of_portchannels_on_dst = src_dut_nexthops.keys() + request.config.portchannels_on_dut = "dst" + request.config.selected_portchannels = list_of_portchannels_on_dst - # Shutdown PortChannel members - for portchannel_interface in list_of_portchannels_on_src: + # Shutdown PortChannels on destination dut + for interface in list_of_portchannels_on_dst: action = "shutdown" - list_of_portchannel_members_on_src = ( - bfd_base_instance.extract_backend_portchannels(src_dut)[ - portchannel_interface - ]["members"] - ) - request.config.selected_portchannel_members = ( - list_of_portchannel_members_on_src + control_interface_state( + dst_dut, dst_asic, interface, action ) - for each_member in list_of_portchannel_members_on_src: - control_interface_state( - src_dut, src_asic, each_member, action - ) - # Verification of BFD session state. - assert wait_until( - 300, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" - ), - ) + # Verification of BFD session state on src dut assert wait_until( - 300, + 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Down" ), ) # Verify that corresponding static route has been removed on both duts logger.info("BFD & Static route verifications") - verify_static_route( - request, - dst_asic, - dst_prefix, - dst_dut, - "Route Removal", - bfd_base_instance, - version, - ) verify_static_route( request, src_asic, src_prefix, src_dut, "Route Removal", - bfd_base_instance, version, ) - # Bring up of PortChannel members - for portchannel_interface in list_of_portchannels_on_src: + for interface in list_of_portchannels_on_dst: action = "startup" - list_of_portchannel_members_on_src = ( - bfd_base_instance.extract_backend_portchannels(src_dut)[ - portchannel_interface - ]["members"] + control_interface_state( + dst_dut, dst_asic, interface, action ) - for each_member in list_of_portchannel_members_on_src: - control_interface_state( - src_dut, src_asic, each_member, action - ) # Verification of BFD session state. assert wait_until( - 300, + 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) assert wait_until( - 300, + 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) # Verify that corresponding static route has been added on both duts - logger.info("Static route verifications") + logger.info("BFD & Static route verifications") verify_static_route( request, dst_asic, dst_prefix, dst_dut, "Route Addition", - bfd_base_instance, version, ) verify_static_route( @@ -2170,26 +712,24 @@ def test_bfd_portchannel_member_flap_ipv6( src_prefix, src_dut, "Route Addition", - bfd_base_instance, version, ) - def test_bfd_config_reload_ipv4( + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_lc_asic_shutdown( self, duthost, request, - duthosts, tbinfo, get_src_dst_asic_and_duts, - bfd_base_instance, bfd_cleanup_db, + version, ): """ Author: Harsha Golla Email : harsgoll@cisco.com """ - - version = "ipv4" + request.config.interface_shutdown = True # Selecting source, destination dut & prefix & BFD status verification for all nexthops logger.info( @@ -2205,22 +745,21 @@ def test_bfd_config_reload_ipv4( src_prefix, dst_prefix, ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version + request, get_src_dst_asic_and_duts, version ) # Creation of BFD logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - + add_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Verification of BFD session state. assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -2228,234 +767,118 @@ def test_bfd_config_reload_ipv4( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) - # Savings the configs - src_dut.shell("sudo config save -y") - - # Config reload of Source dut - config_reload(src_dut) - - # Waiting for all processes on Source dut - wait_critical_processes(src_dut) + # Extract portchannel interfaces on src + list_of_portchannels_on_src = dst_dut_nexthops.keys() + request.config.portchannels_on_dut = "src" + request.config.selected_portchannels = list_of_portchannels_on_src - assert wait_until( - 300, - 10, - 0, - lambda: check_bgp_status(request), - ) + # Shutdown PortChannels + for interface in list_of_portchannels_on_src: + action = "shutdown" + control_interface_state( + src_dut, src_asic, interface, action + ) # Verification of BFD session state. assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), - ) - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), - ) - - logger.info("BFD deletion on source & destination dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Savings the configs - src_dut.shell("sudo config save -y") - - # Config reload of Source dut - config_reload(src_dut) - - # Waiting for all processes on Source dut - wait_critical_processes(src_dut) - - assert wait_until( - 300, + 180, 10, 0, - lambda: check_bgp_status(request), - ) - - # Verification of BFD session state. - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" + lambda: verify_bfd_state( + dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" ), ) assert wait_until( - 300, - 20, + 180, + 10, 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" + lambda: verify_bfd_state( + src_dut, src_dut_nexthops.values(), src_asic, "Down" ), ) - def test_bfd_config_reload_ipv6( - self, - duthost, - request, - duthosts, - tbinfo, - get_src_dst_asic_and_duts, - bfd_base_instance, - bfd_cleanup_db, - ): - """ - Author: Harsha Golla - Email : harsgoll@cisco.com - """ - - version = "ipv6" - - # Selecting source, destination dut & prefix & BFD status verification for all nexthops - logger.info( - "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" - ) - ( - src_asic, + # Verify that corresponding static route has been removed on both duts + logger.info("BFD & Static route verifications") + verify_static_route( + request, dst_asic, - src_dut, - dst_dut, - src_dut_nexthops, - dst_dut_nexthops, - src_prefix, dst_prefix, - ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version - ) - - # Creation of BFD - logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - - logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Verification of BFD session state. - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" - ), + dst_dut, + "Route Removal", + version, ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" - ), + verify_static_route( + request, + src_asic, + src_prefix, + src_dut, + "Route Removal", + version, ) - # Savings the configs - src_dut.shell("sudo config save -y") - - # Config reload of Source dut - config_reload(src_dut) - - # Waiting for all processes on Source dut - wait_critical_processes(src_dut) - - assert wait_until( - 300, - 10, - 0, - lambda: check_bgp_status(request), - ) + for interface in list_of_portchannels_on_src: + action = "startup" + control_interface_state( + src_dut, src_asic, interface, action + ) # Verification of BFD session state. assert wait_until( - 300, - 20, + 180, + 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) assert wait_until( - 300, - 20, + 180, + 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) - logger.info("BFD deletion on source & destination dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Savings the configs - src_dut.shell("sudo config save -y") - - # Config reload of Source dut - config_reload(src_dut) - - # Waiting for all processes on Source dut - wait_critical_processes(src_dut) - - assert wait_until( - 300, - 10, - 0, - lambda: check_bgp_status(request), - ) - - # Verification of BFD session state. - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" - ), + # Verify that corresponding static route has been added on both duts + logger.info("BFD & Static route verifications") + verify_static_route( + request, + dst_asic, + dst_prefix, + dst_dut, + "Route Addition", + version, ) - assert wait_until( - 300, - 20, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" - ), + verify_static_route( + request, + src_asic, + src_prefix, + src_dut, + "Route Addition", + version, ) - def test_bfd_with_rp_config_reload_ipv4( + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_portchannel_member_flap( self, - localhost, duthost, request, - duthosts, tbinfo, get_src_dst_asic_and_duts, - bfd_base_instance, - enum_supervisor_dut_hostname, bfd_cleanup_db, + version, ): """ Author: Harsha Golla Email : harsgoll@cisco.com """ - - version = "ipv4" - - rp = duthosts[enum_supervisor_dut_hostname] + request.config.interface_shutdown = True # Selecting source, destination dut & prefix & BFD status verification for all nexthops logger.info( @@ -2471,22 +894,21 @@ def test_bfd_with_rp_config_reload_ipv4( src_prefix, dst_prefix, ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version + request, get_src_dst_asic_and_duts, version ) # Creation of BFD logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) - + add_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Verification of BFD session state. assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -2494,107 +916,133 @@ def test_bfd_with_rp_config_reload_ipv4( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) - # Savings the configs - src_dut.shell("sudo config save -y") - dst_dut.shell("sudo config save -y") - - # Perform a cold reboot on source dut - config_reload(rp) - - # Waiting for all processes on Source & destination dut - wait_critical_processes(src_dut) - wait_critical_processes(dst_dut) + # Extract portchannel interfaces on src + list_of_portchannels_on_src = dst_dut_nexthops.keys() + request.config.portchannels_on_dut = "src" + request.config.selected_portchannels = list_of_portchannels_on_src - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) + # Shutdown PortChannel members + for portchannel_interface in list_of_portchannels_on_src: + action = "shutdown" + list_of_portchannel_members_on_src = ( + extract_backend_portchannels(src_dut)[ + portchannel_interface + ]["members"] + ) + request.config.selected_portchannel_members = ( + list_of_portchannel_members_on_src + ) + for each_member in list_of_portchannel_members_on_src: + control_interface_state( + src_dut, src_asic, each_member, action + ) # Verification of BFD session state. assert wait_until( 300, - 20, + 10, 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" + lambda: verify_bfd_state( + dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" ), ) assert wait_until( 300, - 20, + 10, 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Up" + lambda: verify_bfd_state( + src_dut, src_dut_nexthops.values(), src_asic, "Down" ), ) - logger.info("BFD deletion on source & destination dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) - - # Savings the configs - src_dut.shell("sudo config save -y") - dst_dut.shell("sudo config save -y") - - # Config reload of Source dut - config_reload(rp) + # Verify that corresponding static route has been removed on both duts + logger.info("BFD & Static route verifications") + verify_static_route( + request, + dst_asic, + dst_prefix, + dst_dut, + "Route Removal", + version, + ) + verify_static_route( + request, + src_asic, + src_prefix, + src_dut, + "Route Removal", + version, + ) - # Waiting for all processes on Source & destination dut - wait_critical_processes(src_dut) - wait_critical_processes(dst_dut) + # Bring up of PortChannel members + for portchannel_interface in list_of_portchannels_on_src: + action = "startup" + list_of_portchannel_members_on_src = ( + extract_backend_portchannels(src_dut)[ + portchannel_interface + ]["members"] + ) + for each_member in list_of_portchannel_members_on_src: + control_interface_state( + src_dut, src_asic, each_member, action + ) - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) # Verification of BFD session state. assert wait_until( 300, - 20, + 10, 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" + lambda: verify_bfd_state( + dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) assert wait_until( 300, - 20, + 10, 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" + lambda: verify_bfd_state( + src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) - def test_bfd_with_rp_config_reload_ipv6( + # Verify that corresponding static route has been added on both duts + logger.info("Static route verifications") + verify_static_route( + request, + dst_asic, + dst_prefix, + dst_dut, + "Route Addition", + version, + ) + verify_static_route( + request, + src_asic, + src_prefix, + src_dut, + "Route Addition", + version, + ) + + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_config_reload( self, - localhost, duthost, request, - duthosts, tbinfo, get_src_dst_asic_and_duts, - bfd_base_instance, - enum_supervisor_dut_hostname, bfd_cleanup_db, + version, ): """ Author: Harsha Golla Email : harsgoll@cisco.com """ - - version = "ipv6" - - rp = duthosts[enum_supervisor_dut_hostname] - # Selecting source, destination dut & prefix & BFD status verification for all nexthops logger.info( "Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix" @@ -2609,22 +1057,22 @@ def test_bfd_with_rp_config_reload_ipv6( src_prefix, dst_prefix, ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version + request, get_src_dst_asic_and_duts, version ) # Creation of BFD logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) + add_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Verification of BFD session state. assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -2632,35 +1080,28 @@ def test_bfd_with_rp_config_reload_ipv6( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) # Savings the configs src_dut.shell("sudo config save -y") - dst_dut.shell("sudo config save -y") - # Perform a cold reboot on source dut - config_reload(rp) + # Config reload of Source dut + config_reload(src_dut) - # Waiting for all processes on Source & destination dut + # Waiting for all processes on Source dut wait_critical_processes(src_dut) - wait_critical_processes(dst_dut) - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) + check_bgp_status(request) # Verification of BFD session state. assert wait_until( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -2668,39 +1109,32 @@ def test_bfd_with_rp_config_reload_ipv6( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) logger.info("BFD deletion on source & destination dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + delete_bfd(src_asic.asic_index, src_prefix, src_dut) + delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Savings the configs src_dut.shell("sudo config save -y") - dst_dut.shell("sudo config save -y") # Config reload of Source dut - config_reload(rp) + config_reload(src_dut) - # Waiting for all processes on Source & destination dut + # Waiting for all processes on Source dut wait_critical_processes(src_dut) - wait_critical_processes(dst_dut) - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) + check_bgp_status(request) # Verification of BFD session state. assert wait_until( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" ), ) @@ -2708,12 +1142,13 @@ def test_bfd_with_rp_config_reload_ipv6( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" ), ) - def test_bfd_with_bad_fc_asic_ipv4( + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_with_rp_config_reload( self, localhost, duthost, @@ -2721,17 +1156,14 @@ def test_bfd_with_bad_fc_asic_ipv4( duthosts, tbinfo, get_src_dst_asic_and_duts, - bfd_base_instance, enum_supervisor_dut_hostname, bfd_cleanup_db, + version, ): """ Author: Harsha Golla Email : harsgoll@cisco.com """ - - version = "ipv4" - rp = duthosts[enum_supervisor_dut_hostname] # Selecting source, destination dut & prefix & BFD status verification for all nexthops @@ -2748,22 +1180,22 @@ def test_bfd_with_bad_fc_asic_ipv4( src_prefix, dst_prefix, ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version + request, get_src_dst_asic_and_duts, version ) # Creation of BFD logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) + add_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Verification of BFD session state. assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -2771,7 +1203,7 @@ def test_bfd_with_bad_fc_asic_ipv4( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) @@ -2780,54 +1212,21 @@ def test_bfd_with_bad_fc_asic_ipv4( src_dut.shell("sudo config save -y") dst_dut.shell("sudo config save -y") - # Extract asic ids - docker_output = rp.shell("docker ps | grep swss | awk '{print $NF}'")[ - "stdout" - ].split("\n") - asic_ids = [int(element.split("swss")[1]) for element in docker_output] - - # Shut down corresponding asic on supervisor to simulate bad asic - for id in asic_ids: - rp.shell("systemctl stop swss@{}".format(id)) - - # Verify that BFD sessions are down - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" - ), - ) - assert wait_until( - 180, - 10, - 0, - lambda: bfd_base_instance.verify_bfd_state( - src_dut, src_dut_nexthops.values(), src_asic, "Down" - ), - ) - - # Config reload RP to bring up the swss containers + # Perform a cold reboot on source dut config_reload(rp) # Waiting for all processes on Source & destination dut wait_critical_processes(src_dut) wait_critical_processes(dst_dut) - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) + check_bgp_status(request) # Verification of BFD session state. assert wait_until( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -2835,39 +1234,34 @@ def test_bfd_with_bad_fc_asic_ipv4( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) - logger.info("BFD deletion on source dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + logger.info("BFD deletion on source & destination dut") + delete_bfd(src_asic.asic_index, src_prefix, src_dut) + delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Savings the configs src_dut.shell("sudo config save -y") dst_dut.shell("sudo config save -y") - # Config reload RP + # Config reload of Source dut config_reload(rp) # Waiting for all processes on Source & destination dut wait_critical_processes(src_dut) wait_critical_processes(dst_dut) - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) + check_bgp_status(request) # Verification of BFD session state. assert wait_until( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" ), ) @@ -2875,12 +1269,13 @@ def test_bfd_with_bad_fc_asic_ipv4( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" ), ) - def test_bfd_with_bad_fc_asic_ipv6( + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_with_bad_fc_asic( self, localhost, duthost, @@ -2888,17 +1283,14 @@ def test_bfd_with_bad_fc_asic_ipv6( duthosts, tbinfo, get_src_dst_asic_and_duts, - bfd_base_instance, enum_supervisor_dut_hostname, bfd_cleanup_db, + version, ): """ Author: Harsha Golla Email : harsgoll@cisco.com """ - - version = "ipv6" - rp = duthosts[enum_supervisor_dut_hostname] # Selecting source, destination dut & prefix & BFD status verification for all nexthops @@ -2915,22 +1307,22 @@ def test_bfd_with_bad_fc_asic_ipv6( src_prefix, dst_prefix, ) = select_src_dst_dut_with_asic( - request, get_src_dst_asic_and_duts, bfd_base_instance, version + request, get_src_dst_asic_and_duts, version ) # Creation of BFD logger.info("BFD addition on source dut") - bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) + add_bfd(src_asic.asic_index, src_prefix, src_dut) logger.info("BFD addition on destination dut") - bfd_base_instance.add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + add_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Verification of BFD session state. assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -2938,7 +1330,7 @@ def test_bfd_with_bad_fc_asic_ipv6( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) @@ -2954,15 +1346,15 @@ def test_bfd_with_bad_fc_asic_ipv6( asic_ids = [int(element.split("swss")[1]) for element in docker_output] # Shut down corresponding asic on supervisor to simulate bad asic - for id in asic_ids: - rp.shell("systemctl stop swss@{}".format(id)) + for asic_id in asic_ids: + rp.shell("systemctl stop swss@{}".format(asic_id)) # Verify that BFD sessions are down assert wait_until( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Down" ), ) @@ -2970,7 +1362,7 @@ def test_bfd_with_bad_fc_asic_ipv6( 180, 10, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Down" ), ) @@ -2982,19 +1374,14 @@ def test_bfd_with_bad_fc_asic_ipv6( wait_critical_processes(src_dut) wait_critical_processes(dst_dut) - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) + check_bgp_status(request) # Verification of BFD session state. assert wait_until( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "Up" ), ) @@ -3002,14 +1389,14 @@ def test_bfd_with_bad_fc_asic_ipv6( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "Up" ), ) logger.info("BFD deletion on source dut") - bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) - bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + delete_bfd(src_asic.asic_index, src_prefix, src_dut) + delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) # Savings the configs src_dut.shell("sudo config save -y") @@ -3022,19 +1409,14 @@ def test_bfd_with_bad_fc_asic_ipv6( wait_critical_processes(src_dut) wait_critical_processes(dst_dut) - assert wait_until( - 600, - 10, - 0, - lambda: check_bgp_status(request), - ) + check_bgp_status(request) # Verification of BFD session state. assert wait_until( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( dst_dut, dst_dut_nexthops.values(), dst_asic, "No BFD sessions found" ), ) @@ -3042,7 +1424,7 @@ def test_bfd_with_bad_fc_asic_ipv6( 300, 20, 0, - lambda: bfd_base_instance.verify_bfd_state( + lambda: verify_bfd_state( src_dut, src_dut_nexthops.values(), src_asic, "No BFD sessions found" ), ) diff --git a/tests/bfd/test_bfd_traffic.py b/tests/bfd/test_bfd_traffic.py new file mode 100644 index 0000000000..a49b09193c --- /dev/null +++ b/tests/bfd/test_bfd_traffic.py @@ -0,0 +1,439 @@ +import logging + +import pytest + +from tests.bfd.bfd_base import BfdBase +from tests.bfd.bfd_helpers import get_ptf_src_port, get_backend_interface_in_use_by_counter, \ + prepare_traffic_test_variables, get_random_bgp_neighbor_ip_of_asic, toggle_port_channel_or_member, \ + get_port_channel_by_member, wait_until_bfd_up, wait_until_given_bfd_down, assert_traffic_switching + +pytestmark = [ + pytest.mark.topology("t2"), + pytest.mark.device_type('physical') +] + +logger = logging.getLogger(__name__) + + +class TestBfdTraffic(BfdBase): + PACKET_COUNT = 10000 + + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_traffic_remote_port_channel_shutdown( + self, + request, + tbinfo, + ptfadapter, + get_src_dst_asic, + bfd_cleanup_db, + version, + ): + ( + dut, + src_asic, + src_asic_index, + dst_asic, + dst_asic_index, + src_asic_next_hops, + dst_asic_next_hops, + src_asic_router_mac, + backend_port_channels, + ) = prepare_traffic_test_variables(get_src_dst_asic, request, version) + + dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dut, dst_asic_index, version) + if not dst_neighbor_ip: + pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dut.hostname)) + + ptf_src_port = get_ptf_src_port(src_asic, tbinfo) + src_bp_iface_before_shutdown, dst_bp_iface_before_shutdown = get_backend_interface_in_use_by_counter( + dut, + self.PACKET_COUNT, + version, + src_asic_router_mac, + ptfadapter, + ptf_src_port, + dst_neighbor_ip, + src_asic_index, + dst_asic_index, + ) + + dst_port_channel_before_shutdown = get_port_channel_by_member( + backend_port_channels, + dst_bp_iface_before_shutdown, + ) + + if not dst_port_channel_before_shutdown: + pytest.fail("No port channel found with interface in use") + + toggle_port_channel_or_member( + dst_port_channel_before_shutdown, + dut, + dst_asic, + request, + "shutdown", + ) + + src_port_channel_before_shutdown = get_port_channel_by_member( + backend_port_channels, + src_bp_iface_before_shutdown, + ) + + wait_until_given_bfd_down( + src_asic_next_hops, + src_port_channel_before_shutdown, + src_asic_index, + dst_asic_next_hops, + dst_port_channel_before_shutdown, + dst_asic_index, + dut, + ) + + src_bp_iface_after_shutdown, dst_bp_iface_after_shutdown = get_backend_interface_in_use_by_counter( + dut, + self.PACKET_COUNT, + version, + src_asic_router_mac, + ptfadapter, + ptf_src_port, + dst_neighbor_ip, + src_asic_index, + dst_asic_index, + ) + + assert_traffic_switching( + dut, + backend_port_channels, + src_asic_index, + src_bp_iface_before_shutdown, + src_bp_iface_after_shutdown, + src_port_channel_before_shutdown, + dst_asic_index, + dst_bp_iface_after_shutdown, + dst_bp_iface_before_shutdown, + dst_port_channel_before_shutdown, + ) + + toggle_port_channel_or_member( + dst_port_channel_before_shutdown, + dut, + dst_asic, + request, + "startup", + ) + + wait_until_bfd_up(dut, src_asic_next_hops, src_asic, dst_asic_next_hops, dst_asic) + + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_traffic_local_port_channel_shutdown( + self, + request, + tbinfo, + ptfadapter, + get_src_dst_asic, + bfd_cleanup_db, + version, + ): + ( + dut, + src_asic, + src_asic_index, + dst_asic, + dst_asic_index, + src_asic_next_hops, + dst_asic_next_hops, + src_asic_router_mac, + backend_port_channels, + ) = prepare_traffic_test_variables(get_src_dst_asic, request, version) + + dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dut, dst_asic_index, version) + if not dst_neighbor_ip: + pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dut.hostname)) + + ptf_src_port = get_ptf_src_port(src_asic, tbinfo) + src_bp_iface_before_shutdown, dst_bp_iface_before_shutdown = get_backend_interface_in_use_by_counter( + dut, + self.PACKET_COUNT, + version, + src_asic_router_mac, + ptfadapter, + ptf_src_port, + dst_neighbor_ip, + src_asic_index, + dst_asic_index, + ) + + src_port_channel_before_shutdown = get_port_channel_by_member( + backend_port_channels, + src_bp_iface_before_shutdown, + ) + + if not src_port_channel_before_shutdown: + pytest.fail("No port channel found with interface in use") + + toggle_port_channel_or_member( + src_port_channel_before_shutdown, + dut, + src_asic, + request, + "shutdown", + ) + + dst_port_channel_before_shutdown = get_port_channel_by_member( + backend_port_channels, + dst_bp_iface_before_shutdown, + ) + + wait_until_given_bfd_down( + src_asic_next_hops, + src_port_channel_before_shutdown, + src_asic_index, + dst_asic_next_hops, + dst_port_channel_before_shutdown, + dst_asic_index, + dut, + ) + + src_bp_iface_after_shutdown, dst_bp_iface_after_shutdown = get_backend_interface_in_use_by_counter( + dut, + self.PACKET_COUNT, + version, + src_asic_router_mac, + ptfadapter, + ptf_src_port, + dst_neighbor_ip, + src_asic_index, + dst_asic_index, + ) + + assert_traffic_switching( + dut, + backend_port_channels, + src_asic_index, + src_bp_iface_before_shutdown, + src_bp_iface_after_shutdown, + src_port_channel_before_shutdown, + dst_asic_index, + dst_bp_iface_after_shutdown, + dst_bp_iface_before_shutdown, + dst_port_channel_before_shutdown, + ) + + toggle_port_channel_or_member( + src_port_channel_before_shutdown, + dut, + src_asic, + request, + "startup", + ) + + wait_until_bfd_up(dut, src_asic_next_hops, src_asic, dst_asic_next_hops, dst_asic) + + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_traffic_remote_port_channel_member_shutdown( + self, + request, + tbinfo, + ptfadapter, + get_src_dst_asic, + bfd_cleanup_db, + version, + ): + ( + dut, + src_asic, + src_asic_index, + dst_asic, + dst_asic_index, + src_asic_next_hops, + dst_asic_next_hops, + src_asic_router_mac, + backend_port_channels, + ) = prepare_traffic_test_variables(get_src_dst_asic, request, version) + + dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dut, dst_asic_index, version) + if not dst_neighbor_ip: + pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dut.hostname)) + + ptf_src_port = get_ptf_src_port(src_asic, tbinfo) + src_bp_iface_before_shutdown, dst_bp_iface_before_shutdown = get_backend_interface_in_use_by_counter( + dut, + self.PACKET_COUNT, + version, + src_asic_router_mac, + ptfadapter, + ptf_src_port, + dst_neighbor_ip, + src_asic_index, + dst_asic_index, + ) + + toggle_port_channel_or_member( + dst_bp_iface_before_shutdown, + dut, + dst_asic, + request, + "shutdown", + ) + + src_port_channel_before_shutdown = get_port_channel_by_member( + backend_port_channels, + src_bp_iface_before_shutdown, + ) + + dst_port_channel_before_shutdown = get_port_channel_by_member( + backend_port_channels, + dst_bp_iface_before_shutdown, + ) + + if not src_port_channel_before_shutdown or not dst_port_channel_before_shutdown: + pytest.fail("No port channel found with interface in use") + + wait_until_given_bfd_down( + src_asic_next_hops, + src_port_channel_before_shutdown, + src_asic_index, + dst_asic_next_hops, + dst_port_channel_before_shutdown, + dst_asic_index, + dut, + ) + + src_bp_iface_after_shutdown, dst_bp_iface_after_shutdown = get_backend_interface_in_use_by_counter( + dut, + self.PACKET_COUNT, + version, + src_asic_router_mac, + ptfadapter, + ptf_src_port, + dst_neighbor_ip, + src_asic_index, + dst_asic_index, + ) + + assert_traffic_switching( + dut, + backend_port_channels, + src_asic_index, + src_bp_iface_before_shutdown, + src_bp_iface_after_shutdown, + src_port_channel_before_shutdown, + dst_asic_index, + dst_bp_iface_after_shutdown, + dst_bp_iface_before_shutdown, + dst_port_channel_before_shutdown, + ) + + toggle_port_channel_or_member( + dst_bp_iface_before_shutdown, + dut, + dst_asic, + request, + "startup", + ) + + wait_until_bfd_up(dut, src_asic_next_hops, src_asic, dst_asic_next_hops, dst_asic) + + @pytest.mark.parametrize("version", ["ipv4", "ipv6"]) + def test_bfd_traffic_local_port_channel_member_shutdown( + self, + request, + tbinfo, + ptfadapter, + get_src_dst_asic, + bfd_cleanup_db, + version, + ): + ( + dut, + src_asic, + src_asic_index, + dst_asic, + dst_asic_index, + src_asic_next_hops, + dst_asic_next_hops, + src_asic_router_mac, + backend_port_channels, + ) = prepare_traffic_test_variables(get_src_dst_asic, request, version) + + dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dut, dst_asic_index, version) + if not dst_neighbor_ip: + pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dut.hostname)) + + ptf_src_port = get_ptf_src_port(src_asic, tbinfo) + src_bp_iface_before_shutdown, dst_bp_iface_before_shutdown = get_backend_interface_in_use_by_counter( + dut, + self.PACKET_COUNT, + version, + src_asic_router_mac, + ptfadapter, + ptf_src_port, + dst_neighbor_ip, + src_asic_index, + dst_asic_index, + ) + + toggle_port_channel_or_member( + src_bp_iface_before_shutdown, + dut, + src_asic, + request, + "shutdown", + ) + + src_port_channel_before_shutdown = get_port_channel_by_member( + backend_port_channels, + src_bp_iface_before_shutdown, + ) + + dst_port_channel_before_shutdown = get_port_channel_by_member( + backend_port_channels, + dst_bp_iface_before_shutdown, + ) + + if not src_port_channel_before_shutdown or not dst_port_channel_before_shutdown: + pytest.fail("No port channel found with interface in use") + + wait_until_given_bfd_down( + src_asic_next_hops, + src_port_channel_before_shutdown, + src_asic_index, + dst_asic_next_hops, + dst_port_channel_before_shutdown, + dst_asic_index, + dut, + ) + + src_bp_iface_after_shutdown, dst_bp_iface_after_shutdown = get_backend_interface_in_use_by_counter( + dut, + self.PACKET_COUNT, + version, + src_asic_router_mac, + ptfadapter, + ptf_src_port, + dst_neighbor_ip, + src_asic_index, + dst_asic_index, + ) + + assert_traffic_switching( + dut, + backend_port_channels, + src_asic_index, + src_bp_iface_before_shutdown, + src_bp_iface_after_shutdown, + src_port_channel_before_shutdown, + dst_asic_index, + dst_bp_iface_after_shutdown, + dst_bp_iface_before_shutdown, + dst_port_channel_before_shutdown, + ) + + toggle_port_channel_or_member( + src_bp_iface_before_shutdown, + dut, + src_asic, + request, + "startup", + ) + + wait_until_bfd_up(dut, src_asic_next_hops, src_asic, dst_asic_next_hops, dst_asic) diff --git a/tests/bgp/bgp_helpers.py b/tests/bgp/bgp_helpers.py index 231f370395..72d408f2fb 100644 --- a/tests/bgp/bgp_helpers.py +++ b/tests/bgp/bgp_helpers.py @@ -17,7 +17,6 @@ from tests.common.helpers.parallel import reset_ansible_local_tmp from tests.common.helpers.parallel import parallel_run from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 BASE_DIR = os.path.dirname(os.path.realpath(__file__)) DUT_TMP_DIR = os.path.join('tmp', os.path.basename(BASE_DIR)) diff --git a/tests/bgp/conftest.py b/tests/bgp/conftest.py index 993dabb37f..31811961d4 100644 --- a/tests/bgp/conftest.py +++ b/tests/bgp/conftest.py @@ -24,7 +24,6 @@ from tests.common.helpers.constants import DEFAULT_NAMESPACE from tests.common.dualtor.dual_tor_utils import mux_cable_server_ip from tests.common import constants -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) @@ -720,3 +719,15 @@ def is_quagga(duthosts, enum_rand_one_per_hwsku_frontend_hostname): @pytest.fixture(scope="module") def is_dualtor(tbinfo): return "dualtor" in tbinfo["topo"]["name"] + + +@pytest.fixture(scope="module") +def traffic_shift_community(duthost): + community = duthost.shell('sonic-cfggen -y /etc/sonic/constants.yml -v constants.bgp.traffic_shift_community')[ + 'stdout'] + return community + + +@pytest.fixture(scope='module') +def get_function_completeness_level(pytestconfig): + return pytestconfig.getoption("--completeness_level") diff --git a/tests/bgp/constants.py b/tests/bgp/constants.py new file mode 100644 index 0000000000..6c48eedf0e --- /dev/null +++ b/tests/bgp/constants.py @@ -0,0 +1,5 @@ + +TS_NORMAL = "System Mode: Normal" +TS_MAINTENANCE = "System Mode: Maintenance" +TS_INCONSISTENT = "System Mode: Not consistent" +TS_NO_NEIGHBORS = "System Mode: No external neighbors" diff --git a/tests/bgp/route_checker.py b/tests/bgp/route_checker.py new file mode 100644 index 0000000000..322288718d --- /dev/null +++ b/tests/bgp/route_checker.py @@ -0,0 +1,269 @@ +import logging +import ipaddr as ipaddress +import re +import json +from bgp_helpers import parse_rib +from tests.common.devices.eos import EosHost +from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.parallel import parallel_run + +logger = logging.getLogger(__name__) + + +def verify_loopback_route_with_community(dut_hosts, duthost, neigh_hosts, ip_ver, community): + logger.info("Verifying only loopback routes are announced to bgp neighbors") + device_lo_addr_prefix_set = set() + device_ipv6_lo_addr_subnet_len_set = set() + device_traffic_shift_community_set = set() + device_traffic_shift_community_set.add(community) + device_ipv6_lo_addr_subnet_len_set.add('64') + for dut_host in dut_hosts: + if dut_host.is_supervisor_node(): + continue + mg_facts = dut_host.minigraph_facts(host=dut_host.hostname)['ansible_facts'] + for i in range(0, 2): + addr = mg_facts['minigraph_lo_interfaces'][i]['addr'] + if ipaddress.IPNetwork(addr).version == 4: + if 4 == ip_ver: + device_lo_addr_prefix_set.add(addr + "/32") + else: + # The IPv6 Loopback announced to neighbors is /64 + if 6 == ip_ver: + device_lo_addr_prefix_set.add(ipaddress.IPv6Address(addr).exploded[:20]) + routes_on_all_nbrs = parse_routes_on_neighbors(duthost, neigh_hosts, ip_ver) + for hostname, routes in list(routes_on_all_nbrs.items()): + logger.info("Verifying only loopback routes(ipv{}) are announced to {}".format(ip_ver, hostname)) + nbr_prefix_set = set() + nbr_prefix_community_set = set() + nbr_prefix_ipv6_subnet_len_set = set() + for prefix, received_community in list(routes.items()): + if 4 == ip_ver: + nbr_prefix_set.add(prefix) + else: + nbr_prefix_set.add(ipaddress.IPv6Address(prefix.split('/')[0]).exploded[:20]) + nbr_prefix_ipv6_subnet_len_set.add(prefix.split('/')[1]) + nbr_prefix_community_set.add(received_community) + if nbr_prefix_set != device_lo_addr_prefix_set: + logger.warn("missing loopback address or some other routes present on neighbor") + return False + if 6 == ip_ver and device_ipv6_lo_addr_subnet_len_set != nbr_prefix_ipv6_subnet_len_set: + logger.warn("ipv6 subnet is not /64 for loopback") + return False + if isinstance(list(neigh_hosts.items())[0][1]['host'], EosHost): + if nbr_prefix_community_set != device_traffic_shift_community_set: + logger.warn("traffic shift away community not present on neighbor") + return False + return True + + +def parse_routes_on_eos(dut_host, neigh_hosts, ip_ver, exp_community=[]): + """ + Parse the output of 'show ip bgp neigh received-routes' on eos, and store in a dict + """ + mg_facts = dut_host.minigraph_facts( + host=dut_host.hostname)['ansible_facts'] + asn = mg_facts['minigraph_bgp_asn'] + all_routes = {} + BGP_ENTRY_HEADING = r"BGP routing table entry for " + BGP_COMMUNITY_HEADING = r"Community: " + + # {'VM0122': 'ARISTA11T0',...} + host_name_map = {} + for hostname, neigh_host in list(neigh_hosts.items()): + host_name_map[neigh_host['host'].hostname] = hostname + + # Retrieve the routes on all VMs in parallel by using a thread poll + def parse_routes_process(node=None, results=None, my_community=exp_community): + """ + The process to parse routes on a VM. + :param neigh_host_item: tuple of hostname and host_conf dict + :return: no return value + """ + # get hostname('ARISTA11T0') by VM name('VM0122') + hostname = host_name_map[node['host'].hostname] + host = node['host'] + peer_ips = node['conf']['bgp']['peers'][asn] + for ip in peer_ips: + if ipaddress.IPNetwork(ip).version == 4: + peer_ip_v4 = ip + else: + peer_ip_v6 = ip + # The json formatter on EOS consumes too much time (over 40 seconds). + # So we have to parse the raw output instead json. + if 4 == ip_ver: + cmd = "show ip bgp neighbors {} received-routes detail | grep -E \"{}|{}\""\ + .format(peer_ip_v4, BGP_ENTRY_HEADING, BGP_COMMUNITY_HEADING) + cmd_backup = "" + else: + cmd = "show ipv6 bgp peers {} received-routes detail | grep -E \"{}|{}\""\ + .format(peer_ip_v6, BGP_ENTRY_HEADING, BGP_COMMUNITY_HEADING) + # For compatibility on EOS of old version + cmd_backup = "show ipv6 bgp neighbors {} received-routes detail | grep -E \"{}|{}\""\ + .format(peer_ip_v6, BGP_ENTRY_HEADING, BGP_COMMUNITY_HEADING) + res = host.eos_command(commands=[cmd], module_ignore_errors=True) + if res['failed'] and cmd_backup != "": + res = host.eos_command( + commands=[cmd_backup], module_ignore_errors=True) + pytest_assert( + not res['failed'], "Failed to retrieve routes from VM {}".format(hostname)) + routes = {} + routes_with_community = {} + entry = None + for line in res['stdout_lines'][0]: + addr = re.findall(BGP_ENTRY_HEADING + r"(.+)", line) + if addr: + if entry: + routes[entry] = "" + entry = None + entry = addr[0] + community = re.findall(BGP_COMMUNITY_HEADING + r"(.+)", line) + if community: + if entry: + routes[entry] = community[0] + if my_community: + for comm in my_community: + if comm in community[0]: + routes_with_community[entry] = comm + break + entry = None + community = "" + if entry: + routes[entry] = community + if my_community: + for comm in my_community: + if comm in community: + routes_with_community[entry] = comm + if my_community: + results[hostname] = routes_with_community + else: + results[hostname] = routes + try: + all_routes = parallel_run(parse_routes_process, (), {}, list( + neigh_hosts.values()), timeout=240, concurrent_tasks=8) + except BaseException as err: + logger.error( + 'Failed to get routes info from VMs. Got error: {}\n\nTrying one more time.'.format(err)) + all_routes = parallel_run(parse_routes_process, (), {}, list( + neigh_hosts.values()), timeout=240, concurrent_tasks=8) + return all_routes + + +def parse_routes_on_vsonic(dut_host, neigh_hosts, ip_ver): + mg_facts = dut_host.minigraph_facts( + host=dut_host.hostname)['ansible_facts'] + asn = mg_facts['minigraph_bgp_asn'] + all_routes = {} + + host_name_map = {} + for hostname, neigh_host in list(neigh_hosts.items()): + host_name_map[neigh_host['host'].hostname] = hostname + + def parse_routes_process_vsonic(node=None, results=None): + hostname = host_name_map[node['host'].hostname] + host = node['host'] + peer_ips = node['conf']['bgp']['peers'][asn] + + for ip in peer_ips: + if ipaddress.IPNetwork(ip).version == 4: + peer_ip_v4 = ip + else: + peer_ip_v6 = ip + + if 4 == ip_ver: + conf_cmd = "sudo vtysh -c 'configure terminal' -c 'router bgp' -c 'address-family ipv4' -c \ + 'neighbor {} soft-reconfiguration inbound' ".format(peer_ip_v4) + bgp_nbr_cmd = "sudo vtysh -c 'show ip bgp neighbors {} received-routes json'".format( + peer_ip_v4) + else: + conf_cmd = "sudo vtysh -c 'configure terminal' -c 'router bgp' -c 'address-family ipv6' -c \ + 'neighbor {} soft-reconfiguration inbound' ".format(peer_ip_v6) + bgp_nbr_cmd = "sudo vtysh -c 'show bgp ipv6 neighbors {} received-routes json'".format( + peer_ip_v6) + + host.shell(conf_cmd) + res = host.shell(bgp_nbr_cmd) + routes_json = json.loads(res['stdout'])['receivedRoutes'] + + routes = {} + for a_route in routes_json: + # empty community string + routes[a_route] = "" + results[hostname] = routes + + all_routes = parallel_run(parse_routes_process_vsonic, (), {}, list(neigh_hosts.values()), + timeout=120, concurrent_tasks=8) + return all_routes + + +def verify_only_loopback_routes_are_announced_to_neighs(dut_hosts, duthost, neigh_hosts, community): + """ + Verify only loopback routes with certain community are announced to neighs in TSA + """ + return verify_loopback_route_with_community(dut_hosts, duthost, neigh_hosts, 4, community) and \ + verify_loopback_route_with_community( + dut_hosts, duthost, neigh_hosts, 6, community) + + +def parse_routes_on_neighbors(dut_host, neigh_hosts, ip_ver, exp_community=[]): + if isinstance(list(neigh_hosts.items())[0][1]['host'], EosHost): + routes_on_all_nbrs = parse_routes_on_eos(dut_host, neigh_hosts, ip_ver, exp_community) + else: + routes_on_all_nbrs = parse_routes_on_vsonic( + dut_host, neigh_hosts, ip_ver) + return routes_on_all_nbrs + + +def verify_current_routes_announced_to_neighs(dut_host, neigh_hosts, orig_routes_on_all_nbrs, + cur_routes_on_all_nbrs, ip_ver, exp_community=[]): + """ + Verify all the original routes are announced to neighbors after TSB + """ + logger.info( + "Verifying all the original routes(ipv{}) are announced to bgp neighbors".format(ip_ver)) + cur_routes_on_all_nbrs.update( + parse_routes_on_neighbors(dut_host, neigh_hosts, ip_ver, exp_community)) + # Compare current routes after TSB with original routes advertised to neighbors + if cur_routes_on_all_nbrs != orig_routes_on_all_nbrs: + return False + return True + + +def check_and_log_routes_diff(duthost, neigh_hosts, orig_routes_on_all_nbrs, cur_routes_on_all_nbrs, ip_ver): + cur_nbrs = set(cur_routes_on_all_nbrs.keys()) + orig_nbrs = set(orig_routes_on_all_nbrs.keys()) + if cur_nbrs != orig_nbrs: + logger.warn("Neighbor list mismatch: {}".format(cur_nbrs ^ orig_nbrs)) + return False + + routes_dut = parse_rib(duthost, ip_ver) + all_diffs_in_host_aspath = True + for hostname in list(orig_routes_on_all_nbrs.keys()): + if orig_routes_on_all_nbrs[hostname] != cur_routes_on_all_nbrs[hostname]: + routes_diff = set(orig_routes_on_all_nbrs[hostname]) ^ set( + cur_routes_on_all_nbrs[hostname]) + for route in routes_diff: + if route not in list(routes_dut.keys()): + all_diffs_in_host_aspath = False + logger.warn( + "Missing route on host {}: {}".format(hostname, route)) + continue + aspaths = routes_dut[route] + # Filter out routes announced by this neigh + skip = False + if isinstance(list(neigh_hosts.items())[0][1]['host'], EosHost): + for aspath in aspaths: + if str(neigh_hosts[hostname]['conf']['bgp']['asn']) in aspath: + logger.debug( + "Skipping route {} on host {}".format(route, hostname)) + skip = True + break + if not skip: + all_diffs_in_host_aspath = False + if route in orig_routes_on_all_nbrs[hostname]: + logger.warn( + "Missing route on host {}: {}".format(hostname, route)) + else: + logger.warn( + "Additional route on host {}: {}".format(hostname, route)) + + return all_diffs_in_host_aspath diff --git a/tests/bgp/test_bgp_4-byte_as_trans.py b/tests/bgp/test_bgp_4-byte_as_trans.py new file mode 100644 index 0000000000..65d81233b1 --- /dev/null +++ b/tests/bgp/test_bgp_4-byte_as_trans.py @@ -0,0 +1,296 @@ +''' + +The test case will verify that the DUT supports 4-byte AS Number translation. + +Step 1: Configure DUT and neighbor with 4-Byte ASN +Step 2: Verify 4-byte BGP session between DUT and neighbor is established +Step 3: Verify BGP is established for 4-byte neighbor and down for 2-byte neighbors +Step 3: Configure DUT to use 2-byte local-ASN for 2-byte neighbors +Step 4: Verify BGP is now established for 4-byte neighbor AND 2-byte neighbors +Step 5: Verify 2-byte neighbors receive routes from upstream 4-byte routers + + +''' +import logging + +import pytest +import time +import textfsm +from tests.common.config_reload import config_reload + +logger = logging.getLogger(__name__) +dut_4byte_asn = 400003 +neighbor_4byte_asn = 400001 +bgp_sleep = 120 +bgp_id_textfsm = "./bgp/templates/bgp_id.template" + +pytestmark = [ + pytest.mark.topology('t2') +] + + +@pytest.fixture(scope='module') +def setup(tbinfo, nbrhosts, duthosts, enum_frontend_dut_hostname, enum_rand_one_frontend_asic_index, request): + # verify neighbors are type sonic + if request.config.getoption("neighbor_type") != "sonic": + pytest.skip("Neighbor type must be sonic") + duthost = duthosts[enum_frontend_dut_hostname] + asic_index = enum_rand_one_frontend_asic_index + + if duthost.is_multi_asic: + cli_options = " -n " + duthost.get_namespace_from_asic_id(asic_index) + else: + cli_options = '' + + dut_asn = tbinfo['topo']['properties']['configuration_properties']['common']['dut_asn'] + neigh1 = duthost.shell("show lldp table")['stdout'].split("\n")[3].split()[1] + neigh2 = duthost.shell("show lldp table")['stdout'].split("\n")[5].split()[1] + + neighbors = dict() + skip_hosts = duthost.get_asic_namespace_list() + bgp_facts = duthost.bgp_facts(instance_id=asic_index)['ansible_facts'] + neigh_asn = dict() + + # verify sessions are established and gather neighbor information + for k, v in bgp_facts['bgp_neighbors'].items(): + if v['description'].lower() not in skip_hosts: + if v['description'] == neigh1: + if v['ip_version'] == 4: + neigh_ip_v4 = k + peer_group_v4 = v['peer group'] + elif v['ip_version'] == 6: + neigh_ip_v6 = k + peer_group_v6 = v['peer group'] + if v['description'] == neigh2: + if v['ip_version'] == 4: + neigh2_ip_v4 = k + elif v['ip_version'] == 6: + neigh2_ip_v6 = k + assert v['state'] == 'established' + neigh_asn[v['description']] = v['remote AS'] + neighbors[v['description']] = nbrhosts[v['description']]["host"] + if neighbors[neigh1].is_multi_asic: + neigh_cli_options = " -n " + neigh1.get_namespace_from_asic_id(asic_index) + else: + neigh_cli_options = '' + + dut_ip_v4 = tbinfo['topo']['properties']['configuration'][neigh1]['bgp']['peers'][dut_asn][0] + dut_ip_v6 = tbinfo['topo']['properties']['configuration'][neigh1]['bgp']['peers'][dut_asn][1] + + dut_ip_bgp_sum = duthost.shell('show ip bgp summary')['stdout'] + neigh_ip_bgp_sum = nbrhosts[neigh1]["host"].shell('show ip bgp summary')['stdout'] + neigh2_ip_bgp_sum = nbrhosts[neigh1]["host"].shell('show ip bgp summary')['stdout'] + with open(bgp_id_textfsm) as template: + fsm = textfsm.TextFSM(template) + dut_bgp_id = fsm.ParseText(dut_ip_bgp_sum)[0][0] + neigh_bgp_id = fsm.ParseText(neigh_ip_bgp_sum)[1][0] + neigh2_bgp_id = fsm.ParseText(neigh2_ip_bgp_sum)[1][0] + + dut_ipv4_network = duthost.shell("show run bgp | grep 'ip prefix-list'")['stdout'].split()[6] + dut_ipv6_network = duthost.shell("show run bgp | grep 'ipv6 prefix-list'")['stdout'].split()[6] + neigh_ipv4_network = nbrhosts[neigh1]["host"].shell("show run bgp | grep 'ip prefix-list'")['stdout'].split()[6] + neigh_ipv6_network = nbrhosts[neigh1]["host"].shell("show run bgp | grep 'ipv6 prefix-list'")['stdout'].split()[6] + + setup_info = { + 'duthost': duthost, + 'neighhost': neighbors[neigh1], + 'neigh2host': neighbors[neigh2], + 'neigh1': neigh1, + 'neigh2': neigh2, + 'dut_asn': dut_asn, + 'neigh_asn': neigh_asn[neigh1], + 'neigh2_asn': neigh_asn[neigh2], + 'asn_dict': neigh_asn, + 'neighbors': neighbors, + 'cli_options': cli_options, + 'neigh_cli_options': neigh_cli_options, + 'dut_ip_v4': dut_ip_v4, + 'dut_ip_v6': dut_ip_v6, + 'neigh_ip_v4': neigh_ip_v4, + 'neigh_ip_v6': neigh_ip_v6, + 'neigh2_ip_v4': neigh2_ip_v4, + 'neigh2_ip_v6': neigh2_ip_v6, + 'peer_group_v4': peer_group_v4, + 'peer_group_v6': peer_group_v6, + 'asic_index': asic_index, + 'dut_bgp_id': dut_bgp_id, + 'neigh_bgp_id': neigh_bgp_id, + 'neigh2_bgp_id': neigh2_bgp_id, + 'dut_ipv4_network': dut_ipv4_network, + 'dut_ipv6_network': dut_ipv6_network, + 'neigh_ipv4_network': neigh_ipv4_network, + 'neigh_ipv6_network': neigh_ipv6_network + } + + logger.debug("DUT BGP Config: {}".format(duthost.shell("show run bgp", module_ignore_errors=True)['stdout'])) + logger.debug("Neighbor BGP Config: {}".format(nbrhosts[neigh1]["host"].shell("show run bgp")['stdout'])) + logger.debug('Setup_info: {}'.format(setup_info)) + + yield setup_info + + # restore config to original state + config_reload(duthost) + config_reload(neighbors[neigh1], is_dut=False) + + # verify sessions are established + bgp_facts = duthost.bgp_facts(instance_id=asic_index)['ansible_facts'] + for k, v in bgp_facts['bgp_neighbors'].items(): + if v['description'].lower() not in skip_hosts: + logger.debug(v['description']) + assert v['state'] == 'established' + + +def test_4_byte_asn_translation(setup): + # copy existing BGP config to a new 4-byte ASN. Use existing route-maps for consistancy. + cmd = 'vtysh{} \ + -c "config" \ + -c "no router bgp {}" \ + -c "router bgp {}" \ + -c "bgp router-id {}" \ + -c "bgp log-neighbor-changes" \ + -c "no bgp ebgp-requires-policy" \ + -c "no bgp default ipv4-unicast" \ + -c "bgp bestpath as-path multipath-relax" \ + -c "neighbor {} peer-group" \ + -c "neighbor {} peer-group" \ + -c "neighbor {} remote-as {}" \ + -c "neighbor {} peer-group {}" \ + -c "neighbor {} description {}" \ + -c "neighbor {} timers 3 10" \ + -c "neighbor {} timers connect 10" \ + -c "neighbor {} remote-as {}" \ + -c "neighbor {} peer-group {}" \ + -c "neighbor {} description {}" \ + -c "neighbor {} timers 3 10" \ + -c "neighbor {} timers connect 10" \ + -c "neighbor {} remote-as {}" \ + -c "neighbor {} peer-group {}" \ + -c "neighbor {} description {}" \ + -c "neighbor {} timers 3 10" \ + -c "neighbor {} timers connect 10" \ + -c "neighbor {} remote-as {}" \ + -c "neighbor {} peer-group {}" \ + -c "neighbor {} description {}" \ + -c "neighbor {} timers 3 10" \ + -c "neighbor {} timers connect 10" \ + -c "address-family ipv4 unicast" \ + -c "network {}" \ + -c "neighbor {} soft-reconfiguration inbound" \ + -c "neighbor {} route-map FROM_BGP_PEER_V4 in" \ + -c "neighbor {} route-map TO_BGP_PEER_V4 out" \ + -c "neighbor {} activate" \ + -c "neighbor {} activate" \ + -c "maximum-paths 64" \ + -c "exit-address-family" \ + -c "address-family ipv6 unicast" \ + -c "network {}" \ + -c "neighbor {} soft-reconfiguration inbound" \ + -c "neighbor {} route-map FROM_BGP_PEER_V6 in" \ + -c "neighbor {} route-map TO_BGP_PEER_V6 out" \ + -c "neighbor {} activate" \ + -c "neighbor {} activate" \ + -c "maximum-paths 64" \ + -c "exit-address-family" \ + '.format(setup['cli_options'], setup['dut_asn'], dut_4byte_asn, setup['dut_bgp_id'], + setup['peer_group_v4'], setup['peer_group_v6'], setup['neigh_ip_v4'], neighbor_4byte_asn, + setup['neigh_ip_v4'], setup['peer_group_v4'], setup['neigh_ip_v4'], setup['neigh1'], setup['neigh_ip_v4'], + setup['neigh_ip_v4'], setup['neigh_ip_v6'], neighbor_4byte_asn, setup['neigh_ip_v6'], + setup['peer_group_v6'], setup['neigh_ip_v6'], setup['neigh1'], setup['neigh_ip_v6'], setup['neigh_ip_v6'], + setup['neigh2_ip_v4'], setup['neigh2_asn'], setup['neigh2_ip_v4'], setup['peer_group_v4'], + setup['neigh2_ip_v4'], setup['neigh2'], setup['neigh2_ip_v4'], setup['neigh2_ip_v4'], + setup['neigh2_ip_v6'], setup['neigh2_asn'], setup['neigh2_ip_v6'], setup['peer_group_v6'], + setup['neigh2_ip_v6'], setup['neigh2'], setup['neigh2_ip_v6'], setup['neigh2_ip_v6'], + setup['dut_ipv4_network'], setup['peer_group_v4'], setup['peer_group_v4'], setup['peer_group_v4'], + setup['neigh_ip_v4'], setup['neigh2_ip_v4'], setup['dut_ipv6_network'], setup['peer_group_v6'], + setup['peer_group_v6'], setup['peer_group_v6'], setup['neigh_ip_v6'], setup['neigh2_ip_v6']) + logger.debug(setup['duthost'].shell(cmd, module_ignore_errors=True)) + + # configure 4-byte ASN on neighbor device, reusing the established BGP config and route-maps + cmd = 'vtysh{} \ + -c "config" \ + -c "no router bgp {}" \ + -c "router bgp {}" \ + -c "bgp router-id {}" \ + -c "bgp log-neighbor-changes" \ + -c "no bgp ebgp-requires-policy" \ + -c "no bgp default ipv4-unicast" \ + -c "bgp bestpath as-path multipath-relax" \ + -c "neighbor {} peer-group" \ + -c "neighbor {} peer-group" \ + -c "neighbor {} remote-as {}" \ + -c "neighbor {} peer-group {}" \ + -c "neighbor {} description {}" \ + -c "neighbor {} timers 3 10" \ + -c "neighbor {} timers connect 10" \ + -c "neighbor {} remote-as {}" \ + -c "neighbor {} peer-group {}" \ + -c "neighbor {} description {}" \ + -c "neighbor {} timers 3 10" \ + -c "neighbor {} timers connect 10" \ + -c "address-family ipv4 unicast" \ + -c "network {}" \ + -c "neighbor {} soft-reconfiguration inbound" \ + -c "neighbor {} route-map FROM_BGP_PEER_V4 in" \ + -c "neighbor {} route-map TO_BGP_PEER_V4 out" \ + -c "neighbor {} activate" \ + -c "maximum-paths 64" \ + -c "exit-address-family" \ + -c "address-family ipv6 unicast" \ + -c "network {}" \ + -c "neighbor {} soft-reconfiguration inbound" \ + -c "neighbor {} route-map FROM_BGP_PEER_V6 in" \ + -c "neighbor {} route-map TO_BGP_PEER_V6 out" \ + -c "neighbor {} activate" \ + -c "maximum-paths 64" \ + -c "exit-address-family" \ + '.format(setup['neigh_cli_options'], setup['neigh_asn'], neighbor_4byte_asn, setup['neigh_bgp_id'], + setup['peer_group_v4'], setup['peer_group_v6'], setup['dut_ip_v4'], dut_4byte_asn, setup['dut_ip_v4'], + setup['peer_group_v4'], setup['dut_ip_v4'], 'DUT', setup['dut_ip_v4'], setup['dut_ip_v4'], + setup['dut_ip_v6'], dut_4byte_asn, setup['dut_ip_v6'], setup['peer_group_v6'], setup['dut_ip_v6'], 'DUT', + setup['dut_ip_v6'], setup['dut_ip_v6'], setup['neigh_ipv4_network'], setup['peer_group_v4'], + setup['peer_group_v4'], setup['peer_group_v4'], setup['dut_ip_v4'], setup['neigh_ipv6_network'], + setup['peer_group_v6'], setup['peer_group_v6'], setup['peer_group_v6'], setup['dut_ip_v6']) + + logger.debug(setup['neighhost'].shell(cmd, module_ignore_errors=True)) + + logger.debug("DUT BGP Config: {}".format(setup['duthost'].shell("show run bgp")['stdout'])) + logger.debug("Neighbor BGP Config: {}".format(setup['neighhost'].shell("show run bgp")['stdout'])) + + time.sleep(bgp_sleep) + + # verify session to 4-byte neighbor is established and 2-byte neighbor is down + bgp_facts = setup['duthost'].bgp_facts(instance_id=setup['asic_index'])['ansible_facts'] + for k, v in bgp_facts['bgp_neighbors'].items(): + if v['description'].lower() == setup['neigh1']: + logger.debug(v['description']) + assert v['state'] == 'established' + elif v['description'].lower() == setup['neigh2']: + logger.debug(v['description']) + assert v['state'] != 'established' + + # Configure DUT to use 2-byte local-ASN for 2-byte neighbors + cmd = 'vtysh{} \ + -c "config" \ + -c "router bgp {}" \ + -c "neighbor {} local-as {}" \ + -c "neighbor {} local-as {}" \ + '.format(setup['cli_options'], dut_4byte_asn, setup['peer_group_v4'], setup['dut_asn'], setup['peer_group_v6'], + setup['dut_asn']) + logger.debug(setup['duthost'].shell(cmd, module_ignore_errors=True)) + cmd = 'vtysh{} -c "clear bgp *"'.format(setup['cli_options']) + logger.debug(setup['duthost'].shell(cmd, module_ignore_errors=True)) + + time.sleep(bgp_sleep) + + # verify session to 4-byte and 2-byte neighbor is established + bgp_facts = setup['duthost'].bgp_facts(instance_id=setup['asic_index'])['ansible_facts'] + for k, v in bgp_facts['bgp_neighbors'].items(): + if v['description'].lower() == setup['neigh1']: + logger.debug(v['description']) + assert v['state'] == 'established' + elif v['description'].lower() == setup['neigh2']: + logger.debug(v['description']) + assert v['state'] == 'established' + + cmd = 'vtysh -c "show bgp ipv4 all neighbors {}" received-routes'.format(setup['neigh_ipv4_network']) + logger.debug(setup['neigh2host'].shell(cmd, module_ignore_errors=True)) diff --git a/tests/bgp/test_bgp_allow_list.py b/tests/bgp/test_bgp_allow_list.py index d193707af8..877ddc9872 100644 --- a/tests/bgp/test_bgp_allow_list.py +++ b/tests/bgp/test_bgp_allow_list.py @@ -11,7 +11,6 @@ from bgp_helpers import check_routes_on_neighbors_empty_allow_list, checkout_bgp_mon_routes, check_routes_on_neighbors # Fixtures from bgp_helpers import bgp_allow_list_setup, prepare_eos_routes # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('t1'), diff --git a/tests/bgp/test_bgp_authentication.py b/tests/bgp/test_bgp_authentication.py index f739160804..722b87fe66 100644 --- a/tests/bgp/test_bgp_authentication.py +++ b/tests/bgp/test_bgp_authentication.py @@ -7,7 +7,6 @@ from tests.common.helpers.constants import DEFAULT_NAMESPACE from tests.common.config_reload import config_reload -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/bgp/test_bgp_bbr.py b/tests/bgp/test_bgp_bbr.py index ea53f40623..e741bdc7ed 100644 --- a/tests/bgp/test_bgp_bbr.py +++ b/tests/bgp/test_bgp_bbr.py @@ -14,14 +14,14 @@ from jinja2 import Template from natsort import natsorted +from tests.common.config_reload import config_reload from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.constants import DEFAULT_NAMESPACE from tests.common.helpers.parallel import reset_ansible_local_tmp from tests.common.helpers.parallel import parallel_run from tests.common.utilities import wait_until, delete_running_config -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from tests.common.gu_utils import apply_patch, expect_op_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile pytestmark = [ @@ -154,23 +154,41 @@ def config_bbr_enabled(duthosts, setup, rand_one_dut_hostname, restore_bbr_defau enable_bbr(duthost, setup['tor1_namespace']) +def get_bbr_default_state(duthost): + bbr_supported = False + bbr_default_state = 'disabled' + + # Check BBR configuration from config_db first + bbr_config_db_exist = int(duthost.shell('redis-cli -n 4 HEXISTS "BGP_BBR|all" "status"')["stdout"]) + if bbr_config_db_exist: + # key exist, BBR is supported + bbr_supported = True + bbr_default_state = duthost.shell('redis-cli -n 4 HGET "BGP_BBR|all" "status"')["stdout"] + else: + # Check BBR configuration from constants.yml + constants = yaml.safe_load(duthost.shell('cat {}'.format(CONSTANTS_FILE))['stdout']) + try: + bbr_supported = constants['constants']['bgp']['bbr']['enabled'] + if not bbr_supported: + return bbr_supported, bbr_default_state + bbr_default_state = constants['constants']['bgp']['bbr']['default_state'] + except KeyError: + return bbr_supported, bbr_default_state + + return bbr_supported, bbr_default_state + + @pytest.fixture(scope='module') def setup(duthosts, rand_one_dut_hostname, tbinfo, nbrhosts): duthost = duthosts[rand_one_dut_hostname] constants_stat = duthost.stat(path=CONSTANTS_FILE) if not constants_stat['stat']['exists']: - pytest.skip('No file {} on DUT, BBR is not supported') + pytest.skip('No constants.yml file on DUT, BBR is not supported') - constants = yaml.safe_load(duthost.shell('cat {}'.format(CONSTANTS_FILE))['stdout']) - bbr_default_state = 'disabled' - try: - bbr_enabled = constants['constants']['bgp']['bbr']['enabled'] - if not bbr_enabled: - pytest.skip('BGP BBR is not enabled') - bbr_default_state = constants['constants']['bgp']['bbr']['default_state'] - except KeyError: - pytest.skip('No BBR configuration in {}, BBR is not supported.'.format(CONSTANTS_FILE)) + bbr_supported, bbr_default_state = get_bbr_default_state(duthost) + if not bbr_supported: + pytest.skip('BGP BBR is not supported') mg_facts = duthost.get_extended_minigraph_facts(tbinfo) @@ -417,3 +435,28 @@ def test_bbr_disabled_dut_asn_in_aspath(duthosts, rand_one_dut_hostname, nbrhost prepare_routes([bbr_route, bbr_route_v6]) for route in (bbr_route, bbr_route_v6): check_bbr_route_propagation(duthost, nbrhosts, setup, route, accepted=False) + + +@pytest.mark.parametrize('bbr_status', ['enabled', 'disabled']) +def test_bbr_status_consistent_after_reload(duthosts, rand_one_dut_hostname, setup, + bbr_status, restore_bbr_default_state): + duthost = duthosts[rand_one_dut_hostname] + if setup['tor1_namespace']: + pytest.skip('Skip test for multi-asic environment') + + # Set BBR status in config_db + duthost.shell('redis-cli -n 4 HSET "BGP_BBR|all" "status" "{}" '.format(bbr_status)) + duthost.shell('sudo config save -y') + config_reload(duthost) + + # Verify BBR status after config reload + bbr_status_after_reload = duthost.shell('redis-cli -n 4 HGET "BGP_BBR|all" "status"')["stdout"] + pytest_assert(bbr_status_after_reload == bbr_status, "BGP BBR status is not consistent after config reload") + + # Check if BBR is enabled or disabled using the running configuration + bbr_status_running_config = duthost.shell("show runningconfiguration bgp | grep allowas", module_ignore_errors=True)\ + ['stdout'] # noqa E211 + if bbr_status == 'enabled': + pytest_assert('allowas-in' in bbr_status_running_config, "BGP BBR is not enabled in running configuration") + else: + pytest_assert('allowas-in' not in bbr_status_running_config, "BGP BBR is not disabled in running configuration") diff --git a/tests/bgp/test_bgp_bbr_default_state.py b/tests/bgp/test_bgp_bbr_default_state.py new file mode 100644 index 0000000000..bf6019c671 --- /dev/null +++ b/tests/bgp/test_bgp_bbr_default_state.py @@ -0,0 +1,134 @@ +'''This script is to test the BGP Bounce Back Routing (BBR) feature default state after restart. +''' +import json +import logging +import time +import pytest +from jinja2 import Template +from natsort import natsorted +from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.constants import DEFAULT_NAMESPACE +from tests.common.utilities import delete_running_config +from tests.common.gu_utils import apply_patch, expect_op_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.config_reload import config_reload + + +pytestmark = [ + pytest.mark.topology('t1'), + pytest.mark.device_type('vs') + ] + + +logger = logging.getLogger(__name__) + +CONSTANTS_FILE = '/etc/sonic/constants.yml' + + +@pytest.fixture(scope='module', autouse=True) +def prepare_bbr_config_files(duthosts, rand_one_dut_hostname): + duthost = duthosts[rand_one_dut_hostname] + bgp_bbr_config = Template(open("./bgp/templates/bgp_bbr_config.json.j2").read()) + duthost.copy(content=bgp_bbr_config.render(BGP_BBR_STATUS='disabled'), dest='/tmp/disable_bbr.json') + duthost.copy(content=bgp_bbr_config.render(BGP_BBR_STATUS='enabled'), dest='/tmp/enable_bbr.json') + yield + del_bbr_json = [{"BGP_BBR": {}}] + delete_running_config(del_bbr_json, duthost) + + +@pytest.fixture(scope='module') +def bbr_default_state(setup): + return setup['bbr_default_state'] + + +def add_bbr_config_to_running_config(duthost, status): + logger.info('Add BGP_BBR config to running config') + json_patch = [ + { + "op": "add", + "path": "/BGP_BBR", + "value": { + "all": { + "status": "{}".format(status) + } + } + } + ] + tmpfile = generate_tmpfile(duthost) + logger.info("tmpfile {}".format(tmpfile)) + try: + output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) + expect_op_success(duthost, output) + finally: + delete_tmpfile(duthost, tmpfile) + time.sleep(3) + + +def config_bbr_by_gcu(duthost, status): + logger.info('Config BGP_BBR by GCU cmd') + json_patch = [ + { + "op": "replace", + "path": "/BGP_BBR/all/status", + "value": "{}".format(status) + } + ] + tmpfile = generate_tmpfile(duthost) + logger.info("tmpfile {}".format(tmpfile)) + try: + output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) + expect_op_success(duthost, output) + finally: + delete_tmpfile(duthost, tmpfile) + time.sleep(3) + + +def disable_bbr(duthost, namespace): + logger.info('Disable BGP_BBR') + # gcu doesn't support multi-asic for now, use sonic-cfggen instead + if namespace: + logger.info('Disable BGP_BBR in namespace {}'.format(namespace)) + duthost.shell('sonic-cfggen {} -j /tmp/disable_bbr.json -w '.format('-n ' + namespace)) + time.sleep(3) + else: + config_bbr_by_gcu(duthost, "disabled") + + +@pytest.fixture +def config_bbr_disabled(duthosts, setup, rand_one_dut_hostname): + duthost = duthosts[rand_one_dut_hostname] + disable_bbr(duthost, setup['tor1_namespace']) + + +@pytest.fixture(scope='module') +def setup(duthosts, rand_one_dut_hostname, tbinfo, nbrhosts): + duthost = duthosts[rand_one_dut_hostname] + constants_stat = duthost.stat(path=CONSTANTS_FILE) + if not constants_stat['stat']['exists']: + pytest.skip('No file {} on DUT, BBR is not supported') + bbr_default_state = 'disabled' + mg_facts = duthost.get_extended_minigraph_facts(tbinfo) + tor_neighbors = natsorted([neighbor for neighbor in list(nbrhosts.keys()) if neighbor.endswith('T0')]) + tor1 = tor_neighbors[0] + tor1_namespace = DEFAULT_NAMESPACE + for dut_port, neigh in list(mg_facts['minigraph_neighbors'].items()): + if tor1 == neigh['name']: + tor1_namespace = neigh['namespace'] + break + setup_info = { + 'bbr_default_state': bbr_default_state, + 'tor1_namespace': tor1_namespace, + } + if not setup_info['tor1_namespace']: + logger.info('non multi-asic environment, add bbr config to running config using gcu cmd') + add_bbr_config_to_running_config(duthost, bbr_default_state) + logger.info('setup_info: {}'.format(json.dumps(setup_info, indent=2))) + return setup_info + + +def test_bbr_disabled_constants_yml_default(duthosts, rand_one_dut_hostname, setup, config_bbr_disabled): + duthost = duthosts[rand_one_dut_hostname] + duthost.shell("sudo config save -y") + config_reload(duthost) + is_bbr_enabled = duthost.shell("show runningconfiguration bgp | grep allowas", module_ignore_errors=True)['stdout'] + pytest_assert(is_bbr_enabled == "", "BBR is not disabled when it should be.") diff --git a/tests/bgp/test_bgp_bounce.py b/tests/bgp/test_bgp_bounce.py index 524071f3dc..4e7b3166b4 100644 --- a/tests/bgp/test_bgp_bounce.py +++ b/tests/bgp/test_bgp_bounce.py @@ -10,7 +10,6 @@ from bgp_helpers import apply_bgp_config from bgp_helpers import get_no_export_output from bgp_helpers import BGP_ANNOUNCE_TIME -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('t1') diff --git a/tests/bgp/test_bgp_command.py b/tests/bgp/test_bgp_command.py new file mode 100644 index 0000000000..7098eb093a --- /dev/null +++ b/tests/bgp/test_bgp_command.py @@ -0,0 +1,105 @@ +import pytest +import logging +import re + +from tests.common.helpers.assertions import pytest_assert + + +pytestmark = [ + pytest.mark.topology("t0", "t1", "m0", "mx"), + pytest.mark.device_type('vs') +] + +logger = logging.getLogger(__name__) + + +# Function to parse the "Displayed X routes and Y total paths" line +def parse_routes_and_paths(output): + match = re.search(r"Displayed\s+(\d+)\s+routes\s+and\s+(\d+)\s+total paths", output) + if match: + routes = int(match.group(1)) + paths = int(match.group(2)) + return routes, paths + return None + + +@pytest.mark.parametrize("ip_version", ["ipv4", "ipv6"]) +def test_bgp_network_command( + duthosts, enum_rand_one_per_hwsku_frontend_hostname, ip_version +): + """ + @summary: This test case is to verify the output of "show ip bgp network" command + if it matches the output of "docker exec -i bgp vtysh -c show bgp ipv4 all" command + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + if ip_version == "ipv4": + bgp_network_cmd = "show ip bgp network" + bgp_docker_cmd = 'docker exec -i bgp vtysh -c "show bgp ipv4 all"' + elif ip_version == "ipv6": + bgp_network_cmd = "show ipv6 bgp network" + bgp_docker_cmd = 'docker exec -i bgp vtysh -c "show bgp ipv6 all"' + + bgp_network_result = duthost.shell(bgp_network_cmd) + bgp_network_output = bgp_network_result["stdout"] + pytest_assert( + bgp_network_result["rc"] == 0, + "{} return value is not 0, output={}".format( + bgp_network_cmd, bgp_network_output + ), + ) + pytest_assert( + "*=" in bgp_network_output, + "Failed to run '{}' command, output={}".format( + bgp_network_cmd, bgp_network_output + ), + ) + + bgp_docker_result = duthost.shell(bgp_docker_cmd) + bgp_docker_output = bgp_docker_result["stdout"] + pytest_assert( + bgp_docker_result["rc"] == 0, + "{} return value is not 0, output:{}".format(bgp_docker_cmd, bgp_docker_output), + ) + pytest_assert( + "*=" in bgp_docker_output, + "Failed to run '{}' command, output={}".format( + bgp_docker_cmd, bgp_docker_output + ), + ) + # Remove the first two lines from the docker command output + bgp_docker_output_lines = bgp_docker_output.splitlines()[2:] + bgp_docker_output_modified = "\n".join(bgp_docker_output_lines) + + pytest_assert( + bgp_network_output == bgp_docker_output_modified, + "The output of {} and {} mismatch".format(bgp_network_cmd, bgp_docker_cmd), + ) + + # Parse routes and paths from both outputs + bgp_network_routes_and_paths = parse_routes_and_paths(bgp_network_output) + bgp_docker_routes_and_paths = parse_routes_and_paths(bgp_docker_output) + logger.info( + "Routes and paths from '{}': {}".format( + bgp_network_cmd, bgp_network_routes_and_paths + ) + ) + logger.info( + "Routes and paths from '{}': {}".format( + bgp_docker_cmd, bgp_docker_routes_and_paths + ) + ) + + if bgp_network_routes_and_paths is None or bgp_docker_routes_and_paths is None: + pytest_assert( + bgp_network_routes_and_paths is not None + and bgp_docker_routes_and_paths is not None, + "Failed to parse routes and paths from one of the outputs.", + ) + + # Compare the routes and paths + pytest_assert( + bgp_network_routes_and_paths == bgp_docker_routes_and_paths, + "Routes and total path value are mismatched: {} != {}".format( + bgp_network_routes_and_paths, bgp_docker_routes_and_paths + ), + ) diff --git a/tests/bgp/test_bgp_dual_asn.py b/tests/bgp/test_bgp_dual_asn.py index 3d7d742bee..9ad279d95f 100644 --- a/tests/bgp/test_bgp_dual_asn.py +++ b/tests/bgp/test_bgp_dual_asn.py @@ -13,15 +13,14 @@ from tests.common.helpers.assertions import pytest_assert from bgp_helpers import update_routes from tests.generic_config_updater.test_bgp_speaker import get_bgp_speaker_runningconfig -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import ( +from tests.common.gu_utils import apply_patch, expect_op_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import ( create_checkpoint, delete_checkpoint, rollback_or_reload, ) from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [pytest.mark.topology("t0")] diff --git a/tests/bgp/test_bgp_fact.py b/tests/bgp/test_bgp_fact.py index f5dc5bf437..7221fc157a 100644 --- a/tests/bgp/test_bgp_fact.py +++ b/tests/bgp/test_bgp_fact.py @@ -1,5 +1,5 @@ import pytest -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from tests.common.helpers.bgp import run_bgp_facts pytestmark = [ pytest.mark.topology('any'), @@ -7,41 +7,5 @@ ] -def run_bgp_facts(duthosts, enum_frontend_dut_hostname, enum_asic_index): - """compare the bgp facts between observed states and target state""" - - duthost = duthosts[enum_frontend_dut_hostname] - - bgp_facts = duthost.bgp_facts(instance_id=enum_asic_index)['ansible_facts'] - namespace = duthost.get_namespace_from_asic_id(enum_asic_index) - config_facts = duthost.config_facts(host=duthost.hostname, source="running", namespace=namespace)['ansible_facts'] - sonic_db_cmd = "sonic-db-cli {}".format("-n " + namespace if namespace else "") - for k, v in list(bgp_facts['bgp_neighbors'].items()): - # Verify bgp sessions are established - assert v['state'] == 'established' - # Verify local ASNs in bgp sessions - assert v['local AS'] == int(config_facts['DEVICE_METADATA']['localhost']['bgp_asn'].encode().decode("utf-8")) - # Check bgpmon functionality by validate STATE DB contains this neighbor as well - state_fact = duthost.shell('{} STATE_DB HGET "NEIGH_STATE_TABLE|{}" "state"' - .format(sonic_db_cmd, k), module_ignore_errors=False)['stdout_lines'] - peer_type = duthost.shell('{} STATE_DB HGET "NEIGH_STATE_TABLE|{}" "peerType"' - .format(sonic_db_cmd, k), - module_ignore_errors=False)['stdout_lines'] - assert state_fact[0] == "Established" - assert peer_type[0] == "i-BGP" if v['remote AS'] == v['local AS'] else "e-BGP" - - # In multi-asic, would have 'BGP_INTERNAL_NEIGHBORS' and possibly no 'BGP_NEIGHBOR' (ebgp) neighbors. - nbrs_in_cfg_facts = {} - nbrs_in_cfg_facts.update(config_facts.get('BGP_NEIGHBOR', {})) - nbrs_in_cfg_facts.update(config_facts.get('BGP_INTERNAL_NEIGHBOR', {})) - # In VoQ Chassis, we would have BGP_VOQ_CHASSIS_NEIGHBOR as well. - nbrs_in_cfg_facts.update(config_facts.get('BGP_VOQ_CHASSIS_NEIGHBOR', {})) - for k, v in list(nbrs_in_cfg_facts.items()): - # Compare the bgp neighbors name with config db bgp neighbors name - assert v['name'] == bgp_facts['bgp_neighbors'][k]['description'] - # Compare the bgp neighbors ASN with config db - assert int(v['asn'].encode().decode("utf-8")) == bgp_facts['bgp_neighbors'][k]['remote AS'] - - def test_bgp_facts(duthosts, enum_frontend_dut_hostname, enum_asic_index): run_bgp_facts(duthosts, enum_frontend_dut_hostname, enum_asic_index) diff --git a/tests/bgp/test_bgp_gr_helper.py b/tests/bgp/test_bgp_gr_helper.py index f6c05b058f..e81bdaa89a 100644 --- a/tests/bgp/test_bgp_gr_helper.py +++ b/tests/bgp/test_bgp_gr_helper.py @@ -7,7 +7,6 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until from tests.common.utilities import is_ipv4_address -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ diff --git a/tests/bgp/test_bgp_multipath_relax.py b/tests/bgp/test_bgp_multipath_relax.py index ea0fafb146..7725439ca3 100644 --- a/tests/bgp/test_bgp_multipath_relax.py +++ b/tests/bgp/test_bgp_multipath_relax.py @@ -1,7 +1,6 @@ import pytest import logging from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('t1') diff --git a/tests/bgp/test_bgp_peer_shutdown.py b/tests/bgp/test_bgp_peer_shutdown.py index ae732da5ae..a678a7ba7a 100644 --- a/tests/bgp/test_bgp_peer_shutdown.py +++ b/tests/bgp/test_bgp_peer_shutdown.py @@ -15,7 +15,7 @@ from tests.common.utilities import wait_until, delete_running_config pytestmark = [ - pytest.mark.topology('t0', 't1'), + pytest.mark.topology('t0', 't1', 't2'), ] TEST_ITERATIONS = 5 @@ -156,13 +156,16 @@ def is_neighbor_session_down(duthost, neighbor): bgp_neighbors[neighbor.ip]["state"] == "idle") -def get_bgp_down_timestamp(duthost, peer_ip, timestamp_before_teardown): +def get_bgp_down_timestamp(duthost, namespace, peer_ip, timestamp_before_teardown): # get the bgp session down timestamp from syslog in the format of seconds (with ms precision) since the Unix Epoch cmd = ( - "grep \"[b]gp#bgpcfgd: Peer 'default|{}' admin state is set to 'down'\" /var/log/syslog | tail -1" - ).format(peer_ip) + "grep \"[b]gp{}#bgpcfgd: Peer 'default|{}' admin state is set to 'down'\" /var/log/syslog | tail -1" + ).format(namespace.split("asic")[1] if namespace else "", peer_ip) bgp_down_msg_list = duthost.shell(cmd)['stdout'].split() + if not bgp_down_msg_list: + pytest.fail("Could not find the BGP session down message in syslog") + try: timestamp = " ".join(bgp_down_msg_list[1:4]) timestamp_in_sec = float(duthost.shell("date -d \"{}\" +%s.%6N".format(timestamp))['stdout']) @@ -230,7 +233,7 @@ def test_bgp_peer_shutdown( bgp_packet.show(dump=True), ) - bgp_session_down_time = get_bgp_down_timestamp(duthost, n0.ip, timestamp_before_teardown) + bgp_session_down_time = get_bgp_down_timestamp(duthost, n0.namespace, n0.ip, timestamp_before_teardown) if not match_bgp_notification(bgp_packet, n0.ip, n0.peer_ip, "cease", bgp_session_down_time): pytest.fail("BGP notification packet does not match expected values") diff --git a/tests/bgp/test_bgp_queue.py b/tests/bgp/test_bgp_queue.py index 5da35b5b64..a87f58f107 100644 --- a/tests/bgp/test_bgp_queue.py +++ b/tests/bgp/test_bgp_queue.py @@ -1,7 +1,6 @@ import time import pytest import logging -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) @@ -60,6 +59,11 @@ def test_bgp_queues(duthosts, enum_frontend_dut_hostname, enum_asic_index, tbinf for k, v in list(bgp_facts['bgp_neighbors'].items()): # Only consider established bgp sessions if v['state'] == 'established': + # For "peer group" if it's internal it will be "INTERNAL_PEER_V4" or "INTERNAL_PEER_V6" + # If it's external it will be "RH_V4", "RH_V6", "AH_V4", "AH_V6", ... + if "INTERNAL" in v["peer group"] and duthost.get_facts().get('modular_chassis'): + # Skip iBGP neighbors since we only want to verify eBGP + continue assert (k in arp_dict.keys() or k in ndp_dict.keys()) if k in arp_dict: ifname = arp_dict[k].split('.', 1)[0] diff --git a/tests/bgp/test_bgp_sentinel.py b/tests/bgp/test_bgp_sentinel.py index 39bf5e7b49..315bb3fb76 100644 --- a/tests/bgp/test_bgp_sentinel.py +++ b/tests/bgp/test_bgp_sentinel.py @@ -13,7 +13,6 @@ from bgp_helpers import BGP_SENTINEL_PORT_V4, BGP_SENTINEL_NAME_V4 from bgp_helpers import BGP_SENTINEL_PORT_V6, BGP_SENTINEL_NAME_V6 from bgp_helpers import BGPMON_TEMPLATE_FILE, BGPMON_CONFIG_FILE, BGP_MONITOR_NAME -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -395,9 +394,27 @@ def prepare_bgp_sentinel_routes(rand_selected_dut, common_setup_teardown, bgp_co announce_route(ptfip, lo_ipv6_addr, route, ptf_bp_v6, BGP_SENTINEL_PORT_V6, community) time.sleep(10) + + # Check if DUT receives the routes that announced from ptf + for ibgp_session in ibgp_sessions: + if request.param == "IPv4": + cmd = "vtysh -c \'show bgp ipv4 neighbors {} received-routes json\'".format(ptf_bp_v4) + else: + cmd = "vtysh -c \'show bgp ipv6 neighbors {} received-routes json\'".format(ptf_bp_v6) + output = json.loads(duthost.shell(cmd)['stdout']) + logger.debug("ibgp_session: {}, neighbors: {}".format(ibgp_session, output)) + # Check if the routes are not announced to ebgp peers with no-export community # or w/o no-export, routes announced to ebgp peers for route in ipv4_routes + ipv6_routes: + # Check the status of signal routes + if route in ipv4_routes: + cmd = "vtysh -c \'show bgp ipv4 {} json\'".format(route) + else: + cmd = "vtysh -c \'show bgp ipv6 {} json\'".format(route) + output = json.loads(duthost.shell(cmd)['stdout']) + logger.debug("route: {}, status: {}".format(route, output)) + if 'no-export' in community: pytest_assert(not is_route_advertised_to_ebgp_peers(duthost, route, ibgp_sessions), "Route {} should not be advertised to bgp peers".format(route)) diff --git a/tests/bgp/test_bgp_session.py b/tests/bgp/test_bgp_session.py new file mode 100644 index 0000000000..75314f4a1a --- /dev/null +++ b/tests/bgp/test_bgp_session.py @@ -0,0 +1,182 @@ +import logging +import pytest +import time +from tests.common.platform.device_utils import fanout_switch_port_lookup +from tests.common.utilities import wait_until +from tests.common.helpers.assertions import pytest_assert +from tests.common.reboot import reboot + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology("t0", "t1"), +] + + +@pytest.fixture(scope='module') +def setup(duthosts, rand_one_dut_hostname, nbrhosts, fanouthosts): + duthost = duthosts[rand_one_dut_hostname] + + config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] + bgp_neighbors = config_facts.get('BGP_NEIGHBOR', {}) + portchannels = config_facts.get('PORTCHANNEL_MEMBER', {}) + dev_nbrs = config_facts.get('DEVICE_NEIGHBOR', {}) + bgp_neighbor = list(bgp_neighbors.keys())[0] + + logger.debug("setup config_facts {}".format(config_facts)) + logger.debug("setup nbrhosts {}".format(nbrhosts)) + logger.debug("setup bgp_neighbors {}".format(bgp_neighbors)) + logger.debug("setup dev_nbrs {}".format(dev_nbrs)) + logger.debug("setup portchannels {}".format(portchannels)) + logger.debug("setup test_neighbor {}".format(bgp_neighbor)) + + # verify sessions are established + pytest_assert(wait_until(30, 5, 0, duthost.check_bgp_session_state, list(bgp_neighbors.keys())), + "Not all BGP sessions are established on DUT") + + ip_intfs = duthost.show_and_parse('show ip interface') + logger.debug("setup ip_intfs {}".format(ip_intfs)) + + # Create a mapping of neighbor IP to interfaces and their details + neighbor_ip_to_interfaces = {} + + # Loop through the ip_intfs list to populate the mapping + for ip_intf in ip_intfs: + neighbor_ip = ip_intf['neighbor ip'] + interface_name = ip_intf['interface'] + if neighbor_ip not in neighbor_ip_to_interfaces: + neighbor_ip_to_interfaces[neighbor_ip] = {} + + # Check if the interface is in portchannels and get the relevant devices + if interface_name in portchannels: + for dev_name in portchannels[interface_name]: + if dev_name in dev_nbrs and dev_nbrs[dev_name]['name'] == ip_intf['bgp neighbor']: + neighbor_ip_to_interfaces[neighbor_ip][dev_name] = dev_nbrs[dev_name] + # If not in portchannels, check directly in dev_nbrs + elif interface_name in dev_nbrs and dev_nbrs[interface_name]['name'] == ip_intf['bgp neighbor']: + neighbor_ip_to_interfaces[neighbor_ip][interface_name] = dev_nbrs[interface_name] + + # Update bgp_neighbors with the new 'interface' key + for ip, details in bgp_neighbors.items(): + if ip in neighbor_ip_to_interfaces: + details['interface'] = neighbor_ip_to_interfaces[ip] + + setup_info = { + 'neighhosts': bgp_neighbors, + "test_neighbor": bgp_neighbor + } + + logger.debug('Setup_info: {}'.format(setup_info)) + + yield setup_info + + # verify sessions are established after test + if not duthost.check_bgp_session_state(bgp_neighbors): + local_interfaces = list(bgp_neighbors[bgp_neighbor]['interface'].keys()) + for port in local_interfaces: + fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, duthost.hostname, port) + if fanout and fanout_port: + logger.info("no shutdown fanout interface, fanout {} port {}".format(fanout, fanout_port)) + fanout.no_shutdown(fanout_port) + neighbor_port = bgp_neighbors[bgp_neighbor]['interface'][port]['port'] + neighbor_name = bgp_neighbors[bgp_neighbor]['name'] + logger.info("no shutdown neighbor interface, neighbor {} port {}".format(neighbor_name, neighbor_port)) + nbrhosts[neighbor_name]['host'].no_shutdown(neighbor_port) + time.sleep(1) + + pytest_assert(wait_until(60, 10, 0, duthost.check_bgp_session_state, list(bgp_neighbors.keys())), + "Not all BGP sessions are established on DUT") + + +def verify_bgp_session_down(duthost, bgp_neighbor): + """Verify the bgp session to the DUT is established.""" + bgp_facts = duthost.bgp_facts()["ansible_facts"] + return ( + bgp_neighbor in bgp_facts["bgp_neighbors"] + and bgp_facts["bgp_neighbors"][bgp_neighbor]["state"] != "established" + ) + + +@pytest.mark.parametrize("test_type", ["bgp_docker", "swss_docker", "reboot"]) +@pytest.mark.parametrize("failure_type", ["interface", "neighbor"]) +@pytest.mark.disable_loganalyzer +def test_bgp_session_interface_down(duthosts, rand_one_dut_hostname, fanouthosts, localhost, + nbrhosts, setup, test_type, failure_type): + ''' + 1: check all bgp sessions are up + 2: inject failure, shutdown fanout physical interface or neighbor port or neighbor session + 4: do the test, reset bgp or swss or do the reboot + 5: Verify all bgp sessions are up + ''' + duthost = duthosts[rand_one_dut_hostname] + + # Skip the test on Virtual Switch due to fanout switch dependency and warm reboot + asic_type = duthost.facts['asic_type'] + if asic_type == "vs" and (failure_type == "interface" or test_type == "reboot"): + pytest.skip("BGP session test is not supported on Virtual Switch") + + # Skip the test if BGP or SWSS autorestart is disabled + autorestart_states = duthost.get_container_autorestart_states() + bgp_autorestart = autorestart_states['bgp'] + swss_autorestart = autorestart_states['swss'] + if bgp_autorestart != "enabled" or swss_autorestart != "enabled": + logger.info("auto restart config bgp {} swss {}".format(bgp_autorestart, swss_autorestart)) + pytest.skip("BGP or SWSS autorestart is disabled") + + neighbor = setup['test_neighbor'] + neighbor_name = setup['neighhosts'][neighbor]['name'] + local_interfaces = list(setup['neighhosts'][neighbor]['interface'].keys()) + + logger.debug("duthost {} neighbor {} interface {} test type {} inject failure type {}".format( + duthost, neighbor_name, local_interfaces, test_type, failure_type)) + + if failure_type == "interface": + for port in local_interfaces: + fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, duthost.hostname, port) + if fanout and fanout_port: + logger.info("shutdown interface fanout {} port {}".format(fanout, fanout_port)) + fanout.shutdown(fanout_port) + time.sleep(1) + + elif failure_type == "neighbor": + for port in local_interfaces: + neighbor_port = setup['neighhosts'][neighbor]['interface'][port]['port'] + logger.info("shutdown interface neighbor {} port {}".format(neighbor_name, neighbor_port)) + nbrhosts[neighbor_name]['host'].shutdown(neighbor_port) + time.sleep(1) + + duthost.shell('show ip bgp summary', module_ignore_errors=True) + pytest_assert( + wait_until(90, 5, 0, verify_bgp_session_down, duthost, neighbor), + "neighbor {} state is still established".format(neighbor) + ) + + if test_type == "bgp_docker": + duthost.shell("docker restart bgp") + elif test_type == "swss_docker": + duthost.shell("docker restart swss") + elif test_type == "reboot": + reboot(duthost, localhost, reboot_type="warm", wait_warmboot_finalizer=True, warmboot_finalizer_timeout=360) + + pytest_assert(wait_until(360, 10, 120, duthost.critical_services_fully_started), + "Not all critical services are fully started") + + if failure_type == "interface": + for port in local_interfaces: + fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, duthost.hostname, port) + if fanout and fanout_port: + logger.info("no shutdown interface fanout {} port {}".format(fanout, fanout_port)) + fanout.no_shutdown(fanout_port) + time.sleep(1) + + elif failure_type == "neighbor": + for port in local_interfaces: + neighbor_port = setup['neighhosts'][neighbor]['interface'][port]['port'] + logger.info("no shutdown interface neighbor {} port {}".format(neighbor_name, neighbor_port)) + nbrhosts[neighbor_name]['host'].no_shutdown(neighbor_port) + time.sleep(1) + + pytest_assert(wait_until(120, 10, 30, duthost.critical_services_fully_started), + "Not all critical services are fully started") + pytest_assert(wait_until(60, 10, 0, duthost.check_bgp_session_state, list(setup['neighhosts'].keys())), + "Not all BGP sessions are established on DUT") diff --git a/tests/bgp/test_bgp_session_flap.py b/tests/bgp/test_bgp_session_flap.py index e5a5274e12..f41fafc689 100644 --- a/tests/bgp/test_bgp_session_flap.py +++ b/tests/bgp/test_bgp_session_flap.py @@ -12,7 +12,6 @@ from tests.common.utilities import InterruptableThread import textfsm import traceback -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 from natsort import natsorted @@ -29,7 +28,7 @@ memSpike = 1.3 pytestmark = [ - pytest.mark.topology('t1') + pytest.mark.topology('t1', 't2') ] @@ -57,24 +56,27 @@ def get_cpu_stats(dut): @pytest.fixture(scope='module') -def setup(tbinfo, nbrhosts, duthosts, rand_one_dut_hostname, enum_rand_one_frontend_asic_index): - duthost = duthosts[rand_one_dut_hostname] +def setup(tbinfo, nbrhosts, duthosts, enum_frontend_dut_hostname, enum_rand_one_frontend_asic_index): + duthost = duthosts[enum_frontend_dut_hostname] asic_index = enum_rand_one_frontend_asic_index namespace = duthost.get_namespace_from_asic_id(asic_index) + bgp_facts = duthost.bgp_facts(instance_id=asic_index)['ansible_facts'] + neigh_keys = [] tor_neighbors = dict() - tor1 = natsorted(nbrhosts.keys())[0] - - skip_hosts = duthost.get_asic_namespace_list() - - bgp_facts = duthost.bgp_facts(instance_id=enum_rand_one_frontend_asic_index)['ansible_facts'] neigh_asn = dict() for k, v in bgp_facts['bgp_neighbors'].items(): - if v['description'].lower() not in skip_hosts: + if 'asic' not in v['description'].lower(): + neigh_keys.append(v['description']) neigh_asn[v['description']] = v['remote AS'] tor_neighbors[v['description']] = nbrhosts[v['description']]["host"] assert v['state'] == 'established' + if not neigh_keys: + pytest.skip("No BGP neighbors found on ASIC {} of DUT {}".format(asic_index, duthost.hostname)) + + tor1 = natsorted(neigh_keys)[0] + # verify sessions are established logger.info(duthost.shell('show ip bgp summary')) logger.info(duthost.shell('show ipv6 bgp summary')) diff --git a/tests/bgp/test_bgp_slb.py b/tests/bgp/test_bgp_slb.py index dd5171af2e..6e2a2f540e 100644 --- a/tests/bgp/test_bgp_slb.py +++ b/tests/bgp/test_bgp_slb.py @@ -6,7 +6,6 @@ toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m # noqa F401 from tests.common.utilities import wait_until, delete_running_config from tests.common.helpers.assertions import pytest_require -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ diff --git a/tests/bgp/test_bgp_speaker.py b/tests/bgp/test_bgp_speaker.py index 9cbacc3f05..6082f19fc9 100644 --- a/tests/bgp/test_bgp_speaker.py +++ b/tests/bgp/test_bgp_speaker.py @@ -15,9 +15,7 @@ from tests.common.utilities import wait_tcp_connection from tests.common.helpers.assertions import pytest_require from tests.common.utilities import wait_until -from tests.flow_counter.flow_counter_utils import RouteFlowCounterTestContext, \ - is_route_flow_counter_supported # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from tests.common.flow_counter.flow_counter_utils import RouteFlowCounterTestContext, is_route_flow_counter_supported # noqa F401 pytestmark = [ @@ -281,7 +279,7 @@ def bgp_speaker_announce_routes_common(common_setup_teardown, tbinfo, duthost, announce_route(ptfip, lo_addr, peer_range, vlan_ips[0].ip, port_num[2]) logger.info("Wait some time to make sure routes announced to dynamic bgp neighbors") - assert wait_until(90, 10, 0, is_all_neighbors_learned, duthost, speaker_ips), \ + assert wait_until(120, 10, 0, is_all_neighbors_learned, duthost, speaker_ips), \ "Not all dynamic neighbors were learned" logger.info("Verify nexthops and nexthop interfaces for accepted prefixes of the dynamic neighbors") diff --git a/tests/bgp/test_bgp_stress_link_flap.py b/tests/bgp/test_bgp_stress_link_flap.py new file mode 100644 index 0000000000..e45619f921 --- /dev/null +++ b/tests/bgp/test_bgp_stress_link_flap.py @@ -0,0 +1,394 @@ +import logging +import pytest +import time +import traceback +import threading +from tests.common.platform.device_utils import fanout_switch_port_lookup +from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import wait_until +from tests.common.utilities import InterruptableThread + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.disable_loganalyzer, + pytest.mark.topology('t0', 't1') +] + +stop_threads = False +SLEEP_DURATION = 0.005 +TEST_RUN_DURATION = 300 +MEMORY_EXHAUST_THRESHOLD = 300 +dut_flap_count = 0 +fanout_flap_count = 0 +neighbor_flap_count = 0 + +LOOP_TIMES_LEVEL_MAP = { + 'debug': 60, + 'basic': 300, + 'confident': 3600, + 'thorough': 21600 +} + + +@pytest.fixture(scope='module') +def setup(duthosts, rand_one_dut_hostname, nbrhosts, fanouthosts): + duthost = duthosts[rand_one_dut_hostname] + + config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] + bgp_neighbors = config_facts.get('BGP_NEIGHBOR', {}) + portchannels = config_facts.get('PORTCHANNEL_MEMBER', {}) + dev_nbrs = config_facts.get('DEVICE_NEIGHBOR', {}) + bgp_neighbor = list(bgp_neighbors.keys())[0] + + logger.debug("setup config_facts {}".format(config_facts)) + logger.debug("setup nbrhosts {}".format(nbrhosts)) + logger.debug("setup bgp_neighbors {}".format(bgp_neighbors)) + logger.debug("setup dev_nbrs {}".format(dev_nbrs)) + logger.debug("setup portchannels {}".format(portchannels)) + logger.debug("setup test_neighbor {}".format(bgp_neighbor)) + + interface_list = dev_nbrs.keys() + logger.debug('interface_list: {}'.format(interface_list)) + + # verify sessions are established + pytest_assert(wait_until(30, 5, 0, duthost.check_bgp_session_state, list(bgp_neighbors.keys())), + "Not all BGP sessions are established on DUT") + + ip_intfs = duthost.show_and_parse('show ip interface') + logger.debug("setup ip_intfs {}".format(ip_intfs)) + + # Create a mapping of neighbor IP to interfaces and their details + neighbor_ip_to_interfaces = {} + + # Loop through the ip_intfs list to populate the mapping + for ip_intf in ip_intfs: + neighbor_ip = ip_intf['neighbor ip'] + interface_name = ip_intf['interface'] + if neighbor_ip not in neighbor_ip_to_interfaces: + neighbor_ip_to_interfaces[neighbor_ip] = {} + + # Check if the interface is in portchannels and get the relevant devices + if interface_name in portchannels: + for dev_name in portchannels[interface_name]: + if dev_name in dev_nbrs and dev_nbrs[dev_name]['name'] == ip_intf['bgp neighbor']: + neighbor_ip_to_interfaces[neighbor_ip][dev_name] = dev_nbrs[dev_name] + # If not in portchannels, check directly in dev_nbrs + elif interface_name in dev_nbrs and dev_nbrs[interface_name]['name'] == ip_intf['bgp neighbor']: + neighbor_ip_to_interfaces[neighbor_ip][interface_name] = dev_nbrs[interface_name] + + # Update bgp_neighbors with the new 'interface' key + for ip, details in bgp_neighbors.items(): + if ip in neighbor_ip_to_interfaces: + details['interface'] = neighbor_ip_to_interfaces[ip] + + setup_info = { + 'neighhosts': bgp_neighbors, + "eth_nbrs": dev_nbrs + } + + logger.debug('Setup_info: {}'.format(setup_info)) + + yield setup_info + + # verify sessions are established after test + if not duthost.check_bgp_session_state(bgp_neighbors): + for port in interface_list: + logger.info("no shutdown dut interface {} port {}".format(duthost, port)) + duthost.no_shutdown(port) + + fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, duthost.hostname, port) + if fanout and fanout_port: + logger.info("no shutdown fanout interface, fanout {} port {}".format(fanout, fanout_port)) + fanout.no_shutdown(fanout_port) + + neighbor = dev_nbrs[port]["name"] + neighbor_port = dev_nbrs[port]["port"] + neighbor_host = nbrhosts.get(neighbor, {}).get('host', None) + if neighbor_host: + neighbor_host.no_shutdown(neighbor_port) + logger.info("no shutdown neighbor interface, neighbor {} port {}".format(neighbor, neighbor_port)) + else: + logger.debug("neighbor host not found for {} port {}".format(neighbor, neighbor_port)) + + time.sleep(1) + + pytest_assert(wait_until(600, 10, 0, duthost.check_bgp_session_state, list(bgp_neighbors.keys())), + "Not all BGP sessions are established on DUT") + + +def flap_dut_interface(duthost, port, stop_event, sleep_duration, test_run_duration): + logger.info("flap dut {} interface {} delay time {} timeout {}".format( + duthost, port, sleep_duration, test_run_duration)) + global dut_flap_count + dut_flap_count = 0 + + start_time = time.time() # Record the start time + while not stop_event.is_set() and time.time() - start_time < test_run_duration: + duthost.shutdown(port) + time.sleep(sleep_duration) + duthost.no_shutdown(port) + time.sleep(sleep_duration) + if stop_threads: + logger.info("Stop flap thread, breaking dut flap dut {} interface {} flap count {}".format( + duthost, port, dut_flap_count)) + break + dut_flap_count += 1 + + +def flap_fanout_interface_all(interface_list, fanouthosts, duthost, stop_event, sleep_duration, test_run_duration): + global fanout_flap_count + fanout_flap_count = 0 + fanout_interfaces = {} + + for port in interface_list: + fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, duthost.hostname, port) + if fanout and fanout_port: + if fanout not in fanout_interfaces: + fanout_interfaces[fanout] = [] + fanout_interfaces[fanout].append(fanout_port) + + logger.info("flap interface fanout port {}".format(fanout_interfaces)) + + start_time = time.time() # Record the start time + while not stop_event.is_set() and time.time() - start_time < test_run_duration: + for fanout_host, fanout_ports in fanout_interfaces.items(): + logger.info("flap interface fanout {} port {}".format(fanout_host, fanout_port)) + + fanout_host.shutdown_multiple(fanout_ports) + time.sleep(sleep_duration) + fanout_host.no_shutdown_multiple(fanout_ports) + time.sleep(sleep_duration) + + fanout_flap_count += 1 + if stop_threads: + logger.info("Stop flap thread, breaking flap fanout {} dut {} flap count {}".format( + fanouthosts, duthost, fanout_flap_count)) + break + + +def flap_fanout_interface(interface_list, fanouthosts, duthost, stop_event, sleep_duration, test_run_duration): + global fanout_flap_count + fanout_flap_count = 0 + + start_time = time.time() # Record the start time + while not stop_event.is_set() and time.time() - start_time < test_run_duration: + for port in interface_list: + if stop_threads: + break + + fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, duthost.hostname, port) + if fanout and fanout_port: + logger.info("flap interface fanout {} port {}".format(fanout, fanout_port)) + fanout.shutdown(fanout_port) + time.sleep(sleep_duration) + fanout.no_shutdown(fanout_port) + time.sleep(sleep_duration) + else: + logger.warning("fanout not found for {} port {}".format(duthost.hostname, port)) + + fanout_flap_count += 1 + if stop_threads: + logger.info("Stop flap thread, breaking flap fanout {} dut {} interface {} flap count {}".format( + fanouthosts, duthost, port, fanout_flap_count)) + break + + +def flap_neighbor_interface(neighbor, neighbor_port, stop_event, sleep_duration, test_run_duration): + logger.info("flap neighbor {} interface {}".format(neighbor, neighbor_port)) + global neighbor_flap_count + neighbor_flap_count = 0 + + start_time = time.time() # Record the start time + while not stop_event.is_set() and time.time() - start_time < test_run_duration: + neighbor.shutdown(neighbor_port) + time.sleep(sleep_duration) + neighbor.no_shutdown(neighbor_port) + time.sleep(sleep_duration) + if stop_threads: + logger.info("Stop flap thread, breaking flap neighbor {} interface {} flap count {}".format( + neighbor, neighbor_port, neighbor_flap_count)) + break + neighbor_flap_count += 1 + + +@pytest.mark.parametrize("test_type", ["dut", "fanout", "neighbor", "all"]) +def test_bgp_stress_link_flap(duthosts, rand_one_dut_hostname, setup, nbrhosts, fanouthosts, test_type, + get_function_completeness_level): + global stop_threads + global dut_flap_count + global fanout_flap_count + global neighbor_flap_count + global TEST_RUN_DURATION + + duthost = duthosts[rand_one_dut_hostname] + + normalized_level = get_function_completeness_level + if normalized_level is None: + normalized_level = 'debug' + TEST_RUN_DURATION = LOOP_TIMES_LEVEL_MAP[normalized_level] + logger.debug('normalized_level {}, set test run duration {}'.format(normalized_level, TEST_RUN_DURATION)) + + # Skip the test on Virtual Switch due to fanout switch dependency and warm reboot + asic_type = duthost.facts['asic_type'] + if asic_type == "vs" and (test_type == "fanout" or test_type == "all"): + pytest.skip("Stress link flap test is not supported on Virtual Switch") + + if asic_type != "vs": + delay_time = SLEEP_DURATION + else: + delay_time = SLEEP_DURATION * 100 + + eth_nbrs = setup.get('eth_nbrs', {}) + interface_list = eth_nbrs.keys() + logger.debug('interface_list: {}'.format(interface_list)) + + stop_threads = False + dut_flap_count = 0 + fanout_flap_count = 0 + neighbor_flap_count = 0 + # Create a stop event + stop_event = threading.Event() + + flap_threads = [] + + if test_type == "dut": + for interface in interface_list: + thread = InterruptableThread( + target=flap_dut_interface, + args=(duthost, interface, stop_event, delay_time, TEST_RUN_DURATION,) + ) + thread.daemon = True + thread.start() + logger.info("Start flap thread {} dut {} interface {}".format(thread, duthost, interface)) + flap_threads.append(thread) + # create only one thread for vs due to memory resource limitation + if asic_type == "vs": + break + elif test_type == "fanout": + thread = InterruptableThread( + target=flap_fanout_interface, + args=(interface_list, fanouthosts, duthost, stop_event, delay_time, TEST_RUN_DURATION,) + ) + thread.daemon = True + thread.start() + logger.info("Start flap thread {} fanout {} dut {}".format(thread, fanouthosts, duthost)) + flap_threads.append(thread) + elif test_type == "neighbor": + for interface in interface_list: + neighbor_name = eth_nbrs[interface]["name"] + neighbor_port = eth_nbrs[interface]["port"] + neighbor_host = nbrhosts.get(neighbor_name, {}).get('host', None) + if neighbor_host: + thread = InterruptableThread( + target=flap_neighbor_interface, + args=(neighbor_host, neighbor_port, stop_event, delay_time, TEST_RUN_DURATION,) + ) + thread.daemon = True + thread.start() + logger.info("Start flap thread {} neighbor {} port {}".format(thread, neighbor_host, neighbor_port)) + flap_threads.append(thread) + else: + logger.debug("neighbor host not found for {} port {}".format(neighbor_name, neighbor_port)) + # create only one thread for vs due to memory resource limitation + if asic_type == "vs": + break + elif test_type == "all": + for interface in interface_list: + logger.info("shutdown all interface {} ".format(interface)) + thread_dut = InterruptableThread( + target=flap_dut_interface, + args=(duthost, interface, stop_event, delay_time, TEST_RUN_DURATION,) + ) + thread_dut.daemon = True + thread_dut.start() + logger.info("Start flap thread {} dut {} interface {}".format(thread_dut, duthost, interface)) + flap_threads.append(thread_dut) + + neighbor_name = eth_nbrs[interface]["name"] + neighbor_port = eth_nbrs[interface]["port"] + neighbor_host = nbrhosts.get(neighbor_name, {}).get('host', None) + if neighbor_host: + thread_neighbor = InterruptableThread( + target=flap_neighbor_interface, + args=(neighbor_host, neighbor_port, stop_event, delay_time, TEST_RUN_DURATION,) + ) + thread_neighbor.daemon = True + thread_neighbor.start() + logger.info("Start flap thread {} neighbor {} port {}".format( + thread_neighbor, neighbor_host, neighbor_port)) + flap_threads.append(thread_neighbor) + else: + logger.debug("neighbor host not found for {} port {}".format(neighbor_name, neighbor_port)) + + thread_fanout = InterruptableThread( + target=flap_fanout_interface, + args=(interface_list, fanouthosts, duthost, stop_event, delay_time, TEST_RUN_DURATION,) + ) + thread_fanout.daemon = True + thread_fanout.start() + logger.info("Start flap thread {} fanout {} dut {} ".format( + thread_fanout, fanouthosts, duthost)) + flap_threads.append(thread_fanout) + + logger.info("flap_threads {} ".format(flap_threads)) + + avail_mem_list = [] + end_time = time.time() + TEST_RUN_DURATION + while time.time() < end_time: + time.sleep(30) + + cmd = "free -m" + cmd_out = duthost.shell(cmd, module_ignore_errors=True).get('stdout', None) + logger.info("cmd {} response: {}".format(cmd, cmd_out)) + lines = cmd_out.split('\n') + for line in lines: + if line.startswith("Mem:"): + fields = line.split() + total_mem, avail_mem = int(fields[1]), int(fields[-1]) + logger.info("Total memory {} Available memory: {}".format(total_mem, avail_mem)) + avail_mem_list.append(avail_mem) + break + + if avail_mem < MEMORY_EXHAUST_THRESHOLD: + logger.error("Available memory {} is less than {}, stopping the test".format( + avail_mem, MEMORY_EXHAUST_THRESHOLD)) + + cmd = "top -b -n 1" + cmd_out = duthost.shell(cmd, module_ignore_errors=True).get('stdout', None) + logger.info("cmd {} response: {}".format(cmd, cmd_out)) + + cmd = "sudo monit status" + cmd_out = duthost.shell(cmd, module_ignore_errors=True).get('stdout', None) + logger.info("cmd {} response: {}".format(cmd, cmd_out)) + + cmd = "docker stats --no-stream" + cmd_out = duthost.shell(cmd, module_ignore_errors=True).get('stdout', None) + logger.info("cmd {} response: {}".format(cmd, cmd_out)) + + break + + logger.info("Test running for {} seconds".format(time.time() + TEST_RUN_DURATION - end_time)) + logger.info("Test run duration dut_flap_count {} fanout_flap_count {} neighbor_flap_count {}".format( + dut_flap_count, fanout_flap_count, neighbor_flap_count)) + stop_event.set() + stop_threads = True + logger.info("stop_threads {} ".format(flap_threads)) + time.sleep(30) + + for thread in flap_threads: + logger.info("waiting thread {} done".format(thread)) + try: + if thread.is_alive(): + thread.join(timeout=30) + logger.info("thread {} joined".format(thread)) + except Exception as e: + logger.debug("Exception occurred in thread %r:", thread) + logger.debug("".join(traceback.format_exception(None, e, e.__traceback__))) + + # Clean up the thread list after joining all threads + logger.info("clear threads {} ".format(flap_threads)) + flap_threads.clear() + logger.debug("avail_mem history {} ".format(avail_mem_list)) + + return diff --git a/tests/bgp/test_bgp_suppress_fib.py b/tests/bgp/test_bgp_suppress_fib.py index 636bf3b408..ea91f4ea46 100644 --- a/tests/bgp/test_bgp_suppress_fib.py +++ b/tests/bgp/test_bgp_suppress_fib.py @@ -24,7 +24,6 @@ from bgp_helpers import restart_bgp_session, get_eth_port, get_exabgp_port, get_vm_name_list, get_bgp_neighbor_ip, \ check_route_install_status, validate_route_propagate_status, operate_orchagent, get_t2_ptf_intfs, \ get_eth_name_from_ptf_port, check_bgp_neighbor, check_fib_route -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology("t1"), @@ -316,7 +315,7 @@ def setup_vrf_cfg(duthost, cfg_facts, nbrhosts, tbinfo): duthost.template(src="bgp/vrf_config_db.j2", dest="/tmp/config_db_vrf.json") duthost.shell("cp -f /tmp/config_db_vrf.json /etc/sonic/config_db.json") - config_reload(duthost) + config_reload(duthost, safe_reload=True) def setup_vrf(duthost, nbrhosts, tbinfo): @@ -798,7 +797,7 @@ def test_bgp_route_with_suppress(duthost, tbinfo, nbrhosts, ptfadapter, localhos if vrf_type == USER_DEFINED_VRF: with allure.step("Clean user defined vrf"): duthost.shell("cp -f /etc/sonic/config_db.json.bak /etc/sonic/config_db.json") - config_reload(duthost) + config_reload(duthost, safe_reload=True) def test_bgp_route_without_suppress(duthost, tbinfo, nbrhosts, ptfadapter, prepare_param, restore_bgp_suppress_fib, diff --git a/tests/bgp/test_bgp_update_timer.py b/tests/bgp/test_bgp_update_timer.py index 5fd653437a..e6472165c7 100644 --- a/tests/bgp/test_bgp_update_timer.py +++ b/tests/bgp/test_bgp_update_timer.py @@ -22,7 +22,6 @@ from tests.common.dualtor.mux_simulator_control import mux_server_url # noqa F401 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m # noqa F401 from tests.common.helpers.constants import DEFAULT_NAMESPACE -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ diff --git a/tests/bgp/test_bgpmon.py b/tests/bgp/test_bgpmon.py index b1a374a82d..a2b413252e 100644 --- a/tests/bgp/test_bgpmon.py +++ b/tests/bgp/test_bgpmon.py @@ -13,7 +13,6 @@ from tests.common.utilities import wait_until from tests.common.utilities import wait_tcp_connection from bgp_helpers import BGPMON_TEMPLATE_FILE, BGPMON_CONFIG_FILE, BGP_MONITOR_NAME, BGP_MONITOR_PORT -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('any'), @@ -152,7 +151,7 @@ def bgpmon_peer_connected(duthost, bgpmon_peer): peer_asn=asn, port=BGP_MONITOR_PORT, passive=True) ptfhost.shell("ip neigh add %s lladdr %s dev %s" % (local_addr, duthost.facts["router_mac"], ptf_interface)) - ptfhost.shell("ip route add %s dev %s" % (local_addr + "/32", ptf_interface)) + ptfhost.shell("ip route replace %s dev %s" % (local_addr + "/32", ptf_interface)) try: pytest_assert(wait_tcp_connection(localhost, ptfhost.mgmt_ip, BGP_MONITOR_PORT, timeout_s=60), "Failed to start bgp monitor session on PTF") diff --git a/tests/bgp/test_bgpmon_v6.py b/tests/bgp/test_bgpmon_v6.py index ee95fa866f..4a5eb14021 100644 --- a/tests/bgp/test_bgpmon_v6.py +++ b/tests/bgp/test_bgpmon_v6.py @@ -14,7 +14,6 @@ from tests.common.utilities import wait_until from tests.common.utilities import wait_tcp_connection from bgp_helpers import BGPMON_TEMPLATE_FILE, BGPMON_CONFIG_FILE, BGP_MONITOR_NAME, BGP_MONITOR_PORT -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('t2'), diff --git a/tests/bgp/test_ipv6_nlri_over_ipv4.py b/tests/bgp/test_ipv6_nlri_over_ipv4.py index 612d435d84..479edf2226 100644 --- a/tests/bgp/test_ipv6_nlri_over_ipv4.py +++ b/tests/bgp/test_ipv6_nlri_over_ipv4.py @@ -11,7 +11,6 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.constants import DEFAULT_NAMESPACE from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/bgp/test_passive_peering.py b/tests/bgp/test_passive_peering.py index 9428a40ad3..56630cc1df 100644 --- a/tests/bgp/test_passive_peering.py +++ b/tests/bgp/test_passive_peering.py @@ -9,7 +9,6 @@ import time from tests.common.config_reload import config_reload from tests.common.helpers.constants import DEFAULT_NAMESPACE -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) @@ -23,12 +22,12 @@ @pytest.fixture(scope='module') -def setup(tbinfo, nbrhosts, duthosts, rand_one_dut_hostname, request): +def setup(tbinfo, nbrhosts, duthosts, rand_one_dut_front_end_hostname, request): # verify neighbors are type sonic if request.config.getoption("neighbor_type") != "sonic": pytest.skip("Neighbor type must be sonic") - duthost = duthosts[rand_one_dut_hostname] + duthost = duthosts[rand_one_dut_front_end_hostname] dut_asn = tbinfo['topo']['properties']['configuration_properties']['common']['dut_asn'] lldp_table = duthost.shell("show lldp table")['stdout'].split("\n")[3].split() diff --git a/tests/bgp/test_seq_idf_isolation.py b/tests/bgp/test_seq_idf_isolation.py index f15ca673ee..bdaf5dac62 100644 --- a/tests/bgp/test_seq_idf_isolation.py +++ b/tests/bgp/test_seq_idf_isolation.py @@ -5,8 +5,8 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.constants import DEFAULT_ASIC_ID from tests.common.utilities import wait_until -from test_traffic_shift import verify_only_loopback_routes_are_announced_to_neighs, parse_routes_on_neighbors -from test_traffic_shift import verify_current_routes_announced_to_neighs, check_and_log_routes_diff +from route_checker import verify_only_loopback_routes_are_announced_to_neighs, parse_routes_on_neighbors +from route_checker import verify_current_routes_announced_to_neighs, check_and_log_routes_diff pytestmark = [ pytest.mark.topology('t2') diff --git a/tests/bgp/test_startup_tsa_tsb_service.py b/tests/bgp/test_startup_tsa_tsb_service.py index a4230d801a..d52e45a5f3 100644 --- a/tests/bgp/test_startup_tsa_tsb_service.py +++ b/tests/bgp/test_startup_tsa_tsb_service.py @@ -1,17 +1,18 @@ import logging import datetime - +import pexpect import pytest - from tests.common import reboot, config_reload from tests.common.reboot import get_reboot_cause, SONIC_SSH_PORT, SONIC_SSH_REGEX, wait_for_startup from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until from tests.common.platform.processes_utils import wait_critical_processes from tests.common.platform.interface_utils import check_interface_status_of_up_ports -from tests.bgp.test_traffic_shift import get_traffic_shift_state, parse_routes_on_neighbors,\ - check_tsa_persistence_support, verify_current_routes_announced_to_neighs, check_and_log_routes_diff, \ - verify_only_loopback_routes_are_announced_to_neighs +from traffic_checker import get_traffic_shift_state, check_tsa_persistence_support +from route_checker import parse_routes_on_neighbors, check_and_log_routes_diff, \ + verify_current_routes_announced_to_neighs, verify_only_loopback_routes_are_announced_to_neighs +from tests.bgp.constants import TS_NORMAL, TS_MAINTENANCE + pytestmark = [ pytest.mark.topology('t2') @@ -19,10 +20,7 @@ logger = logging.getLogger(__name__) -TS_NORMAL = "System Mode: Normal" -TS_MAINTENANCE = "System Mode: Maintenance" -TS_INCONSISTENT = "System Mode: Not consistent" -TS_NO_NEIGHBORS = "System Mode: No external neighbors" + COLD_REBOOT_CAUSE = 'cold' UNKNOWN_REBOOT_CAUSE = "Unknown" SUP_REBOOT_CAUSE = 'Reboot from Supervisor' @@ -168,6 +166,37 @@ def check_ssh_state(localhost, dut_ip, expected_state, timeout=60): return not res.is_failed and 'Timeout' not in res.get('msg', '') +def exec_tsa_tsb_cmd_on_supervisor(duthosts, enum_supervisor_dut_hostname, creds, tsa_tsb_cmd): + """ + @summary: Issue TSA/TSB command on supervisor card using user credentials + Verify command is executed on supervisor card + @returns: None + """ + try: + suphost = duthosts[enum_supervisor_dut_hostname] + sup_ip = suphost.mgmt_ip + sonic_username = creds['sonicadmin_user'] + sonic_password = creds['sonicadmin_password'] + logger.info('sonic-username: {}, sonic_password: {}'.format(sonic_username, sonic_password)) + ssh_cmd = "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no {}@{}".format(sonic_username, sup_ip) + connect = pexpect.spawn(ssh_cmd) + connect.expect('.*[Pp]assword:') + connect.sendline(sonic_password) + i = connect.expect('{}@{}:'.format(sonic_username, suphost.hostname), timeout=10) + pytest_assert(i == 0, "Failed to connect") + connect.sendline(tsa_tsb_cmd) + connect.expect('.*[Pp]assword for username \'{}\':'.format(sonic_username)) + connect.sendline(sonic_password) + j = connect.expect('{}@{}:'.format(sonic_username, suphost.hostname), timeout=10) + pytest_assert(j == 0, "Failed to connect") + except pexpect.exceptions.EOF: + pytest.fail("EOF reached") + except pexpect.exceptions.TIMEOUT: + pytest.fail("Timeout reached") + except Exception as e: + pytest.fail("Cannot connect to DUT {} host via SSH: {}".format(suphost.hostname, e)) + + @pytest.mark.disable_loganalyzer def test_tsa_tsb_service_with_dut_cold_reboot(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, nbrhosts, traffic_shift_community, tbinfo): @@ -705,3 +734,424 @@ def test_tsa_tsb_service_with_user_init_tsa(duthosts, localhost, enum_rand_one_p reboot_cause = get_reboot_cause(duthost) pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) + + +@pytest.mark.disable_loganalyzer +def test_user_init_tsa_while_service_run_on_dut(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, + nbrhosts, traffic_shift_community, tbinfo): + + """ + Test startup TSA_TSB service after DUT cold reboot + Verify startup_tsa_tsb.service started automatically when dut comes up + Verify this service configures TSA and starts a timer + Issue TSA while the service is running on dut, and make sure the TSA is configured + Make sure TSA_TSB service is stopped and dut continues to be in maintenance mode + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + tsa_tsb_timer = get_startup_tsb_timer(duthost) + if not tsa_tsb_timer: + pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) + dut_nbrhosts = nbrhosts_to_dut(duthost, nbrhosts) + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state") + if not check_tsa_persistence_support(duthost): + pytest.skip("TSA persistence not supported in the image") + + try: + # Get all routes on neighbors before doing reboot + orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) + orig_v6_routes = parse_routes_on_neighbors(duthost, nbrhosts, 6) + + # Reboot dut and wait for startup_tsa_tsb service to start + logger.info("Cold reboot on node: %s", duthost.hostname) + reboot(duthost, localhost, wait=240) + + logger.info('Cold reboot finished on {}'.format(duthost.hostname)) + dut_uptime = duthost.get_up_time() + logger.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + + # Ensure startup_tsa_tsb service is running after dut reboot + pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'running'), + "startup_tsa_tsb service is not started after reboot") + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = duthost.get_up_time() + logging.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(duthost) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + + # Issue TSA on DUT + duthost.shell("TSA") + duthost.shell('sudo config save -y') + + # Ensure startup_tsa_tsb service is in inactive state after user-initiated TSA + pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'inactive'), + "startup_tsa_tsb service is not in inactive state after user init TSA") + + # Verify DUT continues to be in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), + "DUT is not in maintenance state with saved TSA config after reboot") + + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(duthost) + pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, duthost), + "Not all ports that are admin up on are operationally up") + + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, duthost, dut_nbrhosts, traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + finally: + """ + Test TSB after config save and config reload + Verify all routes are announced back to neighbors + """ + # Recover to Normal state + duthost.shell("TSB") + duthost.shell('sudo config save -y') + + # Verify DUT comes back to normal state after TSB. + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), "DUT is not in normal state") + # Wait until all routes are announced to neighbors + cur_v4_routes = {} + cur_v6_routes = {} + # Verify that all routes advertised to neighbor at the start of the test + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v4_routes, cur_v4_routes, 4): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v4_routes, cur_v4_routes, 4): + pytest.fail("Not all ipv4 routes are announced to neighbors") + + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v6_routes, cur_v6_routes, 6): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): + pytest.fail("Not all ipv6 routes are announced to neighbors") + + # Make sure the dut's reboot cause is as expected + logger.info("Check reboot cause of the dut") + reboot_cause = get_reboot_cause(duthost) + pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, + "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) + + +@pytest.mark.disable_loganalyzer +def test_user_init_tsb_while_service_run_on_dut(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, + nbrhosts, traffic_shift_community, tbinfo): + + """ + Test startup TSA_TSB service after DUT cold reboot + Verify startup_tsa_tsb.service started automatically when dut comes up + Verify this service configures TSA and starts a timer + Issue TSB while the service is running on dut, and make sure the TSB is configured + Make sure TSA_TSB service is stopped and dut continues to be in normal mode + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + tsa_tsb_timer = get_startup_tsb_timer(duthost) + if not tsa_tsb_timer: + pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state") + if not check_tsa_persistence_support(duthost): + pytest.skip("TSA persistence not supported in the image") + + try: + # Get all routes on neighbors before doing reboot + orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) + orig_v6_routes = parse_routes_on_neighbors(duthost, nbrhosts, 6) + + # Reboot dut and wait for startup_tsa_tsb service to start + logger.info("Cold reboot on node: %s", duthost.hostname) + reboot(duthost, localhost, wait=240) + + logger.info('Cold reboot finished on {}'.format(duthost.hostname)) + dut_uptime = duthost.get_up_time() + logger.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + + # Ensure startup_tsa_tsb service is running after dut reboot + pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'running'), + "startup_tsa_tsb service is not started after reboot") + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = duthost.get_up_time() + logging.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(duthost) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + + # Issue TSB on DUT + duthost.shell("TSB") + duthost.shell('sudo config save -y') + + # Verify DUT comes back to normal state after TSB. + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), "DUT is not in normal state") + + # Ensure startup_tsa_tsb service is in inactive state after user-initiated TSB + pytest_assert(wait_until(60, 5, 10, get_tsa_tsb_service_status, duthost, 'inactive'), + "startup_tsa_tsb service is not in inactive state after user init TSB") + + # Make sure DUT continues to be in good state after TSB + assert wait_until(300, 20, 2, duthost.critical_services_fully_started), \ + "Not all critical services are fully started on {}".format(duthost.hostname) + + # Wait until all routes are announced to neighbors + cur_v4_routes = {} + cur_v6_routes = {} + # Verify that all routes advertised to neighbor at the start of the test + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v4_routes, cur_v4_routes, 4): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v4_routes, cur_v4_routes, 4): + pytest.fail("Not all ipv4 routes are announced to neighbors") + + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v6_routes, cur_v6_routes, 6): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): + pytest.fail("Not all ipv6 routes are announced to neighbors") + + finally: + + # Verify DUT is in normal state. + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state") + + # Make sure the dut's reboot cause is as expected + logger.info("Check reboot cause of the dut") + reboot_cause = get_reboot_cause(duthost) + pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, + "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) + + +@pytest.mark.disable_loganalyzer +def test_user_init_tsb_on_sup_while_service_run_on_dut(duthosts, localhost, + enum_supervisor_dut_hostname, ptfhost, nbrhosts, + traffic_shift_community, creds, tbinfo): + """ + Test startup TSA_TSB service after DUT cold reboot + Verify startup_tsa_tsb.service started automatically when dut comes up + Verify this service configures TSA and starts a timer + Issue TSB from supervisor, while the service is running on dut, and make sure the TSB is configured on linecards + Make sure TSA_TSB service is stopped and dut changes from maintenance mode to normal mode + """ + tsa_tsb_cmd = 'sudo TSB' + suphost = duthosts[enum_supervisor_dut_hostname] + tsa_tsb_timer = dict() + dut_nbrhosts = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() + for linecard in duthosts.frontend_nodes: + tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) + if not tsa_tsb_timer[linecard]: + pytest.skip("startup_tsa_tsb.service is not supported on the duts under {}".format(suphost.hostname)) + dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), + "DUT is not in normal state") + if not check_tsa_persistence_support(linecard): + pytest.skip("TSA persistence not supported in the image") + + try: + for linecard in duthosts.frontend_nodes: + # Get all routes on neighbors before doing reboot + orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) + orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) + + # Get a dut uptime before reboot + sup_uptime_before = suphost.get_up_time() + # Reboot dut and wait for startup_tsa_tsb service to start on linecards + logger.info("Cold reboot on supervisor node: %s", suphost.hostname) + reboot(suphost, localhost, wait=240) + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(suphost) + + sup_uptime = suphost.get_up_time() + logger.info('DUT {} up since {}'.format(suphost.hostname, sup_uptime)) + rebooted = float(sup_uptime_before.strftime("%s")) != float(sup_uptime.strftime("%s")) + assert rebooted, "Device {} did not reboot".format(suphost.hostname) + + for linecard in duthosts.frontend_nodes: + wait_for_startup(linecard, localhost, delay=10, timeout=300) + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = linecard.get_up_time() + logging.info('DUT {} up since {}'.format(linecard.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(linecard) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(linecard) + + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") + + # Execute user initiated TSB from supervisor card + exec_tsa_tsb_cmd_on_supervisor(duthosts, enum_supervisor_dut_hostname, creds, tsa_tsb_cmd) + + for linecard in duthosts.frontend_nodes: + # Ensure dut comes back to normal state + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), + "DUT is not in normal state after TSB command from supervisor") + + # Ensure startup_tsa_tsb service is in inactive state after user-initiated TSB on supervisor + pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, linecard, 'inactive'), + "startup_tsa_tsb service is not in inactive state after user init TSB from supervisor") + + pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, linecard), + "Not all ports that are admin up on are operationally up") + + # Wait until all routes are announced to neighbors + cur_v4_routes = {} + cur_v6_routes = {} + # Verify that all routes advertised to neighbor at the start of the test + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, dut_nbrhosts[linecard], + orig_v4_routes[linecard], cur_v4_routes, 4): + if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], + orig_v4_routes[linecard], cur_v4_routes, 4): + pytest.fail("Not all ipv4 routes are announced to neighbors") + + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, dut_nbrhosts[linecard], + orig_v6_routes[linecard], cur_v6_routes, 6): + if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], + orig_v6_routes[linecard], cur_v6_routes, 6): + pytest.fail("Not all ipv6 routes are announced to neighbors") + + finally: + for linecard in duthosts.frontend_nodes: + # Make sure linecards are in Normal state and save the config to proceed further + linecard.shell("TSB") + linecard.shell('sudo config save -y') + # Verify DUT is in normal state. + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), + "DUT {} is not in normal state".format(linecard)) + # Make sure the dut's reboot cause is as expected + logger.info("Check reboot cause of the dut {}".format(linecard)) + reboot_cause = get_reboot_cause(linecard) + pytest_assert(reboot_cause == SUP_REBOOT_CAUSE, + "Reboot cause {} did not match the trigger {}".format(reboot_cause, SUP_REBOOT_CAUSE)) + + # Make sure the Supervisor's reboot cause is as expected + logger.info("Check reboot cause of the supervisor") + reboot_cause = get_reboot_cause(suphost) + pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, + "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) + + +@pytest.mark.disable_loganalyzer +def test_tsa_tsb_timer_efficiency(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, + nbrhosts, traffic_shift_community, tbinfo): + """ + Test startup TSA_TSB service after DUT cold reboot + Verify the configured tsa_tsb_timer is sufficient for system to be stable + Verify this service configures TSA and starts a timer and configures TSB once the timer is expired + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + tsa_tsb_timer = get_startup_tsb_timer(duthost) + if not tsa_tsb_timer: + pytest.skip("startup_tsa_tsb.service is not supported on the {}".format(duthost.hostname)) + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state") + if not check_tsa_persistence_support(duthost): + pytest.skip("TSA persistence not supported in the image") + + try: + # Get all routes on neighbors before doing reboot + orig_v4_routes = parse_routes_on_neighbors(duthost, nbrhosts, 4) + orig_v6_routes = parse_routes_on_neighbors(duthost, nbrhosts, 6) + + up_bgp_neighbors = duthost.get_bgp_neighbors_per_asic("established") + + # Reboot dut and wait for startup_tsa_tsb service to start + logger.info("Cold reboot on node: %s", duthost.hostname) + reboot(duthost, localhost, wait=240) + + logger.info('Cold reboot finished on {}'.format(duthost.hostname)) + dut_uptime = duthost.get_up_time() + logger.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + + # Ensure startup_tsa_tsb service is running after dut reboot + pytest_assert(wait_until(60, 5, 0, get_tsa_tsb_service_status, duthost, 'running'), + "startup_tsa_tsb service is not started after reboot") + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = duthost.get_up_time() + logging.info('DUT {} up since {}'.format(duthost.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(duthost) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 120, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + + logging.info("Wait until all critical services are fully started") + pytest_assert(wait_until(300, 20, 2, duthost.critical_services_fully_started)), \ + "Not all critical services are fully started on {}".format(duthost.hostname) + + logging.info("Wait until all critical processes are fully started") + wait_critical_processes(duthost) + pytest_assert(wait_until(600, 20, 0, check_interface_status_of_up_ports, duthost), + "Not all ports that are admin up on are operationally up") + + pytest_assert(wait_until(300, 10, 0, + duthost.check_bgp_session_state_all_asics, up_bgp_neighbors, "established")) + + stability_check_time = datetime.datetime.now() + time_to_stabilize = (stability_check_time - service_uptime).total_seconds() + logging.info("Time taken for system stability : {}".format(time_to_stabilize)) + + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(duthost), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + + # Verify startup_tsa_tsb service stopped after expected time + pytest_assert(wait_until(tsa_tsb_timer, 20, 0, get_tsa_tsb_service_status, duthost, 'exited'), + "startup_tsa_tsb service is not stopped even after configured timer expiry") + + # Verify tsa_tsb_timer configured is sufficient + pytest_assert(time_to_stabilize < tsa_tsb_timer, + "Configured tsa_tsb_timer is not sufficient for the system to be stable") + + # Ensure dut comes back to normal state after timer expiry + if not get_tsa_tsb_service_status(duthost, 'running'): + # Verify TSB is configured on the dut after startup_tsa_tsb service is stopped + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state after startup_tsa_tsb service is stopped") + + # Wait until all routes are announced to neighbors + cur_v4_routes = {} + cur_v6_routes = {} + # Verify that all routes advertised to neighbor at the start of the test + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v4_routes, cur_v4_routes, 4): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v4_routes, cur_v4_routes, 4): + pytest.fail("Not all ipv4 routes are announced to neighbors") + + if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, duthost, nbrhosts, + orig_v6_routes, cur_v6_routes, 6): + if not check_and_log_routes_diff(duthost, nbrhosts, orig_v6_routes, cur_v6_routes, 6): + pytest.fail("Not all ipv6 routes are announced to neighbors") + + finally: + + # Verify DUT is in normal state. + pytest_assert(TS_NORMAL == get_traffic_shift_state(duthost), + "DUT is not in normal state") + # Make sure the dut's reboot cause is as expected + logger.info("Check reboot cause of the dut") + reboot_cause = get_reboot_cause(duthost) + pytest_assert(reboot_cause == COLD_REBOOT_CAUSE, + "Reboot cause {} did not match the trigger {}".format(reboot_cause, COLD_REBOOT_CAUSE)) diff --git a/tests/bgp/test_traffic_shift.py b/tests/bgp/test_traffic_shift.py index 98c4947741..e7f8bf6206 100644 --- a/tests/bgp/test_traffic_shift.py +++ b/tests/bgp/test_traffic_shift.py @@ -1,19 +1,17 @@ import logging import re -from tests.common.devices.eos import EosHost -import json - -import ipaddr as ipaddress import pytest - -from bgp_helpers import parse_rib, get_routes_not_announced_to_bgpmon, remove_bgp_neighbors, restore_bgp_neighbors +from tests.common.devices.eos import EosHost +from bgp_helpers import get_routes_not_announced_to_bgpmon, remove_bgp_neighbors, restore_bgp_neighbors from tests.common import config_reload from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.constants import DEFAULT_ASIC_ID -from tests.common.helpers.parallel import parallel_run from tests.common.platform.processes_utils import wait_critical_processes from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from route_checker import verify_only_loopback_routes_are_announced_to_neighs, parse_routes_on_neighbors, \ + verify_current_routes_announced_to_neighs, check_and_log_routes_diff +from traffic_checker import get_traffic_shift_state, check_tsa_persistence_support, verify_traffic_shift_per_asic +from tests.bgp.constants import TS_NORMAL, TS_MAINTENANCE, TS_NO_NEIGHBORS pytestmark = [ pytest.mark.topology('t1', 't2') @@ -21,18 +19,6 @@ logger = logging.getLogger(__name__) -TS_NORMAL = "System Mode: Normal" -TS_MAINTENANCE = "System Mode: Maintenance" -TS_INCONSISTENT = "System Mode: Not consistent" -TS_NO_NEIGHBORS = "System Mode: No external neighbors" - - -@pytest.fixture -def traffic_shift_community(duthost): - community = duthost.shell('sonic-cfggen -y /etc/sonic/constants.yml -v constants.bgp.traffic_shift_community')[ - 'stdout'] - return community - @pytest.fixture(scope="module") def nbrhosts_to_dut(duthosts, enum_rand_one_per_hwsku_frontend_hostname, nbrhosts): @@ -46,185 +32,6 @@ def nbrhosts_to_dut(duthosts, enum_rand_one_per_hwsku_frontend_hostname, nbrhost return nbrhosts_to_dut -def verify_traffic_shift_per_asic(host, outputs, match_result, asic_index): - prefix = "BGP{} : ".format( - asic_index) if asic_index != DEFAULT_ASIC_ID else '' - result_str = "{}{}".format(prefix, match_result) - if result_str in outputs: - return True - else: - return False - - -def verify_traffic_shift(host, outputs, match_result): - for asic_index in host.get_frontend_asic_ids(): - if verify_traffic_shift_per_asic(host, outputs, TS_NO_NEIGHBORS, asic_index): - continue - if not verify_traffic_shift_per_asic(host, outputs, match_result, asic_index): - return "ERROR" - - return match_result - - -def get_traffic_shift_state(host, cmd="TSC"): - outputs = host.shell(cmd)['stdout_lines'] - if verify_traffic_shift(host, outputs, TS_NORMAL) != "ERROR": - return TS_NORMAL - if verify_traffic_shift(host, outputs, TS_MAINTENANCE) != "ERROR": - return TS_MAINTENANCE - if verify_traffic_shift(host, outputs, TS_INCONSISTENT) != "ERROR": - return TS_INCONSISTENT - pytest.fail("{} return unexpected state {}".format(cmd, "ERROR")) - - -def parse_routes_on_vsonic(dut_host, neigh_hosts, ip_ver): - mg_facts = dut_host.minigraph_facts( - host=dut_host.hostname)['ansible_facts'] - asn = mg_facts['minigraph_bgp_asn'] - all_routes = {} - - host_name_map = {} - for hostname, neigh_host in list(neigh_hosts.items()): - host_name_map[neigh_host['host'].hostname] = hostname - - def parse_routes_process_vsonic(node=None, results=None): - hostname = host_name_map[node['host'].hostname] - host = node['host'] - peer_ips = node['conf']['bgp']['peers'][asn] - - for ip in peer_ips: - if ipaddress.IPNetwork(ip).version == 4: - peer_ip_v4 = ip - else: - peer_ip_v6 = ip - - if 4 == ip_ver: - conf_cmd = "sudo vtysh -c 'configure terminal' -c 'router bgp' -c 'address-family ipv4' -c \ - 'neighbor {} soft-reconfiguration inbound' ".format(peer_ip_v4) - bgp_nbr_cmd = "sudo vtysh -c 'show ip bgp neighbors {} received-routes json'".format( - peer_ip_v4) - else: - conf_cmd = "sudo vtysh -c 'configure terminal' -c 'router bgp' -c 'address-family ipv6' -c \ - 'neighbor {} soft-reconfiguration inbound' ".format(peer_ip_v6) - bgp_nbr_cmd = "sudo vtysh -c 'show bgp ipv6 neighbors {} received-routes json'".format( - peer_ip_v6) - - host.shell(conf_cmd) - res = host.shell(bgp_nbr_cmd) - routes_json = json.loads(res['stdout'])['receivedRoutes'] - - routes = {} - for a_route in routes_json: - # empty community string - routes[a_route] = "" - results[hostname] = routes - - all_routes = parallel_run(parse_routes_process_vsonic, (), {}, list(neigh_hosts.values()), - timeout=120, concurrent_tasks=8) - return all_routes - - -def parse_routes_on_eos(dut_host, neigh_hosts, ip_ver, exp_community=[]): - """ - Parse the output of 'show ip bgp neigh received-routes' on eos, and store in a dict - """ - mg_facts = dut_host.minigraph_facts( - host=dut_host.hostname)['ansible_facts'] - asn = mg_facts['minigraph_bgp_asn'] - all_routes = {} - BGP_ENTRY_HEADING = r"BGP routing table entry for " - BGP_COMMUNITY_HEADING = r"Community: " - - # {'VM0122': 'ARISTA11T0',...} - host_name_map = {} - for hostname, neigh_host in list(neigh_hosts.items()): - host_name_map[neigh_host['host'].hostname] = hostname - - # Retrieve the routes on all VMs in parallel by using a thread poll - def parse_routes_process(node=None, results=None, my_community=exp_community): - """ - The process to parse routes on a VM. - :param neigh_host_item: tuple of hostname and host_conf dict - :return: no return value - """ - # get hostname('ARISTA11T0') by VM name('VM0122') - hostname = host_name_map[node['host'].hostname] - host = node['host'] - peer_ips = node['conf']['bgp']['peers'][asn] - for ip in peer_ips: - if ipaddress.IPNetwork(ip).version == 4: - peer_ip_v4 = ip - else: - peer_ip_v6 = ip - # The json formatter on EOS consumes too much time (over 40 seconds). - # So we have to parse the raw output instead json. - if 4 == ip_ver: - cmd = "show ip bgp neighbors {} received-routes detail | grep -E \"{}|{}\""\ - .format(peer_ip_v4, BGP_ENTRY_HEADING, BGP_COMMUNITY_HEADING) - cmd_backup = "" - else: - cmd = "show ipv6 bgp peers {} received-routes detail | grep -E \"{}|{}\""\ - .format(peer_ip_v6, BGP_ENTRY_HEADING, BGP_COMMUNITY_HEADING) - # For compatibility on EOS of old version - cmd_backup = "show ipv6 bgp neighbors {} received-routes detail | grep -E \"{}|{}\""\ - .format(peer_ip_v6, BGP_ENTRY_HEADING, BGP_COMMUNITY_HEADING) - res = host.eos_command(commands=[cmd], module_ignore_errors=True) - if res['failed'] and cmd_backup != "": - res = host.eos_command( - commands=[cmd_backup], module_ignore_errors=True) - pytest_assert( - not res['failed'], "Failed to retrieve routes from VM {}".format(hostname)) - routes = {} - routes_with_community = {} - entry = None - for line in res['stdout_lines'][0]: - addr = re.findall(BGP_ENTRY_HEADING + r"(.+)", line) - if addr: - if entry: - routes[entry] = "" - entry = None - entry = addr[0] - community = re.findall(BGP_COMMUNITY_HEADING + r"(.+)", line) - if community: - if entry: - routes[entry] = community[0] - if my_community: - for comm in my_community: - if comm in community[0]: - routes_with_community[entry] = comm - break - entry = None - community = "" - if entry: - routes[entry] = community - if my_community: - for comm in my_community: - if comm in community: - routes_with_community[entry] = comm - if my_community: - results[hostname] = routes_with_community - else: - results[hostname] = routes - try: - all_routes = parallel_run(parse_routes_process, (), {}, list( - neigh_hosts.values()), timeout=240, concurrent_tasks=8) - except BaseException as err: - logger.error( - 'Failed to get routes info from VMs. Got error: {}\n\nTrying one more time.'.format(err)) - all_routes = parallel_run(parse_routes_process, (), {}, list( - neigh_hosts.values()), timeout=240, concurrent_tasks=8) - return all_routes - - -def parse_routes_on_neighbors(dut_host, neigh_hosts, ip_ver, exp_community=[]): - if isinstance(list(neigh_hosts.items())[0][1]['host'], EosHost): - routes_on_all_nbrs = parse_routes_on_eos(dut_host, neigh_hosts, ip_ver, exp_community) - else: - routes_on_all_nbrs = parse_routes_on_vsonic( - dut_host, neigh_hosts, ip_ver) - return routes_on_all_nbrs - - def verify_all_routes_announce_to_neighs(dut_host, neigh_hosts, routes_dut, ip_ver): """ Verify all routes are announced to neighbors in TSB @@ -254,131 +61,6 @@ def verify_all_routes_announce_to_neighs(dut_host, neigh_hosts, routes_dut, ip_v return True -def verify_current_routes_announced_to_neighs(dut_host, neigh_hosts, orig_routes_on_all_nbrs, - cur_routes_on_all_nbrs, ip_ver, exp_community=[]): - """ - Verify all the original routes are announced to neighbors after TSB - """ - logger.info( - "Verifying all the original routes(ipv{}) are announced to bgp neighbors".format(ip_ver)) - cur_routes_on_all_nbrs.update( - parse_routes_on_neighbors(dut_host, neigh_hosts, ip_ver, exp_community)) - # Compare current routes after TSB with original routes advertised to neighbors - if cur_routes_on_all_nbrs != orig_routes_on_all_nbrs: - return False - return True - - -def check_and_log_routes_diff(duthost, neigh_hosts, orig_routes_on_all_nbrs, cur_routes_on_all_nbrs, ip_ver): - cur_nbrs = set(cur_routes_on_all_nbrs.keys()) - orig_nbrs = set(orig_routes_on_all_nbrs.keys()) - if cur_nbrs != orig_nbrs: - logger.warn("Neighbor list mismatch: {}".format(cur_nbrs ^ orig_nbrs)) - return False - - routes_dut = parse_rib(duthost, ip_ver) - all_diffs_in_host_aspath = True - for hostname in list(orig_routes_on_all_nbrs.keys()): - if orig_routes_on_all_nbrs[hostname] != cur_routes_on_all_nbrs[hostname]: - routes_diff = set(orig_routes_on_all_nbrs[hostname]) ^ set( - cur_routes_on_all_nbrs[hostname]) - for route in routes_diff: - if route not in list(routes_dut.keys()): - all_diffs_in_host_aspath = False - logger.warn( - "Missing route on host {}: {}".format(hostname, route)) - continue - aspaths = routes_dut[route] - # Filter out routes announced by this neigh - skip = False - if isinstance(list(neigh_hosts.items())[0][1]['host'], EosHost): - for aspath in aspaths: - if str(neigh_hosts[hostname]['conf']['bgp']['asn']) in aspath: - logger.debug( - "Skipping route {} on host {}".format(route, hostname)) - skip = True - break - if not skip: - all_diffs_in_host_aspath = False - if route in orig_routes_on_all_nbrs[hostname]: - logger.warn( - "Missing route on host {}: {}".format(hostname, route)) - else: - logger.warn( - "Additional route on host {}: {}".format(hostname, route)) - - return all_diffs_in_host_aspath - - -def verify_loopback_route_with_community(dut_hosts, duthost, neigh_hosts, ip_ver, community): - logger.info("Verifying only loopback routes are announced to bgp neighbors") - device_lo_addr_prefix_set = set() - device_ipv6_lo_addr_subnet_len_set = set() - device_traffic_shift_community_set = set() - device_traffic_shift_community_set.add(community) - device_ipv6_lo_addr_subnet_len_set.add('64') - for dut_host in dut_hosts: - if dut_host.is_supervisor_node(): - continue - mg_facts = dut_host.minigraph_facts(host=dut_host.hostname)['ansible_facts'] - for i in range(0, 2): - addr = mg_facts['minigraph_lo_interfaces'][i]['addr'] - if ipaddress.IPNetwork(addr).version == 4: - if 4 == ip_ver: - device_lo_addr_prefix_set.add(addr + "/32") - else: - # The IPv6 Loopback announced to neighbors is /64 - if 6 == ip_ver: - device_lo_addr_prefix_set.add(ipaddress.IPv6Address(addr).exploded[:20]) - routes_on_all_nbrs = parse_routes_on_neighbors(duthost, neigh_hosts, ip_ver) - for hostname, routes in list(routes_on_all_nbrs.items()): - logger.info("Verifying only loopback routes(ipv{}) are announced to {}".format(ip_ver, hostname)) - nbr_prefix_set = set() - nbr_prefix_community_set = set() - nbr_prefix_ipv6_subnet_len_set = set() - for prefix, received_community in list(routes.items()): - if 4 == ip_ver: - nbr_prefix_set.add(prefix) - else: - nbr_prefix_set.add(ipaddress.IPv6Address(prefix.split('/')[0]).exploded[:20]) - nbr_prefix_ipv6_subnet_len_set.add(prefix.split('/')[1]) - nbr_prefix_community_set.add(received_community) - if nbr_prefix_set != device_lo_addr_prefix_set: - logger.warn("missing loopback address or some other routes present on neighbor") - return False - if 6 == ip_ver and device_ipv6_lo_addr_subnet_len_set != nbr_prefix_ipv6_subnet_len_set: - logger.warn("ipv6 subnet is not /64 for loopback") - return False - if isinstance(list(neigh_hosts.items())[0][1]['host'], EosHost): - if nbr_prefix_community_set != device_traffic_shift_community_set: - logger.warn("traffic shift away community not present on neighbor") - return False - return True - - -def verify_only_loopback_routes_are_announced_to_neighs(dut_hosts, duthost, neigh_hosts, community): - """ - Verify only loopback routes with certain community are announced to neighs in TSA - """ - return verify_loopback_route_with_community(dut_hosts, duthost, neigh_hosts, 4, community) and \ - verify_loopback_route_with_community( - dut_hosts, duthost, neigh_hosts, 6, community) - - -# API to check if the image has support for BGP_DEVICE_GLOBAL table in the configDB -def check_tsa_persistence_support(duthost): - # For multi-asic, check DB in one of the namespaces - asic_index = 0 if duthost.is_multi_asic else DEFAULT_ASIC_ID - namespace = duthost.get_namespace_from_asic_id(asic_index) - sonic_db_cmd = "sonic-db-cli {}".format("-n " + - namespace if namespace else "") - tsa_in_configdb = duthost.shell('{} CONFIG_DB HGET "BGP_DEVICE_GLOBAL|STATE" "tsa_enabled"'.format(sonic_db_cmd), - module_ignore_errors=False)['stdout_lines'] - if not tsa_in_configdb: - return False - return True - - def test_TSA(duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, nbrhosts_to_dut, bgpmon_setup_teardown, traffic_shift_community, tbinfo): """ diff --git a/tests/bgp/test_traffic_shift_sup.py b/tests/bgp/test_traffic_shift_sup.py index d0578ec30e..6a36c85d61 100644 --- a/tests/bgp/test_traffic_shift_sup.py +++ b/tests/bgp/test_traffic_shift_sup.py @@ -4,8 +4,7 @@ import time from tests.common.helpers.assertions import pytest_assert from tests.common import config_reload -from test_traffic_shift import get_traffic_shift_state -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from traffic_checker import get_traffic_shift_state pytestmark = [ pytest.mark.topology('t2') diff --git a/tests/bgp/traffic_checker.py b/tests/bgp/traffic_checker.py new file mode 100644 index 0000000000..581d48775e --- /dev/null +++ b/tests/bgp/traffic_checker.py @@ -0,0 +1,49 @@ +import pytest +from tests.common.helpers.constants import DEFAULT_ASIC_ID +from tests.bgp.constants import TS_NORMAL, TS_MAINTENANCE, TS_INCONSISTENT, TS_NO_NEIGHBORS + + +def verify_traffic_shift_per_asic(host, outputs, match_result, asic_index): + prefix = "BGP{} : ".format( + asic_index) if asic_index != DEFAULT_ASIC_ID else '' + result_str = "{}{}".format(prefix, match_result) + if result_str in outputs: + return True + else: + false = False + return false + + +def verify_traffic_shift(host, outputs, match_result): + for asic_index in host.get_frontend_asic_ids(): + if verify_traffic_shift_per_asic(host, outputs, TS_NO_NEIGHBORS, asic_index): + continue + if not verify_traffic_shift_per_asic(host, outputs, match_result, asic_index): + return "ERROR" + + return match_result + + +def get_traffic_shift_state(host, cmd="TSC"): + outputs = host.shell(cmd)['stdout_lines'] + if verify_traffic_shift(host, outputs, TS_NORMAL) != "ERROR": + return TS_NORMAL + if verify_traffic_shift(host, outputs, TS_MAINTENANCE) != "ERROR": + return TS_MAINTENANCE + if verify_traffic_shift(host, outputs, TS_INCONSISTENT) != "ERROR": + return TS_INCONSISTENT + pytest.fail("{} return unexpected state {}".format(cmd, "ERROR")) + + +# API to check if the image has support for BGP_DEVICE_GLOBAL table in the configDB +def check_tsa_persistence_support(duthost): + # For multi-asic, check DB in one of the namespaces + asic_index = 0 if duthost.is_multi_asic else DEFAULT_ASIC_ID + namespace = duthost.get_namespace_from_asic_id(asic_index) + sonic_db_cmd = "sonic-db-cli {}".format("-n " + + namespace if namespace else "") + tsa_in_configdb = duthost.shell('{} CONFIG_DB HGET "BGP_DEVICE_GLOBAL|STATE" "tsa_enabled"'.format(sonic_db_cmd), + module_ignore_errors=False)['stdout_lines'] + if not tsa_in_configdb: + return False + return True diff --git a/tests/cacl/test_cacl_application.py b/tests/cacl/test_cacl_application.py index 51a1976266..2cc13da532 100644 --- a/tests/cacl/test_cacl_application.py +++ b/tests/cacl/test_cacl_application.py @@ -8,7 +8,6 @@ from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_upper_tor # noqa F401 from tests.common.dualtor.dual_tor_utils import upper_tor_host, lower_tor_host # noqa F401 from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) @@ -405,6 +404,9 @@ def generate_expected_rules(duthost, tbinfo, docker_network, asic_index, expecte if asic_index is None: # Allow Communication among docker containers for k, v in list(docker_network['container'].items()): + # network mode for dhcp_server container is bridge, but this rule is not expected to be seen + if k == "dhcp_server": + continue iptables_rules.append("-A INPUT -s {}/32 -d {}/32 -j ACCEPT" .format(docker_network['bridge']['IPv4Address'], docker_network['bridge']['IPv4Address'])) @@ -590,8 +592,13 @@ def generate_expected_rules(duthost, tbinfo, docker_network, asic_index, expecte generate_and_append_block_ip2me_traffic_rules(duthost, iptables_rules, ip6tables_rules, asic_index) # Allow all packets with a TTL/hop limit of 0 or 1 - iptables_rules.append("-A INPUT -m ttl --ttl-lt 2 -j ACCEPT") - ip6tables_rules.append("-A INPUT -p tcp -m hl --hl-lt 2 -j ACCEPT") + iptables_rules.append("-A INPUT -p icmp -m ttl --ttl-lt 2 -j ACCEPT") + iptables_rules.append("-A INPUT -p udp -m ttl --ttl-lt 2 -m udp --dport 1025:65535 -j ACCEPT") + iptables_rules.append("-A INPUT -p tcp -m ttl --ttl-lt 2 -m tcp --dport 1025:65535 -j ACCEPT") + + ip6tables_rules.append("-A INPUT -p ipv6-icmp -m hl --hl-lt 2 -j ACCEPT") + ip6tables_rules.append("-A INPUT -p udp -m hl --hl-lt 2 -m udp --dport 1025:65535 -j ACCEPT") + ip6tables_rules.append("-A INPUT -p tcp -m hl --hl-lt 2 -m tcp --dport 1025:65535 -j ACCEPT") # If we have added rules from the device config, we lastly add default drop rules if rules_applied_from_config > 0: diff --git a/tests/cacl/test_cacl_function.py b/tests/cacl/test_cacl_function.py index bae0d25f3a..0e853be7d1 100644 --- a/tests/cacl/test_cacl_function.py +++ b/tests/cacl/test_cacl_function.py @@ -3,7 +3,6 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.snmp_helpers import get_snmp_facts from tests.common.utilities import get_data_acl, recover_acl_rule -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 try: import ntplib diff --git a/tests/cacl/test_ebtables_application.py b/tests/cacl/test_ebtables_application.py index 0f6b7af49c..790aab44e1 100644 --- a/tests/cacl/test_ebtables_application.py +++ b/tests/cacl/test_ebtables_application.py @@ -1,6 +1,5 @@ import pytest from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.disable_loganalyzer, # disable automatic loganalyzer globally diff --git a/tests/clock/test_clock.py b/tests/clock/test_clock.py index 53757a8049..409472bc30 100755 --- a/tests/clock/test_clock.py +++ b/tests/clock/test_clock.py @@ -7,7 +7,6 @@ from tests.common.errors import RunAnsibleModuleFail from tests.common.plugins.allure_wrapper import allure_step_wrapper as allure -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('any'), @@ -101,13 +100,13 @@ def verify_and_parse_show_clock_output(show_clock_output): """ with allure.step('Verify output of show clock'): try: - timezone_str = show_clock_output.split()[-1].strip() + timezone_str = show_clock_output.split()[-2].strip() logging.info(f'Timezone str: "{timezone_str}"') date_time_to_parse = show_clock_output.replace(timezone_str, '').strip() logging.info(f'Time and date to parse: "{date_time_to_parse}"') - datetime_obj = dt.datetime.strptime(date_time_to_parse, '%a %d %b %Y %I:%M:%S %p') + datetime_obj = dt.datetime.strptime(date_time_to_parse, '%a %b %d %H:%M:%S %p %Y') logging.info(f'Datetime object: "{datetime_obj}"\t|\tType: {type(datetime_obj)}') except ValueError: pytest.fail(f'Show clock output is not valid.\nOutput: "{show_clock_output}"') diff --git a/tests/common/__init__.py b/tests/common/__init__.py index 9fb3b651d2..7e2434c7ce 100644 --- a/tests/common/__init__.py +++ b/tests/common/__init__.py @@ -1,5 +1,6 @@ from .reboot import reboot -from .config_reload import config_reload +from .config_reload import config_reload, config_reload_minigraph_with_rendered_golden_config_override from .port_toggle import port_toggle -__all__ = ['reboot', 'config_reload', 'port_toggle'] +__all__ = ['reboot', 'config_reload', 'port_toggle', + 'config_reload_minigraph_with_rendered_golden_config_override'] diff --git a/tests/common/arp_utils.py b/tests/common/arp_utils.py new file mode 100644 index 0000000000..280819ae41 --- /dev/null +++ b/tests/common/arp_utils.py @@ -0,0 +1,211 @@ +import json +import logging +from tests.common.helpers.assertions import pytest_assert +from tests.ptf_runner import ptf_runner + +# Globals +PTFRUNNER_QLEN = 1000 +VXLAN_CONFIG_FILE = '/tmp/vxlan_decap.json' +DEFAULT_TEST_DURATION = 370 + +logger = logging.getLogger(__name__) + + +def __prepareVxlanConfigData(duthost, ptfhost, tbinfo): + ''' + Prepares Vxlan Configuration data for Ferret service running on PTF host + + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ptfhost (AnsibleHost): Packet Test Framework (PTF) + + Returns: + None + ''' + mgFacts = duthost.get_extended_minigraph_facts(tbinfo) + vlan_facts = duthost.vlan_facts()['ansible_facts']['ansible_vlan_facts'] + vxlanConfigData = { + 'minigraph_port_indices': mgFacts['minigraph_ptf_indices'], + 'minigraph_portchannel_interfaces': mgFacts['minigraph_portchannel_interfaces'], + 'minigraph_portchannels': mgFacts['minigraph_portchannels'], + 'minigraph_lo_interfaces': mgFacts['minigraph_lo_interfaces'], + 'vlan_facts': vlan_facts, + 'dut_mac': duthost.facts['router_mac'] + } + with open(VXLAN_CONFIG_FILE, 'w') as file: + file.write(json.dumps(vxlanConfigData, indent=4)) + + logger.info('Copying ferret config file to {0}'.format(ptfhost.hostname)) + ptfhost.copy(src=VXLAN_CONFIG_FILE, dest='/tmp/') + + +def setupFerret(duthost, ptfhost, tbinfo): + ''' + Sets Ferret service on PTF host. This class-scope fixture runs once before test start + + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ptfhost (AnsibleHost): Packet Test Framework (PTF) + + Returns: + None + ''' + ptfhost.copy(src="arp/files/ferret.py", dest="/opt") + + ''' + Get the IP which will be used by ferret script from the "ip route show type unicast" + command output. The output looks as follows: + + default proto 186 src 10.1.0.32 metric 20 + nexthop via 10.0.0.57 dev PortChannel0001 weight 1 + nexthop via 10.0.0.59 dev PortChannel0002 weight 1 + nexthop via 10.0.0.61 dev PortChannel0003 weight 1 + nexthop via 10.0.0.63 dev PortChannel0004 weight 1 + 10.0.0.56/31 dev PortChannel0001 proto kernel scope link src 10.0.0.56 + 10.232.24.0/23 dev eth0 proto kernel scope link src 10.232.24.122 + 100.1.0.29 via 10.0.0.57 dev PortChannel0001 proto 186 src 10.1.0.32 metric 20 + 192.168.0.0/21 dev Vlan1000 proto kernel scope link src 192.168.0.1 + 192.168.8.0/25 proto 186 src 10.1.0.32 metric 20 + nexthop via 10.0.0.57 dev PortChannel0001 weight 1 + nexthop via 10.0.0.59 dev PortChannel0002 weight 1 + nexthop via 10.0.0.61 dev PortChannel0003 weight 1 + nexthop via 10.0.0.63 dev PortChannel0004 weight 1 + 192.168.16.0/25 proto 186 src 10.1.0.32 metric 20 + ... + + We'll use the first subnet IP taken from zebra protocol as the base for the host IP. + As in the new SONiC image the proto will look as '186'(201911) or bgp (master) + instead of 'zebra' (like it looks in 201811)the filtering output command below will pick + the first line containing either 'proto zebra' (or 'proto 186' or 'proto bgp') + (except the one for the deafult route) and take host IP from the subnet IP. For the output + above 192.168.8.0/25 subnet will be taken and host IP given to ferret script will be 192.168.8.1 + ''' + result = duthost.shell( + cmd=r'''ip route show type unicast | + sed -e '/proto 186\|proto zebra\|proto bgp/!d' -e '/default/d' -ne '/0\//p' | + head -n 1 | + sed -ne 's/0\/.*$/1/p' + ''' + ) + + pytest_assert(len(result['stdout'].strip()) > 0, 'Empty DIP returned') + + dip = result['stdout'] + logger.info('VxLan Sender {0}'.format(dip)) + + vxlan_port_out = duthost.shell('redis-cli -n 0 hget "SWITCH_TABLE:switch" "vxlan_port"') + if 'stdout' in vxlan_port_out and vxlan_port_out['stdout'].isdigit(): + vxlan_port = int(vxlan_port_out['stdout']) + ferret_args = '-f /tmp/vxlan_decap.json -s {0} -a {1} -p {2}'.format( + dip, duthost.facts["asic_type"], vxlan_port) + else: + ferret_args = '-f /tmp/vxlan_decap.json -s {0} -a {1}'.format(dip, duthost.facts["asic_type"]) + + ptfhost.host.options['variable_manager'].extra_vars.update({'ferret_args': ferret_args}) + + logger.info('Copying ferret config file to {0}'.format(ptfhost.hostname)) + ptfhost.template(src='arp/files/ferret.conf.j2', dest='/etc/supervisor/conf.d/ferret.conf') + + logger.info('Generate pem and key files for ssl') + ptfhost.command( + cmd='''openssl req -new -x509 -keyout test.key -out test.pem -days 365 -nodes + -subj "/C=10/ST=Test/L=Test/O=Test/OU=Test/CN=test.com"''', + chdir='/opt' + ) + + __prepareVxlanConfigData(duthost, ptfhost, tbinfo) + + logger.info('Refreshing supervisor control with ferret configuration') + ptfhost.shell('supervisorctl reread && supervisorctl update') + + +def setupRouteToPtfhost(duthost, ptfhost): + ''' + Sets routes up on DUT to PTF host. This class-scope fixture runs once before test start + + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ptfhost (AnsibleHost): Packet Test Framework (PTF) + + Returns: + None + ''' + result = duthost.shell(cmd="ip route show table default | sed -n 's/default //p'") + assert len(result['stderr_lines']) == 0, 'Could not find the gateway for management port' + + gwIp = result['stdout'] + ptfIp = ptfhost.host.options['inventory_manager'].get_host(ptfhost.hostname).vars['ansible_host'] + + route = duthost.shell(cmd='ip route get {0}'.format(ptfIp))['stdout'] + if 'PortChannel' in route: + logger.info( + "Add explicit route for PTF host ({0}) through eth0 (mgmt) interface ({1})".format(ptfIp, gwIp) + ) + duthost.shell(cmd='ip route add {0}/32 {1}'.format(ptfIp, gwIp)) + + return route, ptfIp, gwIp + + +def teardownRouteToPtfhost(duthost, route, ptfIp, gwIp): + """ + Teardown the routes added by setupRouteToPtfhost + """ + if 'PortChannel' in route: + logger.info( + "Delete explicit route for PTF host ({0}) through eth0 (mgmt) interface ({1})".format(ptfIp, gwIp) + ) + result = duthost.shell(cmd='ip route delete {0}/32 {1}'.format(ptfIp, gwIp), module_ignore_errors=True) + assert result["rc"] == 0 or "No such process" in result["stderr"], \ + "Failed to delete route with error '{0}'".format(result["stderr"]) + + +def set_up(duthost, ptfhost, tbinfo): + """ + A setup function that do the exactly same thing as the autoused fixtures do + Will be called in vnet_vxlan test + """ + setupFerret(duthost, ptfhost, tbinfo) + route, ptfIp, gwIp = setupRouteToPtfhost(duthost, ptfhost) + return route, ptfIp, gwIp + + +def tear_down(duthost, route, ptfIp, gwIp): + """ + A teardown function that do some cleanup after test + Will be called in vnet_vxlan test + """ + logger.info("Clear ARP cache on DUT") + duthost.command('sonic-clear arp') + teardownRouteToPtfhost(duthost, route, ptfIp, gwIp) + + +def testWrArp(request, duthost, ptfhost, creds, skip_traffic_test): + testDuration = request.config.getoption('--test_duration', default=DEFAULT_TEST_DURATION) + ptfIp = ptfhost.host.options['inventory_manager'].get_host(ptfhost.hostname).vars['ansible_host'] + dutIp = duthost.host.options['inventory_manager'].get_host(duthost.hostname).vars['ansible_host'] + + logger.info('Warm-Reboot Control-Plane assist feature') + sonicadmin_alt_password = duthost.host.options['variable_manager']. \ + _hostvars[duthost.hostname]['sonic_default_passwords'] + if skip_traffic_test is True: + return + ptf_runner( + ptfhost, + 'ptftests', + 'wr_arp.ArpTest', + qlen=PTFRUNNER_QLEN, + platform_dir='ptftests', + platform='remote', + params={ + 'ferret_ip': ptfIp, + 'dut_ssh': dutIp, + 'dut_username': creds['sonicadmin_user'], + 'dut_password': creds['sonicadmin_password'], + "alt_password": sonicadmin_alt_password, + 'config_file': VXLAN_CONFIG_FILE, + 'how_long': testDuration, + 'advance': False, + }, + log_file='/tmp/wr_arp.ArpTest.log', + is_python3=True + ) diff --git a/tests/common/cache/facts_cache.py b/tests/common/cache/facts_cache.py index 98aa90ddf0..07f35f2913 100644 --- a/tests/common/cache/facts_cache.py +++ b/tests/common/cache/facts_cache.py @@ -19,6 +19,7 @@ SIZE_LIMIT = 1000000000 # 1G bytes, max disk usage allowed by cache ENTRY_LIMIT = 1000000 # Max number of pickle files allowed in cache. +DISABLE_CACHE_PARAM = "disable_cache" class Singleton(type): @@ -79,17 +80,17 @@ def read(self, zone, key): """ # Lazy load if zone in self._cache and key in self._cache[zone]: - logger.debug('Read cached facts "{}.{}"'.format(zone, key)) + logger.debug('[Cache] Read cached facts "{}.{}"'.format(zone, key)) return self._cache[zone][key] else: facts_file = os.path.join(self._cache_location, '{}/{}.pickle'.format(zone, key)) try: with open(facts_file, 'rb') as f: self._cache[zone][key] = pickle.load(f) - logger.debug('Loaded cached facts "{}.{}" from {}'.format(zone, key, facts_file)) + logger.debug('[Cache] Loaded cached facts "{}.{}" from {}'.format(zone, key, facts_file)) return self._cache[zone][key] except (IOError, ValueError) as e: - logger.info('Load cache file "{}" failed with exception: {}' + logger.info('[Cache] Load cache file "{}" failed with exception: {}' .format(os.path.abspath(facts_file), repr(e))) return self.NOTEXIST @@ -111,16 +112,16 @@ def write(self, zone, key, value): try: cache_subfolder = os.path.join(self._cache_location, zone) if not os.path.exists(cache_subfolder): - logger.info('Create cache dir {}'.format(cache_subfolder)) + logger.info('[Cache] Create cache dir {}'.format(cache_subfolder)) os.makedirs(cache_subfolder) with open(facts_file, 'wb') as f: pickle.dump(value, f, pickle.HIGHEST_PROTOCOL) self._cache[zone][key] = value - logger.info('Cached facts "{}.{}" to {}'.format(zone, key, facts_file)) + logger.info('[Cache] Cached facts "{}.{}" to {}'.format(zone, key, facts_file)) return True except (IOError, ValueError) as e: - logger.error('Dump cache file "{}" failed with exception: {}'.format(facts_file, repr(e))) + logger.error('[Cache] Dump cache file "{}" failed with exception: {}'.format(facts_file, repr(e))) return False def cleanup(self, zone=None, key=None): @@ -136,30 +137,31 @@ def cleanup(self, zone=None, key=None): if key: if zone in self._cache and key in self._cache[zone]: del self._cache[zone][key] - logger.debug('Removed "{}.{}" from cache.'.format(zone, key)) + logger.debug('[Cache] Removed "{}.{}" from cache.'.format(zone, key)) try: cache_file = os.path.join(self._cache_location, zone, '{}.pickle'.format(key)) os.remove(cache_file) - logger.debug('Removed cache file "{}.pickle"'.format(cache_file)) + logger.debug('[Cache] Removed cache file "{}.pickle"'.format(cache_file)) except OSError as e: - logger.error('Cleanup cache {}.{}.pickle failed with exception: {}'.format(zone, key, repr(e))) + logger.error('[Cache] Cleanup cache {}.{}.pickle failed with exception: {}' + .format(zone, key, repr(e))) else: if zone in self._cache: del self._cache[zone] - logger.debug('Removed zone "{}" from cache'.format(zone)) + logger.debug('[Cache] Removed zone "{}" from cache'.format(zone)) try: cache_subfolder = os.path.join(self._cache_location, zone) shutil.rmtree(cache_subfolder) - logger.debug('Removed cache subfolder "{}"'.format(cache_subfolder)) + logger.debug('[Cache] Removed cache subfolder "{}"'.format(cache_subfolder)) except OSError as e: - logger.error('Remove cache subfolder "{}" failed with exception: {}'.format(zone, repr(e))) + logger.error('[Cache] Remove cache subfolder "{}" failed with exception: {}'.format(zone, repr(e))) else: self._cache = defaultdict(dict) try: shutil.rmtree(self._cache_location) - logger.debug('Removed all cache files under "{}"'.format(self._cache_location)) + logger.debug('[Cache] Removed all cache files under "{}"'.format(self._cache_location)) except OSError as e: - logger.error('Remove cache folder "{}" failed with exception: {}' + logger.error('[Cache] Remove cache folder "{}" failed with exception: {}' .format(self._cache_location, repr(e))) @@ -192,6 +194,25 @@ def _get_default_zone(function, func_args, func_kargs): return zone +def _get_disable_cache(target, args, kwargs): + """ + For the function with signature: + + @cached(name='feature_status') + def get_feature_status(self, disable_cache=True): + + If the disable_cache is not explicitly passed, like it get called by .get_feature_status() + disable_cache will not show in **kwargs, + Need to fetch it with inspect. + """ + # Get the function signature + sig = inspect.signature(target) + bound_args = sig.bind_partial(*args, **kwargs) + bound_args.apply_defaults() + + return bound_args.arguments.get(DISABLE_CACHE_PARAM, False) + + def cached(name, zone_getter=None, after_read=None, before_write=None): """Decorator for enabling cache for facts. @@ -216,6 +237,12 @@ def cached(name, zone_getter=None, after_read=None, before_write=None): def decorator(target): def wrapper(*args, **kargs): + + # Support to choose enable/disable cache by function param + disable_cache = _get_disable_cache(target, args, kargs) + if disable_cache: + return target(*args, **kargs) + _zone_getter = zone_getter or _get_default_zone zone = _zone_getter(target, args, kargs) @@ -223,6 +250,7 @@ def wrapper(*args, **kargs): if after_read: cached_facts = after_read(cached_facts, target, args, kargs) if cached_facts is not FactsCache.NOTEXIST: + logger.debug(f"[Cache] Use cache for func[{target}], zone[{zone}], key[{name}]") return cached_facts else: facts = target(*args, **kargs) diff --git a/tests/common/config_reload.py b/tests/common/config_reload.py index 89a131ba1e..1fc41582c2 100644 --- a/tests/common/config_reload.py +++ b/tests/common/config_reload.py @@ -1,5 +1,6 @@ import time import logging +import os from tests.common.helpers.assertions import pytest_assert from tests.common.plugins.loganalyzer.utils import ignore_loganalyzer @@ -13,6 +14,11 @@ config_sources = ['config_db', 'minigraph', 'running_golden_config'] +BASE_DIR = os.path.dirname(os.path.realpath(__file__)) +TEMPLATE_DIR = os.path.join(BASE_DIR, 'templates') +GOLDEN_CONFIG_TEMPLATE = os.path.join(TEMPLATE_DIR, 'golden_config_db.j2') +DEFAULT_GOLDEN_CONFIG_PATH = '/etc/sonic/golden_config_db.json' + def config_system_checks_passed(duthost, delayed_services=[]): logging.info("Checking if system is running") @@ -61,10 +67,55 @@ def config_force_option_supported(duthost): return False +def config_reload_minigraph_with_rendered_golden_config_override( + sonic_host, wait=120, start_bgp=True, start_dynamic_buffer=True, + safe_reload=False, wait_before_force_reload=0, wait_for_bgp=False, + check_intf_up_ports=False, traffic_shift_away=False, + golden_config_path=DEFAULT_GOLDEN_CONFIG_PATH, + local_golden_config_template=GOLDEN_CONFIG_TEMPLATE, + dut_golden_config_template=None, remote_src=False, is_dut=True): + """ + This function facilitates new feature table testing without minigraph parser modification. It + reloads the minigraph using a j2 file to render Golden Config, which overrides the ConfigDB. + This function includes all parameters from config_reload() with the restraint parameters + listed below: + :param config_source: Always set to 'minigraph' cuz Golden Config override are embeded in + load_minigraph + :param override_config: Always True as it needs Golden Config to override + :param golden_config_path: Path of Golden Config on DUT + :param local_golden_config_template: template in sonic-mgmt repo and will be copy to DUT to parse + :param dut_golden_config_template: template that located in remote DUT if there is any. + :param remote_src: Whether `src` is on the remote host or on the calling device. + """ + # If dut_template_path is being set, we can directly generate Golden Config from there. + # Otherwise, we can copy and parse the template in sonic-mgmt repo + if dut_golden_config_template: + sonic_host.shell("sonic-cfggen -d -t {} > {}".format(dut_golden_config_template, golden_config_path)) + else: + dut_golden_config_template = '/tmp/golden_config_db.j2' + # default src: tests/common/templates/golden_config_db.j2 + # The src could be specified in the template dir under your test. + # Check test_config_reload_with_rendered_golden_config.py + sonic_host.copy(src=local_golden_config_template, dest=dut_golden_config_template, remote_src=remote_src) + # run sonic-cfggen to generate golden_config_db.json with existing config. + sonic_host.shell("sonic-cfggen -d -t {} > {}".format(dut_golden_config_template, golden_config_path)) + + config_reload(sonic_host, 'minigraph', wait, start_bgp, start_dynamic_buffer, safe_reload, + wait_before_force_reload, wait_for_bgp, check_intf_up_ports, traffic_shift_away, + override_config=True, golden_config_path=golden_config_path, is_dut=is_dut) + + +def pfcwd_feature_enabled(duthost): + device_metadata = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts']['DEVICE_METADATA'] + pfc_status = device_metadata['localhost']["default_pfcwd_status"] + return pfc_status == 'enable' + + @ignore_loganalyzer def config_reload(sonic_host, config_source='config_db', wait=120, start_bgp=True, start_dynamic_buffer=True, safe_reload=False, wait_before_force_reload=0, wait_for_bgp=False, - check_intf_up_ports=False, traffic_shift_away=False, override_config=False, is_dut=True): + check_intf_up_ports=False, traffic_shift_away=False, override_config=False, + golden_config_path=DEFAULT_GOLDEN_CONFIG_PATH, is_dut=True): """ reload SONiC configuration :param sonic_host: SONiC host object @@ -108,6 +159,8 @@ def _config_reload_cmd_wrapper(cmd, executable): cmd += ' -t' if override_config: cmd += ' -o' + if golden_config_path: + cmd += ' -p {} '.format(golden_config_path) sonic_host.shell(cmd, executable="/bin/bash") time.sleep(60) if start_bgp: @@ -124,20 +177,21 @@ def _config_reload_cmd_wrapper(cmd, executable): reloading = wait_until(wait_before_force_reload, 10, 0, _config_reload_cmd_wrapper, cmd, "/bin/bash") cmd = 'config reload -y -f &>/dev/null' if not reloading: + time.sleep(30) sonic_host.shell(cmd, executable="/bin/bash") elif config_source == 'running_golden_config': golden_path = '/etc/sonic/running_golden_config.json' if sonic_host.is_multi_asic: for asic in sonic_host.asics: - golden_path = f'{golden_path},/etc/sonic/running_golden_config{asic.asic_index}.json' + golden_path = f'{golden_path},/etc/sonic/running_golden_config{asic.asic_index}.json' # noqa: E231 cmd = f'config reload -y -l {golden_path} &>/dev/null' if config_force_option_supported(sonic_host): cmd = f'config reload -y -f -l {golden_path} &>/dev/null' sonic_host.shell(cmd, executable="/bin/bash") modular_chassis = sonic_host.get_facts().get("modular_chassis") - wait = max(wait, 240) if modular_chassis.lower() == 'true' else wait + wait = max(wait, 900) if modular_chassis else wait if safe_reload: # The wait time passed in might not be guaranteed to cover the actual @@ -147,17 +201,21 @@ def _config_reload_cmd_wrapper(cmd, executable): pytest_assert(wait_until(wait + 300, 20, 0, sonic_host.critical_services_fully_started), "All critical services should be fully started!") wait_critical_processes(sonic_host) - if config_source == 'minigraph': - pytest_assert(wait_until(300, 20, 0, chk_for_pfc_wd, sonic_host), - "PFC_WD is missing in CONFIG-DB") - + # PFCWD feature does not enable on some topology, for example M0 + if config_source == 'minigraph' and pfcwd_feature_enabled(sonic_host): + # Supervisor node doesn't have PFC_WD + if not sonic_host.is_supervisor_node(): + pytest_assert(wait_until(wait + 300, 20, 0, chk_for_pfc_wd, sonic_host), + "PFC_WD is missing in CONFIG-DB") if check_intf_up_ports: - pytest_assert(wait_until(300, 20, 0, check_interface_status_of_up_ports, sonic_host), + pytest_assert(wait_until(wait + 300, 20, 0, check_interface_status_of_up_ports, sonic_host), "Not all ports that are admin up on are operationally up") else: time.sleep(wait) if wait_for_bgp: - bgp_neighbors = sonic_host.get_bgp_neighbors().keys() - pytest_assert(wait_until(120, 10, 0, sonic_host.check_bgp_session_state, bgp_neighbors), - "Not all bgp sessions are established after config reload") + bgp_neighbors = sonic_host.get_bgp_neighbors_per_asic() + pytest_assert( + wait_until(wait + 120, 10, 0, sonic_host.check_bgp_session_state_all_asics, bgp_neighbors), + "Not all bgp sessions are established after config reload", + ) diff --git a/tests/common/connections/ssh_console_conn.py b/tests/common/connections/ssh_console_conn.py index c6bb511e1c..ecc93b090a 100644 --- a/tests/common/connections/ssh_console_conn.py +++ b/tests/common/connections/ssh_console_conn.py @@ -2,6 +2,7 @@ import re from .base_console_conn import BaseConsoleConn, CONSOLE_SSH from netmiko.ssh_exception import NetMikoAuthenticationException +from paramiko.ssh_exception import SSHException class SSHConsoleConn(BaseConsoleConn): @@ -26,7 +27,13 @@ def __init__(self, **kwargs): super(SSHConsoleConn, self).__init__(**kwargs) def session_preparation(self): - self._test_channel_read() + session_init_msg = self._test_channel_read() + self.logger.debug(session_init_msg) + + if re.search(r"Port is in use. Closing connection...", session_init_msg, flags=re.M): + console_port = self.username.split(':')[-1] + raise PortInUseException(f"Host closed connection, as console port '{console_port}' is currently occupied.") + if (self.menu_port): # For devices logining via menu port, 2 additional login are needed # Since we have attempted all passwords in __init__ of base class until successful login @@ -151,3 +158,8 @@ def cleanup(self): # and any other login is prevented self.remote_conn.close() del self.remote_conn + + +class PortInUseException(SSHException): + '''Exception to denote a console port is in use.''' + pass diff --git a/tests/common/devices/aos.py b/tests/common/devices/aos.py index d8d317b1e0..14365bc065 100644 --- a/tests/common/devices/aos.py +++ b/tests/common/devices/aos.py @@ -168,6 +168,16 @@ def set_speed(self, interface_name, speed): parents='interface %s' % interface_name) return not self._has_cli_cmd_failed(out) + def is_lldp_disabled(self): + """ + TODO: Add support for AOS device when access to + AOS fanout becomes available. + + Return False always. If AOS device is found as a + fanout the pretest will fail until this check is implemented. + """ + return False + def speed_gb_to_mb(speed): res = re.search(r'(\d+)(\w)', speed) diff --git a/tests/common/devices/eos.py b/tests/common/devices/eos.py index 327a99e646..e24860ac44 100644 --- a/tests/common/devices/eos.py +++ b/tests/common/devices/eos.py @@ -102,6 +102,20 @@ def no_shutdown_multiple(self, interfaces): intf_str = ','.join(interfaces) return self.no_shutdown(intf_str) + def is_lldp_disabled(self): + """ + Checks if LLDP is enabled by neighbors + Returns True if disabled (i.e. neighbors absent) + Returns False if enabled (i.e. found neighbors) + """ + command = 'show lldp neighbors | json' + output = self.eos_command(commands=[command])['stdout'] + logger.debug(f'lldp neighbors returned: {output}') + # check for empty output -> [''] + if output is None or (len(output) == 1 and len(output[0]) == 0): + return True + return False + def check_intf_link_state(self, interface_name): """ This function returns link oper status diff --git a/tests/common/devices/fanout.py b/tests/common/devices/fanout.py index 1c80636ae6..bf8a632620 100644 --- a/tests/common/devices/fanout.py +++ b/tests/common/devices/fanout.py @@ -210,3 +210,11 @@ def links_status_up(self, ports): def set_port_fec(self, interface_name, mode): self.host.set_port_fec(interface_name, mode) + + def is_lldp_disabled(self): + """Check global LLDP status on the device + Returns: + True: if LLDP is disabled + False: if LLDP is enabled + """ + return self.host.is_lldp_disabled() diff --git a/tests/common/devices/ixia.py b/tests/common/devices/ixia.py index 9dbdd8d956..e788f940bd 100644 --- a/tests/common/devices/ixia.py +++ b/tests/common/devices/ixia.py @@ -57,3 +57,13 @@ def execute(self, cmd): """ if (self.os == 'ixia'): eval(cmd) + + def is_lldp_disabled(self): + """ + TODO: Add support for IXIA device when access to + IXIA fanout becomes available. + + Return False always. If IXIA device is found as a + fanout the pretest will fail until this check is implemented. + """ + return False diff --git a/tests/common/devices/multi_asic.py b/tests/common/devices/multi_asic.py index 59884b0340..9660815b6c 100644 --- a/tests/common/devices/multi_asic.py +++ b/tests/common/devices/multi_asic.py @@ -65,7 +65,7 @@ def critical_services_tracking_list(self): service_list = [] active_asics = self.asics if self.sonichost.is_supervisor_node(): - self._DEFAULT_SERVICES.append("lldp") + service_list.append("lldp") if self.get_facts()['asic_type'] != 'vs': active_asics = [] sonic_db_cli_out = \ @@ -98,7 +98,7 @@ def critical_services_tracking_list(self): if config_facts['FEATURE'][service]['state'] == "disabled": self.sonichost.DEFAULT_ASIC_SERVICES.remove(service) else: - self._DEFAULT_SERVICES.append("lldp") + service_list.append("lldp") for asic in active_asics: service_list += asic.get_critical_services() diff --git a/tests/common/devices/onyx.py b/tests/common/devices/onyx.py index b3f467b72d..e3d4f898bd 100644 --- a/tests/common/devices/onyx.py +++ b/tests/common/devices/onyx.py @@ -189,19 +189,6 @@ def set_speed(self, interface_name, speed): speed = 'auto' else: speed = speed[:-3] + 'G' - # The speed support list for onyx is like '1G 10G 25G 40G 50Gx1 50Gx2 100Gx2 100Gx4 200Gx4'. - # We need to set the speed according to the speed support list. - # For example, when dut and fanout all support 50G, - # if support speed list of fanout just includes 50Gx1 not 50G, - # we need to set the speed with 50Gx1 instead of 50G, otherwise, the port can not be up. - all_support_speeds = self.get_supported_speeds(interface_name, raw_data=True) - for support_speed in all_support_speeds: - if speed in support_speed: - logger.info("Speed {} find the matched support speed:{} ".format(speed, support_speed)) - speed = support_speed - break - logger.info("set speed is {}".format(speed)) - if autoneg_mode or speed == 'auto': out = self.host.onyx_config( lines=['shutdown', 'speed {}'.format(speed), 'no shutdown'], @@ -322,3 +309,13 @@ def restore_drop_counter_config(self): is not supported or failed. """ return self.fanout_helper.restore_drop_counter_config() + + def is_lldp_disabled(self): + """ + TODO: Add support for Onyx device when access to + Onyx fanout becomes available. + + Return False always. If Onyx device is found as a + fanout the pretest will fail until this check is implemented. + """ + return False diff --git a/tests/common/devices/sonic.py b/tests/common/devices/sonic.py index b090e176cb..719b3d916f 100644 --- a/tests/common/devices/sonic.py +++ b/tests/common/devices/sonic.py @@ -17,6 +17,7 @@ from tests.common.devices.base import AnsibleHostBase from tests.common.devices.constants import ACL_COUNTERS_UPDATE_INTERVAL_IN_SEC from tests.common.helpers.dut_utils import is_supervisor_node, is_macsec_capable_node +from tests.common.str_utils import str2bool from tests.common.utilities import get_host_visible_vars from tests.common.cache import cached from tests.common.helpers.constants import DEFAULT_ASIC_ID, DEFAULT_NAMESPACE @@ -57,17 +58,17 @@ def __init__(self, ansible_adhoc, hostname, vm = self.host.options['variable_manager'] sonic_conn = vm.get_vars( host=im.get_hosts(pattern='sonic')[0] - )['ansible_connection'] + )['ansible_connection'] hostvars = vm.get_vars(host=im.get_host(hostname=self.hostname)) # parse connection options and reset those options with # passed credentials connection_loader.get(sonic_conn, class_only=True) user_def = ansible_constants.config.get_configuration_definition( "remote_user", "connection", sonic_conn - ) + ) pass_def = ansible_constants.config.get_configuration_definition( "password", "connection", sonic_conn - ) + ) for user_var in (_['name'] for _ in user_def['vars']): if user_var in hostvars: vm.extra_vars.update({user_var: shell_user}) @@ -86,7 +87,7 @@ def __init__(self, ansible_adhoc, hostname, self._os_version = self._get_os_version() if 'router_type' in self.facts and self.facts['router_type'] == 'spinerouter': self.DEFAULT_ASIC_SERVICES.append("macsec") - feature_status = self.get_feature_status() + feature_status = self.get_feature_status(disable_cache=False) # Append gbsyncd only for non-VS to avoid pretest check for gbsyncd # e.g. in test_feature_status, test_disable_rsyslog_rate_limit gbsyncd_enabled = 'gbsyncd' in feature_status[0].keys() and feature_status[0]['gbsyncd'] == 'enabled' @@ -214,7 +215,7 @@ def _gather_facts(self): facts["num_asic"] = results[0] facts["router_mac"] = results[1] - facts["modular_chassis"] = results[2] + facts["modular_chassis"] = str2bool(results[2]) facts["mgmt_interface"] = results[3] facts["switch_type"] = results[4] facts["router_type"] = results[5] @@ -251,9 +252,9 @@ def _get_modular_chassis(self): py_res = self.shell("python -c \"import sonic_platform\"", module_ignore_errors=True) if py_res["failed"]: out = self.shell( - "python3 -c \"import sonic_platform.platform as P; \ + "python3 -c \"import sonic_platform.platform as P; \ print(P.Platform().get_chassis().is_modular_chassis()); exit()\"", - module_ignore_errors=True) + module_ignore_errors=True) else: out = self.shell( "python -c \"import sonic_platform.platform as P; \ @@ -329,11 +330,12 @@ def _get_platform_info(self): except Exception: # if platform.json does not exist, then it's not added currently for certain platforms # eventually all the platforms should have the platform.json - logging.debug("platform.json is not available for this platform, " - + "DUT facts will not contain complete platform information.") + logging.debug("platform.json is not available for this platform, " + + "DUT facts will not contain complete platform information.") return result + @cached(name='os_version') def _get_os_version(self): """ Gets the SONiC OS version that is running on this device. @@ -342,6 +344,7 @@ def _get_os_version(self): output = self.command("sonic-cfggen -y /etc/sonic/sonic_version.yml -v build_version") return output["stdout_lines"][0].strip() + @cached(name='sonic_release') def _get_sonic_release(self): """ Gets the SONiC Release that is running on this device. @@ -357,6 +360,7 @@ def _get_sonic_release(self): return 'none' return output["stdout_lines"][0].strip() + @cached(name='kernel_version') def _get_kernel_version(self): """ Gets the SONiC kernel version @@ -447,6 +451,14 @@ def get_running_containers(self): """ return self.shell(r'docker ps --format \{\{.Names\}\}')['stdout_lines'] + def get_all_containers(self): + """ + Get all containers names + :param duthost: DUT host object + :return: Running container name list + """ + return self.shell(r'docker ps -a --format \{\{.Names\}\}')['stdout_lines'] + def is_container_running(self, service): """ Checks where a container exits. @@ -1150,6 +1162,22 @@ def no_shutdown_multiple(self, ifnames): intf_str = ','.join(ifnames) return self.no_shutdown(intf_str) + def is_lldp_disabled(self): + """ + Checks LLDP feature status + Returns True if disabled + Returns False if enabled + """ + # get lldp status without table header + lldp_status_output = self.command('show feature status lldp | tail -n +3')["stdout_lines"][0] + if lldp_status_output is not None: + fields = lldp_status_output.split() + # State is the second field + state = fields[1] + if state == "enabled": + return False + return True + def get_ip_route_info(self, dstip, ns=""): """ @summary: return route information for a destionation. The destination could an ip address or ip prefix. @@ -1503,10 +1531,14 @@ def get_container_autorestart_states(self): return container_autorestart_states - def get_feature_status(self): + @cached(name='feature_status') + def get_feature_status(self, disable_cache=True): """ Gets the list of features and states + params: + disable_cache: disable cache and get real-time feature status, default True + Returns: dict: feature status dict. { : } bool: status obtained successfully (True | False) @@ -1564,9 +1596,9 @@ def _parse_show(self, output_lines, header_len=1): for idx, line in enumerate(output_lines): if sep_line_pattern.match(line): sep_line_found = True - header_lines = output_lines[idx-header_len:idx] + header_lines = output_lines[idx - header_len:idx] sep_line = output_lines[idx] - content_lines = output_lines[idx+1:] + content_lines = output_lines[idx + 1:] break if not sep_line_found: @@ -2400,6 +2432,28 @@ def add_ip_addr_to_port(self, port, ip, gwaddr): """ self.command("config interface ip add {} {} {}".format(port, ip, gwaddr)) + def remove_ip_addr_from_vlan(self, vlan, ip): + """ + Remove ip addr from the vlan. + :param vlan: vlan name + :param ip: IP address + + Example: + config interface ip remove Vlan1000 192.168.0.0/24 + """ + self.command("config interface ip remove {} {}".format(vlan, ip)) + + def add_ip_addr_to_vlan(self, vlan, ip): + """ + Add ip addr to the vlan. + :param vlan: vlan name + :param ip: IP address + + Example: + config interface ip add Vlan1000 192.168.0.0/24 + """ + self.command("config interface ip add {} {}".format(vlan, ip)) + def remove_vlan(self, vlan_id): """ Remove vlan @@ -2490,6 +2544,50 @@ def get_sfp_type(self, portname): sfp_type = re.search(r'[QO]?SFP-?[\d\w]{0,3}', out["stdout_lines"][0]).group() return sfp_type + def get_switch_hash_capabilities(self): + out = self.shell('show switch-hash capabilities --json') + assert_exit_non_zero(out) + return SonicHost._parse_hash_fields(out) + + def get_switch_hash_configurations(self): + out = self.shell('show switch-hash global --json') + assert_exit_non_zero(out) + return SonicHost._parse_hash_fields(out) + + def set_switch_hash_global(self, hash_type, fields, validate=True): + cmd = 'config switch-hash global {}-hash'.format(hash_type) + for field in fields: + cmd += ' ' + field + out = self.shell(cmd, module_ignore_errors=True) + if validate: + assert_exit_non_zero(out) + return out + + def set_switch_hash_global_algorithm(self, hash_type, algorithm, validate=True): + cmd = 'config switch-hash global {}-hash-algorithm {}'.format(hash_type, algorithm) + out = self.shell(cmd, module_ignore_errors=True) + if validate: + assert_exit_non_zero(out) + return out + + @staticmethod + def _parse_hash_fields(cli_output): + ecmp_hash_fields = [] + lag_hash_fields = [] + ecmp_hash_algorithm = lag_hash_algorithm = '' + if "No configuration is present in CONFIG DB" in cli_output['stdout']: + logger.info("No configuration is present in CONFIG DB") + else: + out_json = json.loads(cli_output['stdout']) + ecmp_hash_fields = out_json["ecmp"]["hash_field"] + lag_hash_fields = out_json["lag"]["hash_field"] + ecmp_hash_algorithm = out_json["ecmp"]["algorithm"] + lag_hash_algorithm = out_json["lag"]["algorithm"] + return {'ecmp': ecmp_hash_fields, + 'lag': lag_hash_fields, + 'ecmp_algo': ecmp_hash_algorithm, + 'lag_algo': lag_hash_algorithm} + def get_counter_poll_status(self): result_dict = {} output = self.shell("counterpoll show")["stdout_lines"][2::] diff --git a/tests/common/devices/sonic_asic.py b/tests/common/devices/sonic_asic.py index ae1b2ee6fe..b08bc2616d 100644 --- a/tests/common/devices/sonic_asic.py +++ b/tests/common/devices/sonic_asic.py @@ -3,7 +3,9 @@ import socket import re +from tests.common.cache import cached from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.cache_utils import sonic_asic_zone_getter from tests.common.helpers.constants import DEFAULT_NAMESPACE, NAMESPACE_PREFIX from tests.common.errors import RunAnsibleModuleFail from tests.common.platform.ssh_utils import ssh_authorize_local_user @@ -30,6 +32,7 @@ def __init__(self, sonichost, asic_index): asic_index: ASIC / namespace id for this asic. """ self.sonichost = sonichost + self.hostname = self.sonichost.hostname self.asic_index = asic_index self.ns_arg = "" if self.sonichost.is_multi_asic: @@ -67,6 +70,7 @@ def get_critical_services(self): service, self.asic_index if self.sonichost.is_multi_asic else "")) return a_service + @cached(name='is_frontend_asic', zone_getter=sonic_asic_zone_getter) def is_it_frontend(self): if self.sonichost.is_multi_asic: sub_role_cmd = 'sudo sonic-cfggen -d -v DEVICE_METADATA.localhost.sub_role -n {}'.format(self.namespace) @@ -75,6 +79,7 @@ def is_it_frontend(self): return True return False + @cached(name='is_backend_asic', zone_getter=sonic_asic_zone_getter) def is_it_backend(self): if self.sonichost.is_multi_asic: sub_role_cmd = 'sudo sonic-cfggen -d -v DEVICE_METADATA.localhost.sub_role -n {}'.format(self.namespace) diff --git a/tests/common/dualtor/data_plane_utils.py b/tests/common/dualtor/data_plane_utils.py index 9787c1c864..cb15313a9e 100644 --- a/tests/common/dualtor/data_plane_utils.py +++ b/tests/common/dualtor/data_plane_utils.py @@ -40,7 +40,8 @@ def arp_setup(ptfhost): ptfhost.shell("supervisorctl reread && supervisorctl update") -def validate_traffic_results(tor_IO, allowed_disruption, delay, allow_disruption_before_traffic=False): +def validate_traffic_results(tor_IO, allowed_disruption, delay, + allow_disruption_before_traffic=False): """ Generates a report (dictionary) of I/O metrics that were calculated as part of the dataplane test. This report is to be used by testcases to verify the @@ -148,7 +149,8 @@ def _validate_long_disruption(disruptions, allowed_disruption, delay): return False -def verify_and_report(tor_IO, verify, delay, allowed_disruption, allow_disruption_before_traffic=False): +def verify_and_report(tor_IO, verify, delay, allowed_disruption, + allow_disruption_before_traffic=False): # Wait for the IO to complete before doing checks if verify: validate_traffic_results(tor_IO, allowed_disruption=allowed_disruption, delay=delay, @@ -198,7 +200,9 @@ def run_test( tor_IO.stop_early = True # Wait for the IO to complete before doing checks send_and_sniff.join() - tor_IO.examine_flow() + # Skip flow examination for VS platform + if activehost.facts["asic_type"] != "vs": + tor_IO.examine_flow() return tor_IO @@ -236,7 +240,8 @@ def save_pcap(request, pytestconfig): @pytest.fixture -def send_t1_to_server_with_action(duthosts, ptfhost, ptfadapter, tbinfo, cable_type, vmhost, save_pcap): # noqa F811 +def send_t1_to_server_with_action(duthosts, ptfhost, ptfadapter, tbinfo, + cable_type, vmhost, save_pcap, skip_traffic_test=False): # noqa F811 """ Starts IO test from T1 router to server. As part of IO test the background thread sends and sniffs packets. @@ -258,7 +263,8 @@ def send_t1_to_server_with_action(duthosts, ptfhost, ptfadapter, tbinfo, cable_t def t1_to_server_io_test(activehost, tor_vlan_port=None, delay=0, allowed_disruption=0, action=None, verify=False, send_interval=0.1, - stop_after=None, allow_disruption_before_traffic=False): + stop_after=None, allow_disruption_before_traffic=False, + skip_traffic_test=False): """ Helper method for `send_t1_to_server_with_action`. Starts sender and sniffer before performing the action on the tor host. @@ -293,6 +299,9 @@ def t1_to_server_io_test(activehost, tor_vlan_port=None, if delay and not allowed_disruption: allowed_disruption = 1 + if skip_traffic_test is True: + logging.info("Skipping traffic test") + return return verify_and_report(tor_IO, verify, delay, allowed_disruption, allow_disruption_before_traffic) yield t1_to_server_io_test @@ -301,7 +310,8 @@ def t1_to_server_io_test(activehost, tor_vlan_port=None, @pytest.fixture -def send_server_to_t1_with_action(duthosts, ptfhost, ptfadapter, tbinfo, cable_type, vmhost, save_pcap): # noqa F811 +def send_server_to_t1_with_action(duthosts, ptfhost, ptfadapter, tbinfo, + cable_type, vmhost, save_pcap, skip_traffic_test=False): # noqa F811 """ Starts IO test from server to T1 router. As part of IO test the background thread sends and sniffs packets. @@ -324,7 +334,7 @@ def send_server_to_t1_with_action(duthosts, ptfhost, ptfadapter, tbinfo, cable_t def server_to_t1_io_test(activehost, tor_vlan_port=None, delay=0, allowed_disruption=0, action=None, verify=False, send_interval=0.01, - stop_after=None): + stop_after=None, skip_traffic_test=False): """ Helper method for `send_server_to_t1_with_action`. Starts sender and sniffer before performing the action on the tor host. @@ -358,6 +368,9 @@ def server_to_t1_io_test(activehost, tor_vlan_port=None, if delay and not allowed_disruption: allowed_disruption = 1 + if skip_traffic_test is True: + logging.info("Skipping traffic test") + return return verify_and_report(tor_IO, verify, delay, allowed_disruption) yield server_to_t1_io_test @@ -366,13 +379,14 @@ def server_to_t1_io_test(activehost, tor_vlan_port=None, @pytest.fixture -def send_soc_to_t1_with_action(duthosts, ptfhost, ptfadapter, tbinfo, cable_type, vmhost, save_pcap): # noqa F811 +def send_soc_to_t1_with_action(duthosts, ptfhost, ptfadapter, tbinfo, + cable_type, vmhost, save_pcap, skip_traffic_test=False): # noqa F811 arp_setup(ptfhost) def soc_to_t1_io_test(activehost, tor_vlan_port=None, delay=0, allowed_disruption=0, action=None, verify=False, send_interval=0.01, - stop_after=None): + stop_after=None, skip_traffic_test=False): tor_IO = run_test(duthosts, activehost, ptfhost, ptfadapter, vmhost, action, tbinfo, tor_vlan_port, send_interval, @@ -382,6 +396,9 @@ def soc_to_t1_io_test(activehost, tor_vlan_port=None, if delay and not allowed_disruption: allowed_disruption = 1 + if skip_traffic_test is True: + logging.info("Skipping traffic test") + return return verify_and_report(tor_IO, verify, delay, allowed_disruption) yield soc_to_t1_io_test @@ -390,13 +407,14 @@ def soc_to_t1_io_test(activehost, tor_vlan_port=None, @pytest.fixture -def send_t1_to_soc_with_action(duthosts, ptfhost, ptfadapter, tbinfo, cable_type, vmhost, save_pcap): # noqa F811 +def send_t1_to_soc_with_action(duthosts, ptfhost, ptfadapter, tbinfo, + cable_type, vmhost, save_pcap, skip_traffic_test=False): # noqa F811 arp_setup(ptfhost) def t1_to_soc_io_test(activehost, tor_vlan_port=None, delay=0, allowed_disruption=0, action=None, verify=False, send_interval=0.01, - stop_after=None): + stop_after=None, skip_traffic_test=False): tor_IO = run_test(duthosts, activehost, ptfhost, ptfadapter, vmhost, action, tbinfo, tor_vlan_port, send_interval, @@ -408,6 +426,9 @@ def t1_to_soc_io_test(activehost, tor_vlan_port=None, if delay and not allowed_disruption: allowed_disruption = 1 + if skip_traffic_test is True: + logging.info("Skipping traffic test") + return return verify_and_report(tor_IO, verify, delay, allowed_disruption) yield t1_to_soc_io_test @@ -432,13 +453,14 @@ def _select_test_mux_ports(cable_type, count): @pytest.fixture -def send_server_to_server_with_action(duthosts, ptfhost, ptfadapter, tbinfo, cable_type, vmhost, save_pcap): # noqa F811 +def send_server_to_server_with_action(duthosts, ptfhost, ptfadapter, tbinfo, + cable_type, vmhost, save_pcap, skip_traffic_test=False): # noqa F811 arp_setup(ptfhost) def server_to_server_io_test(activehost, test_mux_ports, delay=0, allowed_disruption=0, action=None, - verify=False, send_interval=0.01, stop_after=None): + verify=False, send_interval=0.01, stop_after=None, skip_traffic_test=False): tor_IO = run_test(duthosts, activehost, ptfhost, ptfadapter, vmhost, action, tbinfo, test_mux_ports, send_interval, traffic_direction="server_to_server", stop_after=stop_after, @@ -449,6 +471,9 @@ def server_to_server_io_test(activehost, test_mux_ports, delay=0, if delay and not allowed_disruption: allowed_disruption = 1 + if skip_traffic_test is True: + logging.info("Skipping traffic test") + return return verify_and_report(tor_IO, verify, delay, allowed_disruption) yield server_to_server_io_test diff --git a/tests/common/dualtor/dual_tor_utils.py b/tests/common/dualtor/dual_tor_utils.py index 7cd31b9436..4bea108f43 100644 --- a/tests/common/dualtor/dual_tor_utils.py +++ b/tests/common/dualtor/dual_tor_utils.py @@ -831,7 +831,7 @@ def mux_cable_server_ip(dut): def check_tunnel_balance(ptfhost, standby_tor_mac, vlan_mac, active_tor_ip, standby_tor_ip, selected_port, target_server_ip, target_server_ipv6, target_server_port, ptf_portchannel_indices, - completeness_level, check_ipv6=False): + completeness_level, check_ipv6=False, skip_traffic_test=False): """ Function for testing traffic distribution among all avtive T1. A test script will be running on ptf to generate traffic to standby interface, and the traffic will be forwarded to @@ -849,7 +849,9 @@ def check_tunnel_balance(ptfhost, standby_tor_mac, vlan_mac, active_tor_ip, Returns: None. """ - + if skip_traffic_test is True: + logging.info("Skip checking tunnel balance due to traffic test was skipped") + return HASH_KEYS = ["src-port", "dst-port", "src-ip"] params = { "server_ip": target_server_ip, @@ -1104,7 +1106,7 @@ def check_nexthops_balance(rand_selected_dut, ptfadapter, dst_server_addr, pc)) -def check_nexthops_single_uplink(portchannel_ports, port_packet_count, expect_packet_num): +def check_nexthops_single_uplink(portchannel_ports, port_packet_count, expect_packet_num, skip_traffic_test=False): for pc, intfs in portchannel_ports.items(): count = 0 # Collect the packets count within a single portchannel @@ -1113,13 +1115,16 @@ def check_nexthops_single_uplink(portchannel_ports, port_packet_count, expect_pa count = count + port_packet_count.get(uplink_int, 0) logging.info("Packets received on portchannel {}: {}".format(pc, count)) + if skip_traffic_test is True: + logging.info("Skip checking single uplink balance due to traffic test was skipped") + continue if count > 0 and count != expect_packet_num: pytest.fail("Packets not sent up single standby port {}".format(pc)) # verify nexthops are only sent to single active or standby mux def check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_addr, - tbinfo, downlink_ints): + tbinfo, downlink_ints, skip_traffic_test=False): HASH_KEYS = ["src-port", "dst-port", "src-ip"] expect_packet_num = 1000 expect_packet_num_high = expect_packet_num * (0.90) @@ -1134,6 +1139,9 @@ def check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_add port_packet_count = dict() packets_to_send = generate_hashed_packet_to_server(ptfadapter, rand_selected_dut, HASH_KEYS, dst_server_addr, expect_packet_num) + if skip_traffic_test is True: + logging.info("Skip checking single downlink balance due to traffic test was skipped") + return for send_packet, exp_pkt, exp_tunnel_pkt in packets_to_send: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), send_packet, count=1) # expect multi-mux nexthops to focus packets to one downlink @@ -1155,10 +1163,11 @@ def check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_add if len(downlink_ints) == 0: # All nexthops are now connected to standby mux, and the packets will be sent towards a single portchanel int # Check if uplink distribution is towards a single portchannel - check_nexthops_single_uplink(portchannel_ports, port_packet_count, expect_packet_num) + check_nexthops_single_uplink(portchannel_ports, port_packet_count, expect_packet_num, skip_traffic_test) -def verify_upstream_traffic(host, ptfadapter, tbinfo, itfs, server_ip, pkt_num=100, drop=False): +def verify_upstream_traffic(host, ptfadapter, tbinfo, itfs, server_ip, + pkt_num=100, drop=False, skip_traffic_test=False): """ @summary: Helper function for verifying upstream packets @param host: The dut host @@ -1211,6 +1220,9 @@ def verify_upstream_traffic(host, ptfadapter, tbinfo, itfs, server_ip, pkt_num=1 logger.info("Verifying upstream traffic. packet number = {} interface = {} \ server_ip = {} expect_drop = {}".format(pkt_num, itfs, server_ip, drop)) + if skip_traffic_test is True: + logger.info("Skip verifying upstream traffic due to traffic test was skipped") + return for i in range(0, pkt_num): ptfadapter.dataplane.flush() testutils.send(ptfadapter, tx_port, pkt, count=1) @@ -1271,7 +1283,7 @@ def _get_iface_ip(mg_facts, ifacename): res['target_server_ipv6'] = servers[random_server_iface]['server_ipv6'].split('/')[0] res['target_server_port'] = standby_tor_mg_facts['minigraph_ptf_indices'][random_server_iface] - normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough' + normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug' res['completeness_level'] = normalize_level logger.debug("dualtor info is generated {}".format(res)) @@ -1879,3 +1891,4 @@ def disable_timed_oscillation_active_standby(duthosts, tbinfo): for duthost in duthosts: duthost.shell('sonic-db-cli CONFIG_DB HSET "MUX_LINKMGR|TIMED_OSCILLATION" "oscillation_enabled" "false"') + duthost.shell("config save -y") diff --git a/tests/common/dualtor/mux_simulator_control.py b/tests/common/dualtor/mux_simulator_control.py index abf187f75f..ce0701d81c 100644 --- a/tests/common/dualtor/mux_simulator_control.py +++ b/tests/common/dualtor/mux_simulator_control.py @@ -176,7 +176,7 @@ def _post(server_url, data): """ try: session = Session() - if "allowed_methods" in inspect.getargspec(Retry).args: + if "allowed_methods" in inspect.signature(Retry).parameters: retry = Retry(total=3, connect=3, backoff_factor=1, allowed_methods=frozenset(['GET', 'POST']), status_forcelist=[x for x in requests.status_codes._codes if x != 200]) @@ -585,8 +585,7 @@ def _check_toggle_done(duthosts, target_dut_hostname, probe=False): data['active_side'] )) _post(mux_server_url, data) - time.sleep(5) - if _check_toggle_done(duthosts, target_dut_hostname): + if utilities.wait_until(15, 5, 0, _check_toggle_done, duthosts, target_dut_hostname, probe=True): is_toggle_done = True break diff --git a/tests/common/dualtor/server_traffic_utils.py b/tests/common/dualtor/server_traffic_utils.py index bcd4162208..33e0293b3c 100644 --- a/tests/common/dualtor/server_traffic_utils.py +++ b/tests/common/dualtor/server_traffic_utils.py @@ -24,22 +24,28 @@ def dump_intf_packets(ansible_host, iface, pcap_save_path, dumped_packets, @pcap_filter: pcap filter used by tcpdump. @cleanup_pcap: True to remove packet capture file. """ - + nohup_output = "/tmp/nohup.out" start_pcap = "tcpdump --immediate-mode -i %s -w %s" % (iface, pcap_save_path) if pcap_filter: start_pcap += (" " + pcap_filter) - start_pcap = "nohup %s > /dev/null 2>&1 & echo $!" % start_pcap + start_pcap = "nohup %s > %s 2>&1 & echo $!" % (start_pcap, nohup_output) + ansible_host.file(path=nohup_output, state="absent") pid = ansible_host.shell(start_pcap)["stdout"] # sleep to let tcpdump starts to capture time.sleep(1) try: yield finally: + # wait for tcpdump to finish + time.sleep(2) ansible_host.shell("kill -s 2 %s" % pid) with tempfile.NamedTemporaryFile() as temp_pcap: ansible_host.fetch(src=pcap_save_path, dest=temp_pcap.name, flat=True) packets = sniff(offline=temp_pcap.name) dumped_packets.extend(packets) + # show the tcpdump run output for debug + ansible_host.shell("cat %s" % nohup_output) + ansible_host.file(path=nohup_output, state="absent") if cleanup_pcap: ansible_host.file(path=pcap_save_path, state="absent") @@ -50,7 +56,8 @@ class ServerTrafficMonitor(object): VLAN_INTERFACE_TEMPLATE = "{external_port}.{vlan_id}" def __init__(self, duthost, ptfhost, vmhost, tbinfo, dut_iface, - conn_graph_facts, exp_pkt, existing=True, is_mocked=False): + conn_graph_facts, exp_pkt, existing=True, is_mocked=False, + skip_traffic_test=False): """ @summary: Initialize the monitor. @@ -75,6 +82,7 @@ def __init__(self, duthost, ptfhost, vmhost, tbinfo, dut_iface, self.conn_graph_facts = conn_graph_facts self.captured_packets = [] self.matched_packets = [] + self.skip_traffic_test = skip_traffic_test if is_mocked: mg_facts = self.duthost.get_extended_minigraph_facts(self.tbinfo) ptf_iface = "eth%s" % mg_facts['minigraph_ptf_indices'][self.dut_iface] @@ -120,6 +128,9 @@ def __exit__(self, exc_type, exc_value, traceback): logging.info("the expected packet:\n%s", str(self.exp_pkt)) self.matched_packets = [p for p in self.captured_packets if match_exp_pkt(self.exp_pkt, p)] logging.info("received %d matched packets", len(self.matched_packets)) + if self.skip_traffic_test is True: + logging.info("Skip matched_packets verify due to traffic test was skipped.") + return if self.matched_packets: logging.info( "display the most recent matched captured packet:\n%s", diff --git a/tests/common/dualtor/tunnel_traffic_utils.py b/tests/common/dualtor/tunnel_traffic_utils.py index 790a28f4fe..7e59f3c2d8 100644 --- a/tests/common/dualtor/tunnel_traffic_utils.py +++ b/tests/common/dualtor/tunnel_traffic_utils.py @@ -250,7 +250,7 @@ def _disassemble_ip_tos(tos): return " ,".join(check_res) def __init__(self, standby_tor, active_tor=None, existing=True, inner_packet=None, - check_items=("ttl", "tos", "queue"), packet_count=10): + check_items=("ttl", "tos", "queue"), packet_count=10, skip_traffic_test=False): """ Init the tunnel traffic monitor. @@ -262,6 +262,7 @@ def __init__(self, standby_tor, active_tor=None, existing=True, inner_packet=Non self.listen_ports = sorted(self._get_t1_ptf_port_indexes(standby_tor, tbinfo)) self.ptfadapter = ptfadapter self.packet_count = packet_count + self.skip_traffic_test = skip_traffic_test standby_tor_cfg_facts = self.standby_tor.config_facts( host=self.standby_tor.hostname, source="running" @@ -278,9 +279,7 @@ def __init__(self, standby_tor, active_tor=None, existing=True, inner_packet=Non ][0] self.existing = existing - self.inner_packet = None - if self.existing: - self.inner_packet = inner_packet + self.inner_packet = inner_packet self.exp_pkt = self._build_tunnel_packet(self.standby_tor_lo_addr, self.active_tor_lo_addr, inner_packet=self.inner_packet) self.rec_pkt = None @@ -294,6 +293,9 @@ def __enter__(self): def __exit__(self, *exc_info): if exc_info[0]: return + if self.skip_traffic_test is True: + logging.info("Skip tunnel traffic verify due to traffic test was skipped.") + return try: port_index, rec_pkt = testutils.verify_packet_any_port( ptfadapter, diff --git a/tests/common/fixtures/conn_graph_facts.py b/tests/common/fixtures/conn_graph_facts.py old mode 100644 new mode 100755 index 593354be83..e7d6cdd38e --- a/tests/common/fixtures/conn_graph_facts.py +++ b/tests/common/fixtures/conn_graph_facts.py @@ -12,10 +12,12 @@ def conn_graph_facts(duthosts, localhost): @pytest.fixture(scope="module") -def fanout_graph_facts(localhost, duthosts, rand_one_dut_hostname, conn_graph_facts): - duthost = duthosts[rand_one_dut_hostname] +def fanout_graph_facts(localhost, duthosts, rand_one_tgen_dut_hostname, conn_graph_facts): + duthost = duthosts[rand_one_tgen_dut_hostname] facts = dict() dev_conn = conn_graph_facts.get('device_conn', {}) + if not dev_conn: + return facts for _, val in list(dev_conn[duthost.hostname].items()): fanout = val["peerdevice"] if fanout not in facts: @@ -23,11 +25,34 @@ def fanout_graph_facts(localhost, duthosts, rand_one_dut_hostname, conn_graph_fa return facts +@pytest.fixture(scope="module") +def fanout_graph_facts_multidut(localhost, duthosts, conn_graph_facts): + facts = dict() + dev_conn = conn_graph_facts.get('device_conn', {}) + if not dev_conn: + return facts + + fanout_set = set() + for duthost in duthosts: + for _, val in list(dev_conn[duthost.hostname].items()): + fanout_set.add(val["peerdevice"]) + + # Only take IXIA/SNAPPI testers into fanout_facts + for fanout in fanout_set: + fanout_data = {k: v[fanout] for k, v in list(get_graph_facts(duthost, localhost, fanout).items())} + if fanout_data['device_info']['HwSku'] in ('SNAPPI-tester', 'IXIA-tester'): + facts[fanout] = fanout_data + + return facts + + @pytest.fixture(scope="module") def enum_fanout_graph_facts(localhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, conn_graph_facts): duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] facts = dict() dev_conn = conn_graph_facts.get('device_conn', {}) + if not dev_conn: + return facts for _, val in list(dev_conn[duthost.hostname].items()): fanout = val["peerdevice"] if fanout not in facts: diff --git a/tests/common/fixtures/consistency_checker/consistency_checker.py b/tests/common/fixtures/consistency_checker/consistency_checker.py new file mode 100644 index 0000000000..44044027b2 --- /dev/null +++ b/tests/common/fixtures/consistency_checker/consistency_checker.py @@ -0,0 +1,346 @@ +import pytest +import logging +import json +import os +import datetime +from collections import defaultdict +from tests.common.fixtures.consistency_checker.constants import SUPPORTED_PLATFORMS_AND_VERSIONS + +logger = logging.getLogger(__name__) + +SYNCD_CONTAINER = "syncd" +QUERY_ASIC_SCRIPT = "query-asic.py" +QUERY_ASIC_PARSER = "parser.py" +LIBSAIREDIS_DEB = "libsairedis.deb" +PYTHON3_PYSAIREDIS_DEB = "python3-pysairedis.deb" +DUT_DST_PATH_HOST = "/tmp/consistency-checker" +DUT_DST_PATH_CONTAINER = "/consistency-checker" + +QUERY_ASIC_PATH_SRC = os.path.dirname(__file__) + "/query-asic" +QUERY_ASIC_SCRIPT_PATH_SRC = QUERY_ASIC_PATH_SRC + "/" + QUERY_ASIC_SCRIPT +QUERY_ASIC_PARSER_PATH_SRC = QUERY_ASIC_PATH_SRC + "/" + QUERY_ASIC_PARSER +QUERY_ASIC_SCRIPT_PATH_DST_HOST = DUT_DST_PATH_HOST + "/" + QUERY_ASIC_SCRIPT +QUERY_ASIC_PARSER_PATH_DST_HOST = DUT_DST_PATH_HOST + "/" + QUERY_ASIC_PARSER +QUERY_ASIC_SCRIPT_PATH_DST_CONTAINER = DUT_DST_PATH_CONTAINER + "/" + QUERY_ASIC_SCRIPT + +LIBSAIREDIS_TEMP = "libsairedis-temp" + + +class ConsistencyChecker: + + def __init__(self, duthost, libsairedis_download_url=None, python3_pysairedis_download_url=None): + """ + If the libsairedis_download_url and python3_pysairedis_download_url are provided, then these artifacts + are downloaded and installed on the DUT, otherwise it's assumed that the environment is already setup + for the consistency checker. + """ + self._duthost = duthost + self._libsairedis_download_url = libsairedis_download_url + self._python3_pysairedis_download_url = python3_pysairedis_download_url + + def __enter__(self): + logger.info("Initializing consistency checker on dut...") + + self._duthost.file(path=DUT_DST_PATH_HOST, state="directory") + self._duthost.copy(src=QUERY_ASIC_SCRIPT_PATH_SRC, dest=QUERY_ASIC_SCRIPT_PATH_DST_HOST) + self._duthost.copy(src=QUERY_ASIC_PARSER_PATH_SRC, dest=QUERY_ASIC_PARSER_PATH_DST_HOST) + + if self._libsairedis_download_url is not None: + self._duthost.command(f"curl -o {DUT_DST_PATH_HOST}/{LIBSAIREDIS_DEB} {self._libsairedis_download_url}") + if self._python3_pysairedis_download_url is not None: + self._duthost.command( + f"curl -o {DUT_DST_PATH_HOST}/{PYTHON3_PYSAIREDIS_DEB} {self._python3_pysairedis_download_url}") + + # Move everything into syncd container + self._duthost.shell(( + f"docker cp {DUT_DST_PATH_HOST} {SYNCD_CONTAINER}:/ && " + f"rm -rf {DUT_DST_PATH_HOST}" + )) + + if self._python3_pysairedis_download_url is not None: + # Install python3-sairedis in syncd container + self._duthost.shell((f"docker exec {SYNCD_CONTAINER} bash -c " + f"'cd {DUT_DST_PATH_CONTAINER} && " + f"dpkg --install {DUT_DST_PATH_CONTAINER}/{PYTHON3_PYSAIREDIS_DEB}'")) + + if self._libsairedis_download_url is not None: + # Extract the libsairedis deb to be used by the query script + self._duthost.shell((f"docker exec {SYNCD_CONTAINER} bash -c " + f"'cd {DUT_DST_PATH_CONTAINER} && " + f"dpkg --extract {DUT_DST_PATH_CONTAINER}/{LIBSAIREDIS_DEB} {LIBSAIREDIS_TEMP}'")) + + logger.info("Consistency checker setup complete.") + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + logger.info("Cleaning up consistency checker on dut...") + + if self._python3_pysairedis_download_url is not None: + # Uninstall python3-sairedis in syncd container + self._duthost.command(f"docker exec {SYNCD_CONTAINER} dpkg --remove python3-pysairedis") + + # Remove all the files from the syncd container + self._duthost.command(f"docker exec {SYNCD_CONTAINER} rm -rf {DUT_DST_PATH_CONTAINER}") + + # NOTE: If consistency checker is used to do write operations (currently it's read-only), then syncd should be + # restarted or minigraph reloaded re-align the ASIC_DB and ASIC state. + + logger.info("Consistency checker cleanup complete.") + + def get_db_and_asic_peers(self, keys=["*"]) -> dict: + """ + Bulk query ASIC data that exists in the ASIC_DB. + + :param keys: Optional list of glob search strings that correspond to the --key arg of sonic-db-dump. + sonic-db-dump doesn't take multiple keys, so a list is passed in to support multiple + keys at the API level. + :return: Dictionary containing the queried ASIC data. + + Example return value: + { + "ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_PROFILE:oid:0x1900000000154f": { + "SAI_BUFFER_PROFILE_ATTR_POOL_ID": { + "dbValue": "oid:0x1800000000154a", + "asicValue": "oid:0x1800000000154a", + "asicQuerySuccess": True + }, + "SAI_BUFFER_PROFILE_ATTR_SHARED_DYNAMIC_TH": { + "dbValue": "0", + "asicValue": -1, + "asicQuerySuccess": False, + "asicQueryErrorMsg": "Failed to query attribute value" + }, + "SAI_BUFFER_PROFILE_ATTR_THRESHOLD_MODE": { + "dbValue": "SAI_BUFFER_PROFILE_THRESHOLD_MODE_DYNAMIC", + "asicValue": "SAI_BUFFER_PROFILE_THRESHOLD_MODE_DYNAMIC", + "asicQuerySuccess": True + }, + ... + }, + ... + } + """ + + db_attributes = self._get_db_attributes(keys) + asic_attributes = self._get_asic_attributes_from_db_results(db_attributes) + + results = defaultdict(dict) + + for object in db_attributes: + db_object = db_attributes[object] + asic_object = asic_attributes[object] + + for attr in db_object["value"].keys(): + db_value = db_object["value"][attr] + asic_value = asic_object[attr]["asicValue"] + + if db_value.startswith("oid:0x"): + # Convert the asic one to the same format + try: + asic_value = f"oid:{hex(int(asic_value))}" + except Exception: + # keep the value as is + pass + + results[object][attr] = { + "dbValue": db_value, + "asicValue": asic_value, + "asicQuerySuccess": asic_object[attr]["success"] + } + + if not asic_object[attr]["success"]: + results[object][attr]["asicQueryErrorMsg"] = asic_object[attr]["error"] + + return dict(results) + + def check_consistency(self, keys=["*"]) -> dict: + """ + Get the out-of-sync ASIC_DB and ASIC attributes. Differences are indicative of an error state. + Same arg style as the get_objects function but returns a list of objects that don't match or couldn't + be queried from the ASIC. If it was successfully queried and has a matching value, then it won't be + included in the response. + + :param keys: Optional list of glob search strings that correspond to the --key arg of sonic-db-dump. + sonic-db-dump doesn't take multiple keys, so a list is passed in to support multiple + keys at the API level. + :return: Dictionary containing the out-of-sync ASIC_DB and ASIC attributes. + + Example return val (matching): + {} + + Example return val (mismatch): + { + "ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_PROFILE:oid:0x1900000000154f": { + "attributes": { + "SAI_BUFFER_PROFILE_ATTR_SHARED_DYNAMIC_TH": { + "dbValue": "0", + "asicValue": -1, + }, + "SAI_BUFFER_PROFILE_ATTR_THRESHOLD_MODE": { + "dbValue": "SAI_BUFFER_PROFILE_THRESHOLD_MODE_DYNAMIC", + "asicValue": "SAI_BUFFER_PROFILE_THRESHOLD_MODE_STATIC" + }, + ... + }, + "failedToQueryAsic": [ + {"SAI_BUFFER_PROFILE_ATTR_SHARED_DYNAMIC_TH": "Failed to query attribute value"} + ], + "mismatchedAttributes": ["SAI_BUFFER_PROFILE_ATTR_THRESHOLD_MODE"] + }, + ... + } + """ + + db_attributes = self._get_db_attributes(keys) + asic_attributes = self._get_asic_attributes_from_db_results(db_attributes) + + inconsistencies = defaultdict(lambda: { + "attributes": {}, + "failedToQueryAsic": [], + "mismatchedAttributes": [] + }) + + for object in db_attributes: + db_object = db_attributes[object] + asic_object = asic_attributes[object] + + for attr in db_object["value"].keys(): + db_value = db_object["value"][attr] + asic_value = asic_object[attr]["asicValue"] + asic_query_success = asic_object[attr]["success"] + + if asic_query_success and db_value == asic_value: + continue + + if db_value.startswith("oid:0x"): + # Convert the asic one to the same format + try: + asic_value = f"oid:{hex(int(asic_value))}" + if db_value == asic_value: + continue + except Exception: + # true error - let below code handle it + pass + + inconsistencies[object]["attributes"][attr] = { + "dbValue": db_value, + "asicValue": asic_value + } + + if asic_query_success: + inconsistencies[object]["mismatchedAttributes"].append(attr) + else: + inconsistencies[object]["failedToQueryAsic"].append({attr: asic_object[attr]["error"]}) + + return dict(inconsistencies) + + def _get_db_attributes(self, keys: list) -> dict: + """ + Fetchs and merges the attributes of the objects returned by the search key from the DB. + """ + db_attributes = {} + for key in keys: + result = self._duthost.command(f"sonic-db-dump -k '{key}' -n ASIC_DB") + if result['rc'] != 0: + raise Exception((f"Failed to fetch attributes for key '{key}' from ASIC_DB. " + f"Return code: {result['rc']}, stdout: {result['stdout']}, " + f"stderr: {result['stderr']}")) + + query_result = json.loads(result['stdout']) + db_attributes.update(query_result) + + return db_attributes + + def _get_asic_attributes_from_db_results(self, db_attributes: dict) -> dict: + """ + Queries the ASIC for the attributes of the objects in db_attributes which are the results + from the ASIC DB query. + + Example return value: + { + "ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_POOL:oid:0x18000000000628": { + "SAI_BUFFER_POOL_ATTR_THRESHOLD_MODE": { + "asicValue": "SAI_BUFFER_POOL_THRESHOLD_MODE_STATIC", + "success": true + }, + "SAI_BUFFER_POOL_ATTR_SIZE": { + "success" false, + "error": "Failed to query attribute value" + }, + "SAI_BUFFER_POOL_ATTR_TYPE": { + "asicValue": "SAI_BUFFER_POOL_TYPE_EGRESS", + "success": true + } + }, + ... + } + """ + # Map to format expected by the query-asic.py + asic_query = {k: list(v["value"].keys()) for k, v in db_attributes.items()} + asic_query_input_filename = f"query-input-{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}.json" + with open(f"/tmp/{asic_query_input_filename}", 'w') as f: + json.dump(asic_query, f) + + # Copy the input file to the syncd container + self._duthost.copy(src=f"/tmp/{asic_query_input_filename}", dest=f"/tmp/{asic_query_input_filename}") + self._duthost.shell((f"docker cp /tmp/{asic_query_input_filename} " + f"{SYNCD_CONTAINER}:{DUT_DST_PATH_CONTAINER} && " + f"rm /tmp/{asic_query_input_filename}")) + + ld_lib_path_arg = f"LD_LIBRARY_PATH={LIBSAIREDIS_TEMP}/usr/lib/x86_64-linux-gnu"\ + if self._libsairedis_download_url is not None else "" + + res = self._duthost.shell((f"docker exec {SYNCD_CONTAINER} bash -c " + f"'cd {DUT_DST_PATH_CONTAINER} && " + f"{ld_lib_path_arg} python3 {QUERY_ASIC_SCRIPT_PATH_DST_CONTAINER} " + f"--input {asic_query_input_filename}'")) + if res['rc'] != 0: + raise Exception((f"Failed to query ASIC attributes. Return code: {res['rc']}, stdout: {res['stdout']}, " + f"stderr: {res['stderr']}")) + asic_results = json.loads(res['stdout']) + + return asic_results + + +class ConsistencyCheckerProvider: + def is_consistency_check_supported(self, dut) -> bool: + """ + Checks if the provided DUT is supported for consistency checking. + + :param dut: SonicHost object + :return bool: True if the DUT is supported, False otherwise + """ + + platform = dut.facts['platform'] + if platform not in SUPPORTED_PLATFORMS_AND_VERSIONS: + return False + + current_version = dut.image_facts()['ansible_facts']['ansible_image_facts']['current'] + supported_versions = SUPPORTED_PLATFORMS_AND_VERSIONS[platform] + if any(v in current_version for v in supported_versions): + return True + + return False + + def get_consistency_checker(self, dut, libsairedis_download_url=None, + python3_pysairedis_download_url=None) -> ConsistencyChecker: + """ + Get a new instance of the ConsistencyChecker class. + + :param dut: SonicHost object + :param libsairedis_download_url: Optional URL that the consistency checker should use to download the + libsairedis deb + :param python3_pysairedis_download_url: Optional URL that the consistency checker should use to + download the python3-pysairedis deb + :return ConsistencyChecker: New instance of the ConsistencyChecker class + """ + return ConsistencyChecker(dut, libsairedis_download_url, python3_pysairedis_download_url) + + +@pytest.fixture +def consistency_checker_provider(): + """ + Fixture that provides the ConsistencyCheckerProvider class. + """ + return ConsistencyCheckerProvider() diff --git a/tests/common/fixtures/consistency_checker/constants.py b/tests/common/fixtures/consistency_checker/constants.py new file mode 100644 index 0000000000..f3a570fc06 --- /dev/null +++ b/tests/common/fixtures/consistency_checker/constants.py @@ -0,0 +1,6 @@ + +# The list of platforms and versions that have been tested to work with the consistency checker +SUPPORTED_PLATFORMS_AND_VERSIONS = { + "x86_64-arista_7060_cx32s": ["202305", "202311"], + "x86_64-arista_7260cx3_64": ["202305", "202311"], +} diff --git a/tests/common/fixtures/consistency_checker/query-asic/parser.py b/tests/common/fixtures/consistency_checker/query-asic/parser.py new file mode 100644 index 0000000000..076d4f0c01 --- /dev/null +++ b/tests/common/fixtures/consistency_checker/query-asic/parser.py @@ -0,0 +1,234 @@ +""" +This module contains utilities for parsing the primitive values out of the ASIC query results. +""" +import ctypes +from sairedis import pysairedis + + +def mac_address_str_from_swig_uint8_t_arr(swig_uint8_p) -> str: + """ + Given a swig pointer to a uint8_t array, return the MAC address string representation + + :param swig_uint8_p: The swig pointer to the uint8_t array + :return: The MAC address string representation + """ + pointer = ctypes.cast(swig_uint8_p.__int__(), ctypes.POINTER(ctypes.c_uint8)) + octets = [pointer[i] for i in range(6)] + fmtd_mac_address = ":".join([f"{octet:02X}" for octet in octets]) + return fmtd_mac_address + + +def extract_attr_value(attr_metadata, attr): + """ + Extract the value from the attribute based on the attribute metadata + + :param attr_metadata: The attribute metadata + :param attr: The attribute + :return: The value of the attribute + """ + + if attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_BOOL: + attr_value = attr.value.booldata + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT8: + attr_value = attr.value.u8 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT8: + attr_value = attr.value.s8 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT16: + attr_value = attr.value.u16 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT16: + attr_value = attr.value.s16 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT32: + attr_value = attr.value.u32 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT32: + attr_value = attr.value.s32 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT64: + attr_value = attr.value.u64 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT64: + attr_value = attr.value.s64 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_OBJECT_ID: + attr_value = attr.value.oid + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT32_LIST: + attr_value = attr.value.u32list + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_MAC: + attr_value = mac_address_str_from_swig_uint8_t_arr(attr.value.mac) + # *************************************************************************** + # NOTE: GPT generated attributes below, likely to be incomplete and/or + # need additional processing. + # *************************************************************************** + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_IP_ADDRESS: + attr_value = attr.value.ipaddr + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_LATCH_STATUS: + attr_value = attr.value.latch + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT8_LIST: + attr_value = attr.value.s8list + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_OBJECT_LIST: + attr_value = attr.value.aclactiondataobjlist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_SYSTEM_PORT_CONFIG_LIST: + attr_value = attr.value.systemportconfiglist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_UINT8_LIST: + attr_value = attr.value.aclfielddatau8list + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT32_LIST: + attr_value = attr.value.u32list + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_NAT_ENTRY_DATA: + attr_value = attr.value.natentrydata + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT32_LIST: + attr_value = attr.value.s32list + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_FABRIC_PORT_REACHABILITY: + attr_value = attr.value.fabricportreachability + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_TLV_LIST: + attr_value = attr.value.tlvlist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT32: + attr_value = attr.value.u32 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT8_LIST: + attr_value = attr.value.u8list + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_IPV4: + attr_value = attr.value.aclfielddataipv4 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT16_RANGE_LIST: + attr_value = attr.value.u16rangelist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_CHAIN_LIST: + attr_value = attr.value.aclchainlist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_MACSEC_SCI: + attr_value = attr.value.aclfielddatamacsecsci + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT16: + attr_value = attr.value.s16 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_UINT16: + attr_value = attr.value.aclactiondatau16 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_IPV6: + attr_value = attr.value.aclactiondataipv6 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_IPV4: + attr_value = attr.value.ip4 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_UINT8: + attr_value = attr.value.aclactiondatau8 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_PORT_SNR_LIST: + attr_value = attr.value.portsnrlist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT16: + attr_value = attr.value.u16 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_BOOL: + attr_value = attr.value.booldata + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_IPV6: + attr_value = attr.value.aclfielddataipv6 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_BOOL: + attr_value = attr.value.aclactiondatabool + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_OBJECT_ID: + attr_value = attr.value.oid + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_UINT8: + attr_value = attr.value.aclfielddatau8 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_POINTER: + attr_value = attr.value.ptr + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_TIMESPEC: + attr_value = attr.value.timespec + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT16_LIST: + attr_value = attr.value.u16list + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT8: + attr_value = attr.value.u8 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_INT16: + attr_value = attr.value.aclfielddatas16 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_PORT_ERR_STATUS_LIST: + attr_value = attr.value.porterrstatuslist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_JSON: + attr_value = attr.value.json + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_UINT16: + attr_value = attr.value.aclfielddatau16 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_IP_ADDRESS: + attr_value = attr.value.aclactiondataipaddr + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_BOOL: + attr_value = attr.value.aclfielddatabool + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_RESOURCE_LIST: + attr_value = attr.value.aclresourcelist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_IP_PREFIX_LIST: + attr_value = attr.value.ipprefixlist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_VLAN_LIST: + attr_value = attr.value.vlanlist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_INT8: + attr_value = attr.value.aclactiondataint8 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_OBJECT_LIST: + attr_value = attr.value.objlist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_TWAMP_STATS_DATA: + attr_value = attr.value.twampstatsdata + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_MACSEC_SALT: + attr_value = attr.value.macsecsalt + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_IPV6: + attr_value = attr.value.ipv6 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_MAC: + attr_value = attr.value.aclfielddatamac + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_AUTH_KEY: + attr_value = attr.value.authkey + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_UINT32: + attr_value = attr.value.aclfielddatau32 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_MACSEC_SAK: + attr_value = attr.value.macsecsak + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT64: + attr_value = attr.value.s64 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_SYSTEM_PORT_CONFIG: + attr_value = attr.value.systemportconfig + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT32_RANGE: + attr_value = attr.value.s32range + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_OBJECT_ID: + attr_value = attr.value.aclactiondataobjid + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_MACSEC_SCI: + attr_value = attr.value.macsecsci + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT64: + attr_value = attr.value.u64 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_PRBS_RX_STATE: + attr_value = attr.value.prbsrxstate + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_INT32: + attr_value = attr.value.aclfielddatas32 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_INT32: + attr_value = attr.value.aclactiondatas32 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_CHARDATA: + attr_value = attr.value.chardata + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_SEGMENT_LIST: + attr_value = attr.value.segmentlist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT8: + attr_value = attr.value.s8 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_PORT_FREQUENCY_OFFSET_PPM_LIST: + attr_value = attr.value.portfreqoffsetppmlist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_MACSEC_AUTH_KEY: + attr_value = attr.value.macsecauthkey + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_MAP_LIST: + attr_value = attr.value.maplist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_OBJECT_ID: + attr_value = attr.value.aclfielddataobjid + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_UINT64: + attr_value = attr.value.aclfielddatau64 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_CAPABILITY: + attr_value = attr.value.aclcapability + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_QOS_MAP_LIST: + attr_value = attr.value.qosmaplist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ENCRYPT_KEY: + attr_value = attr.value.encryptkey + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_OBJECT_LIST: + attr_value = attr.value.aclfielddataobjlist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_IP_PREFIX: + attr_value = attr.value.ipprefix + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_PORT_EYE_VALUES_LIST: + attr_value = attr.value.porteyevalueslist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_MACSEC_SSCI: + attr_value = attr.value.macsecssci + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_UINT32: + attr_value = attr.value.aclactiondatau32 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_FIELD_DATA_INT8: + attr_value = attr.value.aclfielddataint8 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT16_LIST: + attr_value = attr.value.s16list + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_PORT_LANE_LATCH_STATUS_LIST: + attr_value = attr.value.portlanelatchstatuslist + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_INT32: + attr_value = attr.value.s32 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_IPV4: + attr_value = attr.value.aclactiondataipv4 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_MAC: + attr_value = attr.value.aclactiondatamac + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_ACL_ACTION_DATA_INT16: + attr_value = attr.value.aclactiondatas16 + elif attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_IP_ADDRESS_LIST: + attr_value = attr.value.ipaddlist + else: + raise NotImplementedError(f"Unsupported attribute value type: {attr_metadata.attrvaluetype}") + + if attr_metadata.isenum: + # Map to the string value if enum + enum_metadata = attr_metadata.enummetadata + attr_value = pysairedis.sai_metadata_get_enum_value_name(enum_metadata, attr_value) + + return attr_value diff --git a/tests/common/fixtures/consistency_checker/query-asic/query-asic.py b/tests/common/fixtures/consistency_checker/query-asic/query-asic.py new file mode 100644 index 0000000000..564a9aeed1 --- /dev/null +++ b/tests/common/fixtures/consistency_checker/query-asic/query-asic.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 + +import logging +import sys +import argparse +import json +from collections import defaultdict +from parser import extract_attr_value +from sairedis import pysairedis + +logger = logging.getLogger(__name__) + +# Results get written to stdout, so we want to log errors to stderr to avoid mixing them up +logger.addHandler(logging.StreamHandler(sys.stderr)) + +HELP_TEXT = """ +Query ASIC using the json provided in the --input file. The expected format is as follows: + { + "ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_POOL:oid:0x18000000000628": [ + "SAI_BUFFER_POOL_ATTR_THRESHOLD_MODE", + "SAI_BUFFER_POOL_ATTR_SIZE", + "SAI_BUFFER_POOL_ATTR_TYPE" + ], + ... + } + +The results will be printed to stdout, in the following format: + { + "ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_POOL:oid:0x18000000000628": { + "SAI_BUFFER_POOL_ATTR_THRESHOLD_MODE": { + "asicValue": "SAI_BUFFER_POOL_THRESHOLD_MODE_STATIC", + "success": true + }, + "SAI_BUFFER_POOL_ATTR_SIZE": { + "asicValue": null, + "success" false, + "error": "Failed to query attribute value" + }, + "SAI_BUFFER_POOL_ATTR_TYPE": { + "asicValue": "SAI_BUFFER_POOL_TYPE_EGRESS", + "success": true + } + }, + ... + } +""" + + +def load_input(input_file: argparse.FileType) -> dict: + """ + Read the opened input JSON file with contents like so: + { + "ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_POOL:oid:0x18000000000628": [ + "SAI_BUFFER_POOL_ATTR_THRESHOLD_MODE", + "SAI_BUFFER_POOL_ATTR_SIZE", + "SAI_BUFFER_POOL_ATTR_TYPE" + ], + ... + } + + Closes the file after reading. + + :param input_file: Path to the input JSON file + :return: The loaded JSON data + """ + with input_file as f: + return json.load(f) + + +def get_numeric_oid_from_label(oid_label: str) -> int: + """ + From a label like "ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_POOL:oid:0x18000000000628", + extracts and returns the numeric oid part 0x18000000000628. + + NOTE: There's also another form like so: + ASIC_STATE:SAI_OBJECT_TYPE_FDB_ENTRY:{\"bvid\":\"oid:0x260000000008da\",\"mac\":\"98:03:9B:03:22:14\",\"switch_id\":\"oid:0x21000000000000\"} + which isn't currently supported. + + :param oid_label: The label to extract the oid from + :return: The numeric oid value + """ + # Extract the value segment e.g. oid:0x18000000000628 + value_segment = oid_label.split(":", 2)[2] + if not value_segment.startswith("oid:"): + raise NotImplementedError(f"Unsupported oid format: {oid_label}") + + oid_value = value_segment.split(":", 1)[1] + return int(oid_value, 16) + + +def lookup_attribute_value_in_pysairedis(attr: str) -> int: + """ + Given an attribute name like "SAI_BUFFER_POOL_ATTR_THRESHOLD_MODE", return the corresponding + attribute oid from pysairedis. + + :param attr: The attribute name + :return: The attribute oid value + """ + return getattr(pysairedis, attr) + + +# Generate a one-time lookup table for all SAI status codes in the query results +sai_status_map = { + "single": {}, + "range": defaultdict(list), +} +for key, value in vars(pysairedis).items(): + if key.startswith("SAI_STATUS_"): + key = key.replace("SAI_STATUS_", "") + if key.endswith("_0") or key.endswith("_MAX"): + # Range + range_key = key[:-2] if key.endswith("_0") else key[:-4] + sai_status_map["range"][range_key].append(value) # Add to end of list + sai_status_map["range"][range_key].sort() # Only ever 0-2 elements, so this won't be expensive + else: + # Single value + sai_status_map["single"][value] = key + + +def map_sai_status_to_str(status_code: int) -> str: + """ + Given a SAI status code e.g. -196608, return the string representation e.g. SAI_STATUS_ATTR_NOT_SUPPORTED + + :param status_code: The numeric SAI status code + :return: The string representation of the status code + """ + if status_code in sai_status_map["single"]: + return sai_status_map["single"][status_code] + + # See if it falls in range of any status + for status_str, status_code_range in sai_status_map["range"].items(): + if status_code_range[0] <= status_code and status_code <= status_code_range[1]: + return status_str + + return "UNKNOWN_SAI_STATUS" + + +def get_attribute_value_from_asic(oid, attribute_oid): + """ + Given an oid and attribute_oid, query the ASIC for the attribute value. The attribute value + is transformed to match the format of the ASIC_DB. + + :param oid: The oid of the object to query + :param attribute_oid: The attribute oid of the object to query + :return: The attribute value from the ASIC in the format of the ASIC_DB + """ + + oid_type = pysairedis.sai_object_type_query(oid) + object_type_name = pysairedis.sai_metadata_get_object_type_name(oid_type).replace("SAI_OBJECT_TYPE_", "") + class_name = object_type_name.lower() + # Handle special cases where the class name is different + if object_type_name in ["BUFFER_POOL", "BUFFER_PROFILE"]: + class_name = "buffer" + api = getattr(pysairedis, f"sai_{class_name}_api_t")() + status = getattr(pysairedis, f"sai_get_{class_name}_api")(api) + assert status == pysairedis.SAI_STATUS_SUCCESS, (f"Failed to get sai API {api}. " + f"Status: {map_sai_status_to_str(status)} ({status})") + + attr_metadata = pysairedis.sai_metadata_get_attr_metadata(oid_type, attribute_oid) + + attr = pysairedis.sai_attribute_t() + attr.id = attribute_oid + if attr_metadata.attrvaluetype == pysairedis.SAI_ATTR_VALUE_TYPE_UINT32_LIST: + # Extra initialization for reading into a list + attr.value.u32list.count = 32 + attr.value.u32list.list = pysairedis.new_uint32_t_arr(attr.value.u32list.count) + + # Read the attribute from the ASIC into attr + func_name = f"get_{object_type_name.lower()}_attribute" + status = getattr(api, func_name)(oid, 1, attr) + assert status == pysairedis.SAI_STATUS_SUCCESS, \ + (f"Failed to call SAI API {func_name} for oid {oid} and attribute " + f"{attribute_oid}. Status: {map_sai_status_to_str(status)} ({status})") + + # Extract the attribute value from attr + attr_value = extract_attr_value(attr_metadata, attr) + return attr_value + + +def query_asic_objects(query_objects) -> dict: + """ + Query the ASIC for the attributes of the objects provided in deserialized JSON input file format. + + :param query_objects: The deserialized JSON input file format + :return: The deserialized JSON output format + """ + + results = defaultdict(dict) + + for oid_label_key, attributes in query_objects.items(): + try: + logger.debug(f"Querying ASIC for object key {oid_label_key}") + oid = get_numeric_oid_from_label(oid_label_key) + except Exception as e: + err_msg = f"Failed to extract oid from label '{oid_label_key}': {e}" + logger.warning(err_msg) + for attribute in attributes: + results[oid_label_key][attribute] = {"success": False, "error": err_msg, "asicValue": None} + continue + + for attribute in attributes: + try: + logger.debug(f"Querying ASIC object {oid_label_key} ({oid}) for attribute {attribute}") + attribute_oid = lookup_attribute_value_in_pysairedis(attribute) + asic_value = get_attribute_value_from_asic(oid, attribute_oid) + + # Convert to str to match how values are represented in ASIC_DB + if asic_value in [True, False]: + # ASIC_DB represents these as lowercase + asic_value = str(asic_value).lower() + elif asic_value is None: + asic_value = "NULL" + else: + asic_value = str(asic_value) + + # Success + results[oid_label_key][attribute] = {"asicValue": asic_value, "success": True} + logger.debug((f"Got ASIC object {oid_label_key} ({oid}) -> attribute {attribute} ({attribute_oid}) " + f"value {asic_value}")) + + except Exception as e: + err_msg = f"Failed to lookup attribute '{attribute}': {e}" + logger.warning(err_msg) + results[oid_label_key][attribute] = {"success": False, "error": err_msg, "asicValue": None} + + return dict(results) + + +def initialize_sai_api(): + """ + Initialize the SAI API + """ + logger.info("Initializing SAI API") + profileMap = dict() + profileMap[pysairedis.SAI_REDIS_KEY_ENABLE_CLIENT] = "true" + status = pysairedis.sai_api_initialize(0, profileMap) + assert status == pysairedis.SAI_STATUS_SUCCESS, "Failed to initialize SAI API" + logger.info("SAI API initialized") + + +def uninitialize_sai_api(): + """ + Uninitialize the SAI API + """ + logger.info("Uninitializing SAI API") + status = pysairedis.sai_api_uninitialize() + assert status == pysairedis.SAI_STATUS_SUCCESS, "Failed to uninitialize SAI API" + logger.info("SAI API uninitialized") + + +def main(args): + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=HELP_TEXT) + parser.add_argument("-i", "--input", type=argparse.FileType("r"), help="Input JSON file", required=True) + args = parser.parse_args(args) + + try: + query_objects = load_input(args.input) + except Exception as e: + sys.exit(f"Failed to parse JSON input file {args.input}: {e}") + + initialize_sai_api() + + try: + results = query_asic_objects(query_objects) + finally: + uninitialize_sai_api() + + print(json.dumps(results)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tests/common/fixtures/duthost_utils.py b/tests/common/fixtures/duthost_utils.py index 23afe7b4d3..d2913307fd 100644 --- a/tests/common/fixtures/duthost_utils.py +++ b/tests/common/fixtures/duthost_utils.py @@ -9,6 +9,7 @@ import time import json +from pytest_ansible.errors import AnsibleConnectionFailure from paramiko.ssh_exception import AuthenticationException from tests.common import config_reload @@ -16,6 +17,8 @@ from tests.common.utilities import wait_until from jinja2 import Template from netaddr import valid_ipv4, valid_ipv6 +from tests.common.mellanox_data import is_mellanox_device +from tests.common.platform.processes_utils import wait_critical_processes logger = logging.getLogger(__name__) @@ -524,6 +527,15 @@ def dut_qos_maps_module(rand_selected_front_end_dut): return _dut_qos_map(dut) +@pytest.fixture(scope='module') +def is_support_mock_asic(duthosts, rand_one_dut_hostname): + """ + Check if dut supports mock asic. For mellanox device, it doesn't support mock asic + """ + duthost = duthosts[rand_one_dut_hostname] + return not is_mellanox_device(duthost) + + def separated_dscp_to_tc_map_on_uplink(dut_qos_maps_module): """ A helper function to check if separated DSCP_TO_TC_MAP is applied to @@ -741,9 +753,21 @@ def convert_and_restore_config_db_to_ipv6_only(duthosts): for duthost in duthosts.nodes: if config_db_modified[duthost.hostname]: logger.info(f"config changed. Doing config reload for {duthost.hostname}") - config_reload(duthost, wait=120) + try: + config_reload(duthost, wait=120) + except AnsibleConnectionFailure as e: + # IPV4 mgmt interface been deleted by config reload + # In latest SONiC, config reload command will exit after mgmt interface restart + # Then 'duthost' will lost IPV4 connection and throw exception + logger.warning(f'Exception after config reload: {e}') duthosts.reset() + for duthost in duthosts.nodes: + if config_db_modified[duthost.hostname]: + # Wait until all critical processes are up, + # especially snmpd as it needs to be up for SNMP status verification + wait_critical_processes(duthost) + # Verify mgmt-interface status mgmt_intf_name = "eth0" for duthost in duthosts.nodes: diff --git a/tests/common/fixtures/ptfhost_utils.py b/tests/common/fixtures/ptfhost_utils.py index e33045446f..35e7803cd7 100644 --- a/tests/common/fixtures/ptfhost_utils.py +++ b/tests/common/fixtures/ptfhost_utils.py @@ -599,3 +599,16 @@ def ptf_test_port_map_active_active(ptfhost, tbinfo, duthosts, mux_server_url, d ptfhost.copy(content=json.dumps(ports_map), dest=PTF_TEST_PORT_MAP) return PTF_TEST_PORT_MAP + + +@pytest.fixture(scope="function") +def skip_traffic_test(request): + """ + Skip traffic test if the testcase is marked with 'skip_traffic_test' marker. + We are using it to skip traffic test for the testcases that are not supported in specific platforms. + Currently the marker is only be use in tests_mark_conditions_skip_traffic_test.yaml for VS platform. + """ + for m in request.node.iter_markers(): + if m.name == "skip_traffic_test": + return True + return False diff --git a/tests/common/fixtures/tacacs.py b/tests/common/fixtures/tacacs.py index 21d541c535..fe4cc181c8 100644 --- a/tests/common/fixtures/tacacs.py +++ b/tests/common/fixtures/tacacs.py @@ -1,7 +1,9 @@ import logging import pytest + from pytest_ansible.errors import AnsibleConnectionFailure -from tests.tacacs.utils import setup_tacacs_client, setup_tacacs_server, load_tacacs_creds,\ +from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.tacacs.tacacs_helper import setup_tacacs_client, setup_tacacs_server, load_tacacs_creds, \ cleanup_tacacs, restore_tacacs_servers logger = logging.getLogger(__name__) @@ -79,3 +81,16 @@ def setup_tacacs(ptfhost, duthosts, selected_dut, selected_rand_dut, tacacs_cred if request.session.testsfailed: collect_artifact(duthost, request.module.__name__) + + +def get_aaa_sub_options_value(duthost, aaa_type, option): + r""" Verify if AAA sub type's options match with expected value + + Sample output: + admin@vlab-01:~$ show aaa | grep -Po "AAA authentication login \K.*" + local (default) + """ + output = duthost.shell(r'show aaa | grep -Po "AAA {} {} \K.*"'.format(aaa_type, option)) + + pytest_assert(not output['rc'], "Failed to grep AAA {}".format(option)) + return output['stdout'] diff --git a/tests/flow_counter/__init__.py b/tests/common/flow_counter/__init__.py similarity index 100% rename from tests/flow_counter/__init__.py rename to tests/common/flow_counter/__init__.py diff --git a/tests/flow_counter/flow_counter_utils.py b/tests/common/flow_counter/flow_counter_utils.py similarity index 99% rename from tests/flow_counter/flow_counter_utils.py rename to tests/common/flow_counter/flow_counter_utils.py index b7c30b7f5b..ad1907ebb3 100644 --- a/tests/flow_counter/flow_counter_utils.py +++ b/tests/common/flow_counter/flow_counter_utils.py @@ -120,6 +120,7 @@ def is_route_flow_counter_supported(duthosts, tbinfo, enum_rand_one_per_hwsku_ho if rand_selected_dut.facts['asic_type'] == 'vs': # vs platform always set SAI capability to enabled, however, it does not really support all SAI atrributes. # Currently, vs platform does not support route flow counter. + logger.info('Route flow counter is not supported on vs platform') return False skip, _ = check_skip_release(rand_selected_dut, skip_versions) if skip: diff --git a/tests/common/gcu_utils.py b/tests/common/gcu_utils.py new file mode 100644 index 0000000000..d4cd28bfe6 --- /dev/null +++ b/tests/common/gcu_utils.py @@ -0,0 +1,147 @@ +import json +import logging + +import pytest + +from tests.common import config_reload +from tests.common.helpers.assertions import pytest_assert + +logger = logging.getLogger(__name__) +DEFAULT_CHECKPOINT_NAME = "test" + + +def generate_tmpfile(duthost): + """Generate temp file + """ + return duthost.shell('mktemp')['stdout'] + + +def apply_patch(duthost, json_data, dest_file): + """Run apply-patch on target duthost + + Args: + duthost: Device Under Test (DUT) + json_data: Source json patch to apply + dest_file: Destination file on duthost + """ + duthost.copy(content=json.dumps(json_data, indent=4), dest=dest_file) + + cmds = 'config apply-patch {}'.format(dest_file) + + logger.info("Commands: {}".format(cmds)) + output = duthost.shell(cmds, module_ignore_errors=True) + + return output + + +def delete_tmpfile(duthost, tmpfile): + """Delete temp file + """ + duthost.file(path=tmpfile, state='absent') + + +def create_checkpoint(duthost, cp=DEFAULT_CHECKPOINT_NAME): + """Run checkpoint on target duthost + + Args: + duthost: Device Under Test (DUT) + cp: checkpoint filename + """ + cmds = 'config checkpoint {}'.format(cp) + + logger.info("Commands: {}".format(cmds)) + output = duthost.shell(cmds, module_ignore_errors=True) + + pytest_assert( + not output['rc'] + and "Checkpoint created successfully" in output['stdout'] + and verify_checkpoints_exist(duthost, cp), + "Failed to config a checkpoint file: {}".format(cp) + ) + + +def list_checkpoints(duthost): + """List checkpoint on target duthost + + Args: + duthost: Device Under Test (DUT) + """ + cmds = 'config list-checkpoints' + + logger.info("Commands: {}".format(cmds)) + output = duthost.shell(cmds, module_ignore_errors=True) + + pytest_assert( + not output['rc'], + "Failed to list all checkpoint file" + ) + + return output + + +def verify_checkpoints_exist(duthost, cp): + """Check if checkpoint file exist in duthost + """ + output = list_checkpoints(duthost) + return '"{}"'.format(cp) in output['stdout'] + + +def rollback(duthost, cp=DEFAULT_CHECKPOINT_NAME): + """Run rollback on target duthost + + Args: + duthost: Device Under Test (DUT) + cp: rollback filename + """ + cmds = 'config rollback {}'.format(cp) + + logger.info("Commands: {}".format(cmds)) + output = duthost.shell(cmds, module_ignore_errors=True) + + return output + + +def rollback_or_reload(duthost, cp=DEFAULT_CHECKPOINT_NAME): + """Run rollback on target duthost. config_reload if rollback failed. + + Args: + duthost: Device Under Test (DUT) + """ + output = rollback(duthost, cp) + + if output['rc'] or "Config rolled back successfully" not in output['stdout']: + config_reload(duthost) + pytest.fail("config rollback failed. Restored by config_reload") + + +def delete_checkpoint(duthost, cp=DEFAULT_CHECKPOINT_NAME): + """Run checkpoint on target duthost + + Args: + duthost: Device Under Test (DUT) + cp: checkpoint filename + """ + pytest_assert( + verify_checkpoints_exist(duthost, cp), + "Failed to find the checkpoint file: {}".format(cp) + ) + + cmds = 'config delete-checkpoint {}'.format(cp) + + logger.info("Commands: {}".format(cmds)) + output = duthost.shell(cmds, module_ignore_errors=True) + + pytest_assert( + not output['rc'] and "Checkpoint deleted successfully" in output['stdout'], + "Failed to delete a checkpoint file: {}".format(cp) + ) + + +def expect_op_success(duthost, output): + """Expected success from apply-patch output + """ + pytest_assert(not output['rc'], "Command is not running successfully") + pytest_assert( + "Patch applied successfully" in output['stdout'], + "Please check if json file is validate" + ) diff --git a/tests/common/gu_utils.py b/tests/common/gu_utils.py new file mode 100644 index 0000000000..114d2e9a8e --- /dev/null +++ b/tests/common/gu_utils.py @@ -0,0 +1,472 @@ +import json +import logging +import pytest +import os +from jsonpointer import JsonPointer +from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import wait_until +from tests.common.config_reload import config_reload + +logger = logging.getLogger(__name__) + +CONTAINER_SERVICES_LIST = ["swss", "syncd", "radv", "lldp", "dhcp_relay", "teamd", "bgp", "pmon", "telemetry", "acms"] +DEFAULT_CHECKPOINT_NAME = "test" +GCU_FIELD_OPERATION_CONF_FILE = "gcu_field_operation_validators.conf.json" +GET_HWSKU_CMD = "sonic-cfggen -d -v DEVICE_METADATA.localhost.hwsku" + +BASE_DIR = os.path.dirname(os.path.realpath(__file__)) +FILES_DIR = os.path.join(BASE_DIR, "files") +TMP_DIR = '/tmp' + + +def generate_tmpfile(duthost): + """Generate temp file + """ + return duthost.shell('mktemp')['stdout'] + + +def delete_tmpfile(duthost, tmpfile): + """Delete temp file + """ + duthost.file(path=tmpfile, state='absent') + + +def apply_patch(duthost, json_data, dest_file): + """Run apply-patch on target duthost + + Args: + duthost: Device Under Test (DUT) + json_data: Source json patch to apply + dest_file: Destination file on duthost + """ + duthost.copy(content=json.dumps(json_data, indent=4), dest=dest_file) + + cmds = 'config apply-patch {}'.format(dest_file) + + logger.info("Commands: {}".format(cmds)) + output = duthost.shell(cmds, module_ignore_errors=True) + + return output + + +def expect_op_success(duthost, output): + """Expected success from apply-patch output + """ + pytest_assert(not output['rc'], "Command is not running successfully") + pytest_assert( + "Patch applied successfully" in output['stdout'], + "Please check if json file is validate" + ) + + +def expect_op_success_and_reset_check(duthost, output, service_name, timeout, interval, delay): + """Add contianer reset check after op success + + Args: + duthost: Device Under Test (DUT) + output: Command couput + service_name: Service to reset + timeout: Maximum time to wait + interval: Poll interval + delay: Delay time + """ + expect_op_success(duthost, output) + if start_limit_hit(duthost, service_name): + reset_start_limit_hit(duthost, service_name, timeout, interval, delay) + + +def expect_res_success(duthost, output, expected_content_list, unexpected_content_list): + """Check output success with expected and unexpected content + + Args: + duthost: Device Under Test (DUT) + output: Command output + expected_content_list: Expected content from output + unexpected_content_list: Unexpected content from output + """ + for expected_content in expected_content_list: + pytest_assert( + expected_content in output['stdout'], + "{} is expected content".format(expected_content) + ) + + for unexpected_content in unexpected_content_list: + pytest_assert( + unexpected_content not in output['stdout'], + "{} is unexpected content".format(unexpected_content) + ) + + +def expect_op_failure(output): + """Expected failure from apply-patch output + """ + logger.info("return code {}".format(output['rc'])) + pytest_assert( + output['rc'], + "The command should fail with non zero return code" + ) + + +def start_limit_hit(duthost, service_name): + """If start-limit-hit is hit, the service will not start anyway. + + Args: + service_name: Service to reset + """ + service_status = duthost.shell("systemctl status {}.service | grep 'Active'".format(service_name)) + pytest_assert( + not service_status['rc'], + "{} service status cannot be found".format(service_name) + ) + + for line in service_status["stdout_lines"]: + if "start-limit-hit" in line: + return True + + return False + + +def reset_start_limit_hit(duthost, service_name, timeout, interval, delay): + """Reset service if hit start-limit-hit + + Args: + duthost: Device Under Test (DUT) + service_name: Service to reset + timeout: Maximum time to wait + interval: Poll interval + delay: Delay time + """ + logger.info("Reset service '{}' due to start-limit-hit".format(service_name)) + + service_reset_failed = duthost.shell("systemctl reset-failed {}.service".format(service_name)) + pytest_assert( + not service_reset_failed['rc'], + "{} systemctl reset-failed service fails" + ) + + service_start = duthost.shell("systemctl start {}.service".format(service_name)) + pytest_assert( + not service_start['rc'], + "{} systemctl start service fails" + ) + + if service_name not in CONTAINER_SERVICES_LIST: + return + + reset_service = wait_until(timeout, + interval, + delay, + duthost.is_service_fully_started, + service_name) + pytest_assert( + reset_service, + "Failed to reset service '{}' due to start-limit-hit".format(service_name) + ) + + +def list_checkpoints(duthost): + """List checkpoint on target duthost + + Args: + duthost: Device Under Test (DUT) + cp: checkpoint filename + """ + cmds = 'config list-checkpoints' + + logger.info("Commands: {}".format(cmds)) + output = duthost.shell(cmds, module_ignore_errors=True) + + pytest_assert( + not output['rc'], + "Failed to list all checkpoint file" + ) + + return output + + +def verify_checkpoints_exist(duthost, cp): + """Check if checkpoint file exist in duthost + """ + output = list_checkpoints(duthost) + return '"{}"'.format(cp) in output['stdout'] + + +def create_checkpoint(duthost, cp=DEFAULT_CHECKPOINT_NAME): + """Run checkpoint on target duthost + + Args: + duthost: Device Under Test (DUT) + cp: checkpoint filename + """ + cmds = 'config checkpoint {}'.format(cp) + + logger.info("Commands: {}".format(cmds)) + output = duthost.shell(cmds, module_ignore_errors=True) + + pytest_assert( + not output['rc'] + and "Checkpoint created successfully" in output['stdout'] + and verify_checkpoints_exist(duthost, cp), + "Failed to config a checkpoint file: {}".format(cp) + ) + + +def delete_checkpoint(duthost, cp=DEFAULT_CHECKPOINT_NAME): + """Run checkpoint on target duthost + + Args: + duthost: Device Under Test (DUT) + cp: checkpoint filename + """ + pytest_assert( + verify_checkpoints_exist(duthost, cp), + "Failed to find the checkpoint file: {}".format(cp) + ) + + cmds = 'config delete-checkpoint {}'.format(cp) + + logger.info("Commands: {}".format(cmds)) + output = duthost.shell(cmds, module_ignore_errors=True) + + pytest_assert( + not output['rc'] and "Checkpoint deleted successfully" in output['stdout'], + "Failed to delete a checkpoint file: {}".format(cp) + ) + + +def rollback(duthost, cp=DEFAULT_CHECKPOINT_NAME): + """Run rollback on target duthost + + Args: + duthost: Device Under Test (DUT) + cp: rollback filename + """ + cmds = 'config rollback {}'.format(cp) + + logger.info("Commands: {}".format(cmds)) + output = duthost.shell(cmds, module_ignore_errors=True) + + return output + + +def rollback_or_reload(duthost, cp=DEFAULT_CHECKPOINT_NAME): + """Run rollback on target duthost. config_reload if rollback failed. + + Args: + duthost: Device Under Test (DUT) + """ + output = rollback(duthost, cp) + + if output['rc'] or "Config rolled back successfully" not in output['stdout']: + config_reload(duthost) + pytest.fail("config rollback failed. Restored by config_reload") + + +def create_path(tokens): + return JsonPointer.from_parts(tokens).path + + +def check_show_ip_intf(duthost, intf_name, expected_content_list, unexpected_content_list, is_ipv4=True): + """Check lo interface status by show command + + Sample output: + admin@vlab-01:~$ show ip interfaces | grep -w Vlan1000 + Vlan1000 192.168.0.1/21 up/up N/A N/A + admin@vlab-01:~$ show ipv6 interfaces | grep -w Vlan1000 + Vlan1000 fc02:1000::1/64 up/up N/A N/A + fe80::5054:ff:feda:c6af%Vlan1000/64 N/A N/A + """ + address_family = "ip" if is_ipv4 else "ipv6" + output = duthost.shell("show {} interfaces | grep -w {} || true".format(address_family, intf_name)) + + expect_res_success(duthost, output, expected_content_list, unexpected_content_list) + + +def check_vrf_route_for_intf(duthost, vrf_name, intf_name, is_ipv4=True): + """Check ip route for specific vrf + + Sample output: + admin@vlab-01:~$ show ip route vrf Vrf_01 | grep -w Loopback0 + C>* 10.1.0.32/32 is directly connected, Loopback0, 00:00:13 + """ + address_family = "ip" if is_ipv4 else "ipv6" + output = duthost.shell("show {} route vrf {} | grep -w {}".format(address_family, vrf_name, intf_name)) + + pytest_assert(not output['rc'], "Route not found for {} in vrf {}".format(intf_name, vrf_name)) + + +def get_gcu_field_operations_conf(duthost): + get_gcu_dir_path_cmd = 'python3 -c \"import generic_config_updater ; print(generic_config_updater.__path__)\"' + gcu_dir_path = duthost.shell("{}".format(get_gcu_dir_path_cmd))['stdout'].replace("[", "").replace("]", "") + gcu_conf = duthost.shell('cat {}/{}'.format(gcu_dir_path, GCU_FIELD_OPERATION_CONF_FILE))['stdout'] + gcu_conf_json = json.loads(gcu_conf) + return gcu_conf_json + + +def get_asic_name(duthost): + asic_type = duthost.facts["asic_type"] + asic = "unknown" + gcu_conf = get_gcu_field_operations_conf(duthost) + asic_mapping = gcu_conf["helper_data"]["rdma_config_update_validator"] + + def _get_asic_name(asic_type): + cur_hwsku = duthost.shell(GET_HWSKU_CMD)['stdout'].rstrip('\n') + # The key name is like "mellanox_asics" or "broadcom_asics" + asic_key_name = asic_type + "_asics" + if asic_key_name not in asic_mapping: + return "unknown" + asic_hwskus = asic_mapping[asic_key_name] + for asic_name, hwskus in asic_hwskus.items(): + if cur_hwsku.lower() in [hwsku.lower() for hwsku in hwskus]: + return asic_name + return "unknown" + + if asic_type == 'cisco-8000': + asic = "cisco-8000" + elif asic_type in ('mellanox', 'broadcom'): + asic = _get_asic_name(asic_type) + elif asic_type == 'vs': + # We need to check both mellanox and broadcom asics for vs platform + dummy_asic_list = ['broadcom', 'mellanox', 'cisco-8000'] + for dummy_asic in dummy_asic_list: + tmp_asic = _get_asic_name(dummy_asic) + if tmp_asic != "unknown": + asic = tmp_asic + break + + return asic + + +def is_valid_platform_and_version(duthost, table, scenario, operation, field_value=None): + asic = get_asic_name(duthost) + os_version = duthost.os_version + if asic == "unknown": + return False + gcu_conf = get_gcu_field_operations_conf(duthost) + + if operation == "add": + if field_value: + operation = "replace" + + # Ensure that the operation is supported by comparing with conf + try: + valid_ops = gcu_conf["tables"][table]["validator_data"]["rdma_config_update_validator"][scenario]["operations"] + if operation not in valid_ops: + return False + except KeyError: + return False + except IndexError: + return False + + # Ensure that the version is suported by comparing with conf + if "master" in os_version or "internal" in os_version: + return True + try: + version_required = gcu_conf["tables"][table]["validator_data"]["rdma_config_update_validator"][scenario]["platforms"][asic] # noqa E501 + if version_required == "": + return False + # os_version is in format "20220531.04", version_required is in format "20220500" + return os_version[0:8] >= version_required[0:8] + except KeyError: + return False + except IndexError: + return False + + +def apply_formed_json_patch(duthost, json_patch, setup): + + duts_to_apply = [duthost] + outputs = [] + if setup["is_dualtor"]: + duts_to_apply.append(setup["rand_unselected_dut"]) + + for dut in duts_to_apply: + tmpfile = generate_tmpfile(dut) + logger.info("tmpfile {}".format(tmpfile)) + + try: + output = apply_patch(dut, json_data=json_patch, dest_file=tmpfile) + outputs.append(output) + finally: + delete_tmpfile(dut, tmpfile) + + return outputs + + +def expect_acl_table_match_multiple_bindings(duthost, + table_name, + expected_first_line_content, + expected_bindings, + setup): + """Check if acl table show as expected + Acl table with multiple bindings will show as such + + Table_Name Table_Type Ethernet4 Table_Description ingress + Ethernet8 + Ethernet12 + Ethernet16 + + So we must have separate checks for first line and bindings + """ + + cmds = "show acl table {}".format(table_name) + + duts_to_check = [duthost] + if setup["is_dualtor"]: + duts_to_check.append(setup["rand_unselected_dut"]) + + for dut in duts_to_check: + + output = dut.show_and_parse(cmds) + pytest_assert(len(output) > 0, "'{}' is not a table on this device".format(table_name)) + + first_line = output[0] + pytest_assert(set(first_line.values()) == set(expected_first_line_content)) + table_bindings = [first_line["binding"]] + for i in range(len(output)): + table_bindings.append(output[i]["binding"]) + pytest_assert(set(table_bindings) == set(expected_bindings), "ACL Table bindings don't fully match") + + +def expect_acl_rule_match(duthost, rulename, expected_content_list, setup): + """Check if acl rule shows as expected""" + + cmds = "show acl rule DYNAMIC_ACL_TABLE {}".format(rulename) + + duts_to_check = [duthost] + if setup["is_dualtor"]: + duts_to_check.append(setup["rand_unselected_dut"]) + + for dut in duts_to_check: + + output = dut.show_and_parse(cmds) + + rule_lines = len(output) + + pytest_assert(rule_lines >= 1, "'{}' is not a rule on this device".format(rulename)) + + first_line = output[0].values() + + pytest_assert(set(first_line) <= set(expected_content_list), "ACL Rule details do not match!") + + if rule_lines > 1: + for i in range(1, rule_lines): + pytest_assert(output[i]["match"] in expected_content_list, + "Unexpected match condition found: " + str(output[i]["match"])) + + +def expect_acl_rule_removed(duthost, rulename, setup): + """Check if ACL rule has been successfully removed""" + + cmds = "show acl rule DYNAMIC_ACL_TABLE {}".format(rulename) + + duts_to_check = [duthost] + if setup["is_dualtor"]: + duts_to_check.append(setup["rand_unselected_dut"]) + + for dut in duts_to_check: + output = dut.show_and_parse(cmds) + + removed = len(output) == 0 + + pytest_assert(removed, "'{}' showed a rule, this following rule should have been removed".format(cmds)) diff --git a/tests/common/helpers/bgp.py b/tests/common/helpers/bgp.py index a89e371421..932fc9a11e 100644 --- a/tests/common/helpers/bgp.py +++ b/tests/common/helpers/bgp.py @@ -20,6 +20,42 @@ def _write_variable_from_j2_to_configdb(duthost, template_file, **kwargs): duthost.file(path=save_dest_path, state="absent") +def run_bgp_facts(duthosts, enum_frontend_dut_hostname, enum_asic_index): + """compare the bgp facts between observed states and target state""" + + duthost = duthosts[enum_frontend_dut_hostname] + + bgp_facts = duthost.bgp_facts(instance_id=enum_asic_index)['ansible_facts'] + namespace = duthost.get_namespace_from_asic_id(enum_asic_index) + config_facts = duthost.config_facts(host=duthost.hostname, source="running", namespace=namespace)['ansible_facts'] + sonic_db_cmd = "sonic-db-cli {}".format("-n " + namespace if namespace else "") + for k, v in list(bgp_facts['bgp_neighbors'].items()): + # Verify bgp sessions are established + assert v['state'] == 'established' + # Verify local ASNs in bgp sessions + assert v['local AS'] == int(config_facts['DEVICE_METADATA']['localhost']['bgp_asn'].encode().decode("utf-8")) + # Check bgpmon functionality by validate STATE DB contains this neighbor as well + state_fact = duthost.shell('{} STATE_DB HGET "NEIGH_STATE_TABLE|{}" "state"' + .format(sonic_db_cmd, k), module_ignore_errors=False)['stdout_lines'] + peer_type = duthost.shell('{} STATE_DB HGET "NEIGH_STATE_TABLE|{}" "peerType"' + .format(sonic_db_cmd, k), + module_ignore_errors=False)['stdout_lines'] + assert state_fact[0] == "Established" + assert peer_type[0] == "i-BGP" if v['remote AS'] == v['local AS'] else "e-BGP" + + # In multi-asic, would have 'BGP_INTERNAL_NEIGHBORS' and possibly no 'BGP_NEIGHBOR' (ebgp) neighbors. + nbrs_in_cfg_facts = {} + nbrs_in_cfg_facts.update(config_facts.get('BGP_NEIGHBOR', {})) + nbrs_in_cfg_facts.update(config_facts.get('BGP_INTERNAL_NEIGHBOR', {})) + # In VoQ Chassis, we would have BGP_VOQ_CHASSIS_NEIGHBOR as well. + nbrs_in_cfg_facts.update(config_facts.get('BGP_VOQ_CHASSIS_NEIGHBOR', {})) + for k, v in list(nbrs_in_cfg_facts.items()): + # Compare the bgp neighbors name with config db bgp neighbors name + assert v['name'] == bgp_facts['bgp_neighbors'][k]['description'] + # Compare the bgp neighbors ASN with config db + assert int(v['asn'].encode().decode("utf-8")) == bgp_facts['bgp_neighbors'][k]['remote AS'] + + class BGPNeighbor(object): def __init__(self, duthost, ptfhost, name, @@ -114,8 +150,9 @@ def teardown_session(self): self.ptfhost.exabgp(name=self.name, state="stopped") if not self.is_passive: for asichost in self.duthost.asics: - logging.debug("update CONFIG_DB admin_status to down") - asichost.run_sonic_db_cli_cmd("CONFIG_DB hset 'BGP_NEIGHBOR|{}' admin_status down".format(self.ip)) + if asichost.namespace == self.namespace: + logging.debug("update CONFIG_DB admin_status to down on {}".format(asichost.namespace)) + asichost.run_sonic_db_cli_cmd("CONFIG_DB hset 'BGP_NEIGHBOR|{}' admin_status down".format(self.ip)) def announce_route(self, route): if "aspath" in route: diff --git a/tests/common/helpers/cache_utils.py b/tests/common/helpers/cache_utils.py new file mode 100644 index 0000000000..6a358bc893 --- /dev/null +++ b/tests/common/helpers/cache_utils.py @@ -0,0 +1,26 @@ +import logging + +logger = logging.getLogger(__name__) + +SINGLE_ASIC_ZONE = "single_asic" + + +def sonic_asic_zone_getter(function, func_args, func_kargs): + """ + SonicAsic specific zone getter used for decorator cached. + example: asic0 of lc1-1, + zone: lc1-1_asic0 + """ + + hostname = getattr(func_args[0], "hostname", None) + # For the SonicAsic obj of single asic DUT, the namespace is None + # give SINGLE_ASIC_ZONE as the part of the zone + namespace = getattr(func_args[0], "namespace", SINGLE_ASIC_ZONE) + + if not hostname: + raise RuntimeError(f"[Cache] Can't get hostname[{hostname}] for asic[{namespace}]") + + zone = f"{hostname}_{namespace}" + logger.info(f"[Cache] generate zone[{zone}] for asic[{namespace}]") + + return zone diff --git a/tests/common/helpers/drop_counters/drop_counters.py b/tests/common/helpers/drop_counters/drop_counters.py index 40a82d69bd..e8c71dcd33 100644 --- a/tests/common/helpers/drop_counters/drop_counters.py +++ b/tests/common/helpers/drop_counters/drop_counters.py @@ -103,21 +103,25 @@ def _get_drops_across_all_duthosts(): # if the dut_iface is not found ignore this device if dut_iface not in pkt_drops: continue - drop_list.append(int(pkt_drops[dut_iface][column_key].replace(",", ""))) + try: + drop_list.append(int(pkt_drops[dut_iface][column_key].replace(",", ""))) + except ValueError: + # Catch error invalid literal for int() with base 10: 'N/A' + drop_list.append(0) return drop_list def _check_drops_on_dut(): return packets_count in _get_drops_across_all_duthosts() if not wait_until(25, 1, 0, _check_drops_on_dut): - # The actual Drop count should always be equal or 1 or 2 packets more than what is expected - # due to some other drop may occur over the interface being examined. - # When that happens if looking onlyu for exact count it will be a false positive failure. - # So do one more check to allow up to 2 packets more dropped than what was expected as an allowed case. + # We were seeing a few more drop counters than expected, so we are allowing a small margin of error + DEOP_MARGIN = 10 actual_drop = _get_drops_across_all_duthosts() - if ((packets_count+2) in actual_drop) or ((packets_count+1) in actual_drop): - logger.warning("Actual drops {} exceeded expected drops {} on iface {}\n" - .format(actual_drop, packets_count, dut_iface)) + for drop in actual_drop: + if drop >= packets_count and drop <= packets_count + DEOP_MARGIN: + logger.warning("Actual drops {} exceeded expected drops {} on iface {}\n".format( + actual_drop, packets_count, dut_iface)) + break else: fail_msg = "'{}' drop counter was not incremented on iface {}. DUT {} == {}; Sent == {}".format( column_key, dut_iface, column_key, actual_drop, packets_count) diff --git a/tests/common/helpers/dut_utils.py b/tests/common/helpers/dut_utils.py index f6e42e70e1..e7307c7c4d 100644 --- a/tests/common/helpers/dut_utils.py +++ b/tests/common/helpers/dut_utils.py @@ -1,11 +1,17 @@ import logging +import allure +import os from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import get_host_visible_vars from tests.common.utilities import wait_until +from tests.common.errors import RunAnsibleModuleFail from collections import defaultdict CONTAINER_CHECK_INTERVAL_SECS = 1 CONTAINER_RESTART_THRESHOLD_SECS = 180 +# Ansible config files +LAB_CONNECTION_GRAPH_PATH = os.path.normpath((os.path.join(os.path.dirname(__file__), "../../../ansible/files"))) + logger = logging.getLogger(__name__) @@ -175,6 +181,20 @@ def get_program_info(duthost, container_name, program_name): return program_status, program_pid +def kill_process_by_pid(duthost, container_name, program_name, program_pid): + """ + @summary: Kill a process in the specified container by its pid + """ + kill_cmd_result = duthost.shell("docker exec {} kill -SIGKILL {}".format(container_name, program_pid)) + + # Get the exit code of 'kill' command + exit_code = kill_cmd_result["rc"] + pytest_assert(exit_code == 0, "Failed to stop program '{}' before test".format(program_name)) + + logger.info("Program '{}' in container '{}' was stopped successfully" + .format(program_name, container_name)) + + def get_disabled_container_list(duthost): """Gets the container/service names which are disabled. @@ -319,3 +339,57 @@ def ignore_t2_syslog_msgs(duthost): # DUT's loganalyzer would be null if we have disable_loganalyzer specified if a_dut.loganalyzer: a_dut.loganalyzer.ignore_regex.extend(ignoreRegex) + + +def get_sai_sdk_dump_file(duthost, dump_file_name): + full_path_dump_file = f"/tmp/{dump_file_name}" + cmd_gen_sdk_dump = f"docker exec syncd bash -c 'saisdkdump -f {full_path_dump_file}' " + duthost.shell(cmd_gen_sdk_dump) + + cmd_copy_dmp_from_syncd_to_host = f"docker cp syncd:{full_path_dump_file} {full_path_dump_file}" + duthost.shell(cmd_copy_dmp_from_syncd_to_host) + + compressed_dump_file = f"/tmp/{dump_file_name}.tar.gz" + duthost.archive(path=full_path_dump_file, dest=compressed_dump_file, format='gz') + + duthost.fetch(src=compressed_dump_file, dest="/tmp/", flat=True) + allure.attach.file(compressed_dump_file, dump_file_name, extension=".tar.gz") + + +def is_mellanox_devices(hwsku): + """ + A helper function to check if a given sku is Mellanox device + """ + hwsku = hwsku.lower() + return 'mellanox' in hwsku \ + or 'msn' in hwsku \ + or 'mlnx' in hwsku + + +def is_mellanox_fanout(duthost, localhost): + # Ansible localhost fixture which calls ansible playbook on the local host + + if duthost.facts.get("asic_type") == "vs": + return False + + try: + dut_facts = \ + localhost.conn_graph_facts(host=duthost.hostname, filepath=LAB_CONNECTION_GRAPH_PATH)["ansible_facts"] + except RunAnsibleModuleFail as e: + logger.info("Get dut_facts failed, reason:{}".format(e.results['msg'])) + return False + + intf = list(dut_facts["device_conn"][duthost.hostname].keys())[0] + fanout_host = dut_facts["device_conn"][duthost.hostname][intf]["peerdevice"] + + try: + fanout_facts = \ + localhost.conn_graph_facts(host=fanout_host, filepath=LAB_CONNECTION_GRAPH_PATH)["ansible_facts"] + except RunAnsibleModuleFail: + return False + + fanout_sku = fanout_facts['device_info'][fanout_host]['HwSku'] + if not is_mellanox_devices(fanout_sku): + return False + + return True diff --git a/tests/platform_tests/files/invalid_format_policy.json b/tests/common/helpers/files/invalid_format_policy.json similarity index 100% rename from tests/platform_tests/files/invalid_format_policy.json rename to tests/common/helpers/files/invalid_format_policy.json diff --git a/tests/platform_tests/files/invalid_value_policy.json b/tests/common/helpers/files/invalid_value_policy.json similarity index 100% rename from tests/platform_tests/files/invalid_value_policy.json rename to tests/common/helpers/files/invalid_value_policy.json diff --git a/tests/platform_tests/files/valid_policy.json b/tests/common/helpers/files/valid_policy.json similarity index 100% rename from tests/platform_tests/files/valid_policy.json rename to tests/common/helpers/files/valid_policy.json diff --git a/tests/common/helpers/inventory_utils.py b/tests/common/helpers/inventory_utils.py new file mode 100644 index 0000000000..bee5ff4b15 --- /dev/null +++ b/tests/common/helpers/inventory_utils.py @@ -0,0 +1,90 @@ +import logging +import re + +import yaml + +from tests.common.helpers.yaml_utils import BlankNone + +logger = logging.getLogger(__name__) + + +def trim_inventory(inv_files, tbinfo): + """ + Trim the useless topology neighbor the inv_files according to testbed to speed up ansible inventory initialization. + + For every test server, we pre-define ~100 ansible_hosts for the neighbors. + We put all of the ansible_hosts of test servers into one inventory file. + The inventory file contains thousands of ansible_hosts, but most of them are useless to the selected testbed, + Because the testbed only need the definition of the neighbor for its neighbor server. + + During the establishment of the ansible_host, it iterate and compute the ansible_host one by one, + The useless ansible_host extremely slow down the initialization. + + Hence, will trim and generate a temporary inventory file, for example: + ['../ansible/veos', '../ansible/inv1'] -> ['../ansible/veos_kvm-t0_trim_tmp', '../ansible/inv1'] + Then pytest will use the light inventory file '../ansible/veos_kvm-t0_trim_tmp' to initialize the ansible_hosts. + + Args: + inv_files: inventory file list, sample: ['../ansible/veos', '../ansible/inv1'] + tbinfo: tbinfo + + Returns: + Direct update inv_files and return nothing + """ + + vm_base = tbinfo.get("vm_base", None) + tb_name = tbinfo.get("conf-name", None) + pattern = r'VM\d{3,7}' + logger.info(f"Start to trim inventory, tb[{tb_name}] vm_base [{vm_base}], inv file {inv_files}") + + # Find a key in all the levels of a dict, + # Return the val of the key and the vm_base_path of the key + def find_key(d: dict, target_key: str = None, regex: str = None): + # Stack to keep track of dictionaries and their paths + stack = [(d, [])] + while stack: + current_dict, path = stack.pop() + for key, value in current_dict.items(): + # If regex is passed, use regex to match + if regex and re.match(regex, key): + return value, path + [key] + # If regex is None, exact match + if key == target_key: + return value, path + [key] + if isinstance(value, dict): + stack.append((value, path + [key])) + return None, [] + + # Remove all the matched + for idx in range(len(inv_files)): + inv_file = inv_files[idx] + with open(inv_file, 'r') as file: + inv = yaml.safe_load(file) + # If the vm_base is found in the inv_file, + # then other useless topology neighbor definition are in it, + # that's the inventory to be trimmed + _, vm_base_path = find_key(d=inv, target_key=vm_base) + if vm_base_path: + keys_to_del = set() + logger.info(f"Find vm_base {vm_base} in inv file {inv_file}, path: {vm_base_path}") + + for root_key in inv.keys(): + neighbor_val, neighbor_path = find_key(d=inv[root_key], regex=pattern) + if neighbor_path: + # Keep the neighbor server for the testbed + if root_key == vm_base_path[0]: + logger.info(f"vm_base[{vm_base}] located in {root_key}, inv file {inv_file}, keep it") + # Remove all the useless neighbor server ansible_hosts + else: + logger.info(f"Attempt to remove {root_key} in inv file {inv_file}") + keys_to_del.add(root_key) + + for key_to_del in keys_to_del: + del inv[key_to_del] + + # dump and replace trimmed inventory file + trimmed_inventory_file_name = f"{inv_file}_{tb_name}_trim_tmp" + with BlankNone(), open(trimmed_inventory_file_name, 'w') as f: + yaml.dump(inv, f) + + inv_files[idx] = trimmed_inventory_file_name diff --git a/tests/platform_tests/mellanox/mellanox_thermal_control_test_helper.py b/tests/common/helpers/mellanox_thermal_control_test_helper.py similarity index 99% rename from tests/platform_tests/mellanox/mellanox_thermal_control_test_helper.py rename to tests/common/helpers/mellanox_thermal_control_test_helper.py index ff489bcc65..786ce5352c 100644 --- a/tests/platform_tests/mellanox/mellanox_thermal_control_test_helper.py +++ b/tests/common/helpers/mellanox_thermal_control_test_helper.py @@ -5,10 +5,10 @@ import logging import time from pkg_resources import parse_version -from tests.platform_tests.thermal_control_test_helper import mocker, FanStatusMocker, ThermalStatusMocker, \ +from tests.common.helpers.thermal_control_test_helper import mocker, FanStatusMocker, ThermalStatusMocker, \ SingleFanMocker from tests.common.mellanox_data import get_hw_management_version, get_platform_data -from .minimum_table import get_min_table +from tests.common.helpers.minimum_table import get_min_table from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert diff --git a/tests/platform_tests/mellanox/minimum_table.py b/tests/common/helpers/minimum_table.py similarity index 100% rename from tests/platform_tests/mellanox/minimum_table.py rename to tests/common/helpers/minimum_table.py diff --git a/tests/common/helpers/multi_thread_utils.py b/tests/common/helpers/multi_thread_utils.py new file mode 100644 index 0000000000..09c7ca7cab --- /dev/null +++ b/tests/common/helpers/multi_thread_utils.py @@ -0,0 +1,27 @@ +from concurrent.futures import Future, as_completed +from concurrent.futures.thread import ThreadPoolExecutor +from typing import Optional, List + + +class SafeThreadPoolExecutor(ThreadPoolExecutor): + """An enhanced thread pool executor + + Everytime we submit a task, it will store the feature in self.features + On the __exit__ function, it will wait all the tasks to be finished, + And check any exceptions that are raised during the task executing + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.features: Optional[List[Future]] = [] + + def submit(self, __fn, *args, **kwargs): + f = super().submit(__fn, *args, **kwargs) + self.features.append(f) + return f + + def __exit__(self, exc_type, exc_val, exc_tb): + for future in as_completed(self.features): + # if exception caught in the sub-thread, .result() will raise it in the main thread + _ = future.result() + self.shutdown(wait=True) + return False diff --git a/tests/common/helpers/ntp_helper.py b/tests/common/helpers/ntp_helper.py new file mode 100644 index 0000000000..81ca5a2e48 --- /dev/null +++ b/tests/common/helpers/ntp_helper.py @@ -0,0 +1,73 @@ +import pytest +import time +from contextlib import contextmanager +from tests.common.utilities import wait_until +from tests.common.helpers.assertions import pytest_assert + + +@contextmanager +def _context_for_setup_ntp(ptfhost, duthosts, rand_one_dut_hostname, ptf_use_ipv6): + """setup ntp client and server""" + duthost = duthosts[rand_one_dut_hostname] + + ptfhost.lineinfile(path="/etc/ntp.conf", line="server 127.127.1.0 prefer") + + # restart ntp server + ntp_en_res = ptfhost.service(name="ntp", state="restarted") + + pytest_assert(wait_until(120, 5, 0, check_ntp_status, ptfhost), + "NTP server was not started in PTF container {}; NTP service start result {}" + .format(ptfhost.hostname, ntp_en_res)) + + # setup ntp on dut to sync with ntp server + config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] + ntp_servers = config_facts.get('NTP_SERVER', {}) + for ntp_server in ntp_servers: + duthost.command("config ntp del %s" % ntp_server) + + duthost.command("config ntp add %s" % (ptfhost.mgmt_ipv6 if ptf_use_ipv6 else ptfhost.mgmt_ip)) + + yield + + # stop ntp server + ptfhost.service(name="ntp", state="stopped") + # reset ntp client configuration + duthost.command("config ntp del %s" % (ptfhost.mgmt_ipv6 if ptf_use_ipv6 else ptfhost.mgmt_ip)) + for ntp_server in ntp_servers: + duthost.command("config ntp add %s" % ntp_server) + # The time jump leads to exception in lldp_syncd. The exception has been handled by lldp_syncd, + # but it will leave error messages in syslog, which will cause subsequent test cases to fail. + # So we need to wait for a while to make sure the error messages are flushed. + # The default update interval of lldp_syncd is 10 seconds, so we wait for 20 seconds here. + time.sleep(20) + + +@pytest.fixture(scope="function") +def setup_ntp_func(ptfhost, duthosts, rand_one_dut_hostname, ptf_use_ipv6): + with _context_for_setup_ntp(ptfhost, duthosts, rand_one_dut_hostname, ptf_use_ipv6) as result: + yield result + + +def check_ntp_status(host): + res = host.command("ntpstat", module_ignore_errors=True) + if res['rc'] != 0: + return False + return True + + +def run_ntp(duthosts, rand_one_dut_hostname, setup_ntp): + """ Verify that DUT is synchronized with configured NTP server """ + duthost = duthosts[rand_one_dut_hostname] + + ntpsec_conf_stat = duthost.stat(path="/etc/ntpsec/ntp.conf") + using_ntpsec = ntpsec_conf_stat["stat"]["exists"] + + duthost.service(name='ntp', state='stopped') + if using_ntpsec: + duthost.command("timeout 20 ntpd -gq -u ntpsec:ntpsec") + else: + ntp_uid = ":".join(duthost.command("getent passwd ntp")['stdout'].split(':')[2:4]) + duthost.command("timeout 20 ntpd -gq -u {}".format(ntp_uid)) + duthost.service(name='ntp', state='restarted') + pytest_assert(wait_until(720, 10, 0, check_ntp_status, duthost), + "NTP not in sync") diff --git a/tests/common/helpers/parallel_utils.py b/tests/common/helpers/parallel_utils.py new file mode 100644 index 0000000000..d5cb0ee64b --- /dev/null +++ b/tests/common/helpers/parallel_utils.py @@ -0,0 +1,5 @@ +from tests.common import config_reload + + +def config_reload_parallel_compatible(node, results, *args, **kwargs): + return config_reload(node, *args, **kwargs) diff --git a/tests/common/helpers/pfc_gen.py b/tests/common/helpers/pfc_gen.py index 6904bf31ea..9236372362 100755 --- a/tests/common/helpers/pfc_gen.py +++ b/tests/common/helpers/pfc_gen.py @@ -10,34 +10,62 @@ import logging import logging.handlers import time +import multiprocessing + from socket import socket, AF_PACKET, SOCK_RAW logger = logging.getLogger('MyLogger') logger.setLevel(logging.DEBUG) +# Maximum number of processes to be created +MAX_PROCESS_NUM = 4 -def checksum(msg): - s = 0 - - # loop taking 2 characters at a time - for i in range(0, len(msg), 2): - w = ord(msg[i]) + (ord(msg[i+1]) << 8) - s = s + w - - s = (s >> 16) + (s & 0xffff) - s = s + (s >> 16) - - # complement and mask to 4 byte short - s = ~s & 0xffff - return s +class PacketSender(): + """ + A class to send PFC pause frames + """ + def __init__(self, interfaces, packet, num, interval): + # Create RAW socket to send PFC pause frames + self.sockets = [] + try: + for interface in interfaces: + s = socket(AF_PACKET, SOCK_RAW) + s.bind((interface, 0)) + self.sockets.append(s) + except Exception as e: + print("Unable to create socket. Check your permissions: %s" % e) + sys.exit(1) + self.packet_num = num + self.packet_interval = interval + self.process = None + self.packet = packet + + def send_packets(self): + iteration = self.packet_num + while iteration > 0: + for s in self.sockets: + s.send(self.packet) + if self.packet_interval > 0: + time.sleep(self.packet_interval) + iteration -= 1 + + def start(self): + self.process = multiprocessing.Process(target=self.send_packets) + self.process.start() + + def stop(self, timeout=None): + if self.process: + self.process.join(timeout) + for s in self.sockets: + s.close() def main(): usage = "usage: %prog [options] arg1 arg2" parser = optparse.OptionParser(usage=usage) parser.add_option("-i", "--interface", type="string", dest="interface", - help="Interface list to send packets, seperated by ','", metavar="Interface") + help="Interface list to send packets, separated by ','", metavar="Interface") parser.add_option('-p', "--priority", type="int", dest="priority", help="PFC class enable bitmap.", metavar="Priority", default=-1) parser.add_option("-t", "--time", type="int", dest="time", @@ -50,6 +78,8 @@ def main(): help="Send global pause frames (not PFC)", default=False) parser.add_option("-s", "--send_pfc_frame_interval", type="float", dest="send_pfc_frame_interval", help="Interval sending pfc frame", metavar="send_pfc_frame_interval", default=0) + parser.add_option("-m", "--multiprocess", action="store_true", dest="multiprocess", + help="Use multiple processes to send packets", default=False) (options, args) = parser.parse_args() @@ -77,21 +107,10 @@ def main(): interfaces = options.interface.split(',') - try: - sockets = [] - for i in range(0, len(interfaces)): - sockets.append(socket(AF_PACKET, SOCK_RAW)) - except Exception: - print("Unable to create socket. Check your permissions") - sys.exit(1) - # Configure logging handler = logging.handlers.SysLogHandler(address=(options.rsyslog_server, 514)) logger.addHandler(handler) - for s, interface in zip(sockets, interfaces): - s.bind((interface, 0)) - """ Set PFC defined fields and generate the packet @@ -155,14 +174,30 @@ def main(): packet = packet + b"\x00\x00" pre_str = 'GLOBAL_PF' if options.global_pf else 'PFC' - print(("Generating %s Packet(s)" % options.num)) - logger.debug(pre_str + '_STORM_START') - iteration = options.num - while iteration > 0: - for s in sockets: - s.send(packet) - time.sleep(options.send_pfc_frame_interval) - iteration -= 1 + logger.debug(pre_str + '_STORM_DEBUG') + + # Start sending PFC pause frames + if options.multiprocess: + senders = [] + interface_slices = [[] for i in range(MAX_PROCESS_NUM)] + for i in range(0, len(interfaces)): + interface_slices[i % MAX_PROCESS_NUM].append(interfaces[i]) + + for interface_slice in interface_slices: + if interface_slice: + s = PacketSender(interface_slice, packet, options.num, options.send_pfc_frame_interval) + s.start() + senders.append(s) + + logger.debug(pre_str + '_STORM_START') + # Wait PFC packets to be sent + for sender in senders: + sender.stop() + else: + sender = PacketSender(interfaces, packet, options.num, options.send_pfc_frame_interval) + logger.debug(pre_str + '_STORM_START') + sender.send_packets() + logger.debug(pre_str + '_STORM_END') diff --git a/tests/common/helpers/pfc_storm.py b/tests/common/helpers/pfc_storm.py index 0f363bc32d..3571dd78b0 100644 --- a/tests/common/helpers/pfc_storm.py +++ b/tests/common/helpers/pfc_storm.py @@ -1,8 +1,10 @@ import logging import os +import re from jinja2 import Template from tests.common.errors import MissingInputError +from tests.common.devices.sonic import SonicHost TEMPLATES_DIR = os.path.realpath((os.path.join(os.path.dirname(__file__), "../../common/templates"))) ANSIBLE_ROOT = os.path.realpath((os.path.join(os.path.dirname(__file__), "../../../ansible"))) @@ -39,6 +41,7 @@ def __init__(self, duthost, fanout_graph_facts, fanouthosts, **kwargs): self.fanout_info = fanout_graph_facts self.fanout_hosts = fanouthosts self.pfc_gen_file = kwargs.pop('pfc_gen_file', "pfc_gen.py") + self.pfc_gen_multiprocess = kwargs.pop('pfc_gen_multiprocess', False) self.pfc_queue_idx = kwargs.pop('pfc_queue_index', 3) self.pfc_frames_number = kwargs.pop('pfc_frames_number', 100000) self.send_pfc_frame_interval = kwargs.pop('send_pfc_frame_interval', 0) @@ -51,6 +54,8 @@ def __init__(self, duthost, fanout_graph_facts, fanouthosts, **kwargs): self.update_platform_name() self._populate_optional_params(kwargs) self.peer_device = self.fanout_hosts[self.peer_info['peerdevice']] + self.fanout_asic_type = self.peer_device.facts['asic_type'] if isinstance(self.peer_device.host, SonicHost) \ + else None def _populate_peer_hwsku(self): """ @@ -78,6 +83,27 @@ def _populate_optional_params(self, kwargs): self.__dict__.update(kwargs) kwargs.clear() + def _generate_mellanox_lable_port_map(self): + """ + In SONiC the port alias contains mellanox label port + For example, 'etp20' contains label port '20' + This method is used to generate port name and label port map from fanout + """ + port_name_label_port_map = {} + show_int_status = self.fanout_hosts[self.peer_info['peerdevice']].host.show_and_parse("show interface status") + port_name_alias_map = {intf["interface"]: intf['alias'] for intf in show_int_status} + for port_name, alias in port_name_alias_map.items(): + label_port = re.findall(r'\d+[a-z]?', alias) + port_name_label_port_map[port_name] = label_port[0] + return port_name_label_port_map + + def _generate_mellanox_label_ports(self): + label_port_map = self._generate_mellanox_lable_port_map() + port_names = self.peer_info['pfc_fanout_interface'] + label_ports = [label_port_map[port_name] for port_name in port_names.split(',')] + + return ",".join(label_ports) + def _create_pfc_gen(self): """ Create the pfc generation file on the fanout if it does not exist @@ -93,11 +119,17 @@ def deploy_pfc_gen(self): Deploy the pfc generation file on the fanout """ if self.peer_device.os in ('eos', 'sonic'): + src_pfc_gen_file = "common/helpers/{}".format(self.pfc_gen_file) self._create_pfc_gen() + if self.fanout_asic_type == 'mellanox': + src_pfc_gen_file = f"../ansible/roles/test/files/mlnx/docker-tests-pfcgen-asic/{self.pfc_gen_file}" self.peer_device.copy( - src="common/helpers/{}".format(self.pfc_gen_file), + src=src_pfc_gen_file, dest=self._PFC_GEN_DIR[self.peer_device.os] ) + if self.fanout_asic_type == 'mellanox': + cmd = f"docker cp {self._PFC_GEN_DIR[self.peer_device.os]}/{self.pfc_gen_file} syncd:/root/" + self.peer_device.shell(cmd) def update_queue_index(self, q_idx): """ @@ -150,6 +182,10 @@ def _update_template_args(self): self.extra_vars.update({"pfc_storm_stop_defer_time": self.pfc_storm_stop_defer_time}) if getattr(self, "pfc_asym", None): self.extra_vars.update({"pfc_asym": self.pfc_asym}) + if self.fanout_asic_type == 'mellanox' and self.peer_device.os == 'sonic': + self.extra_vars.update({"pfc_fanout_label_port": self._generate_mellanox_label_ports()}) + if self.dut.facts['asic_type'] == "mellanox": + self.extra_vars.update({"pfc_gen_multiprocess": True}) def _prepare_start_template(self): """ @@ -159,6 +195,9 @@ def _prepare_start_template(self): if self.dut.topo_type == 't2' and self.peer_device.os == 'sonic': self.pfc_start_template = os.path.join( TEMPLATES_DIR, "pfc_storm_{}_t2.j2".format(self.peer_device.os)) + elif self.fanout_asic_type == 'mellanox' and self.peer_device.os == 'sonic': + self.pfc_start_template = os.path.join( + TEMPLATES_DIR, "pfc_storm_mlnx_{}.j2".format(self.peer_device.os)) else: self.pfc_start_template = os.path.join( TEMPLATES_DIR, "pfc_storm_{}.j2".format(self.peer_device.os)) @@ -172,6 +211,9 @@ def _prepare_stop_template(self): if self.dut.topo_type == 't2' and self.peer_device.os == 'sonic': self.pfc_stop_template = os.path.join( TEMPLATES_DIR, "pfc_storm_stop_{}_t2.j2".format(self.peer_device.os)) + elif self.fanout_asic_type == 'mellanox' and self.peer_device.os == 'sonic': + self.pfc_stop_template = os.path.join( + TEMPLATES_DIR, "pfc_storm_stop_mlnx_{}.j2".format(self.peer_device.os)) else: self.pfc_stop_template = os.path.join( TEMPLATES_DIR, "pfc_storm_stop_{}.j2".format(self.peer_device.os)) diff --git a/tests/pfcwd/files/pfcwd_helper.py b/tests/common/helpers/pfcwd_helper.py similarity index 81% rename from tests/pfcwd/files/pfcwd_helper.py rename to tests/common/helpers/pfcwd_helper.py index 4de4a99ce4..5b5ecd0c1e 100644 --- a/tests/pfcwd/files/pfcwd_helper.py +++ b/tests/common/helpers/pfcwd_helper.py @@ -4,6 +4,8 @@ import random import pytest import contextlib +import time +import logging from tests.ptf_runner import ptf_runner from tests.common import constants @@ -23,6 +25,8 @@ EXPECT_PFC_WD_RESTORE_RE = ".*storm restored.*" +logger = logging.getLogger(__name__) + class TrafficPorts(object): """ Generate a list of ports needed for the PFC Watchdog test""" @@ -104,21 +108,22 @@ def parse_intf_list(self): if item['peer_addr'] == pfc_wd_test_port_addr: pfc_wd_test_neighbor_addr = item['addr'] - self.test_ports[pfc_wd_test_port] = {'test_neighbor_addr': pfc_wd_test_neighbor_addr, - 'rx_port': [self.pfc_wd_rx_port], - 'rx_neighbor_addr': self.pfc_wd_rx_neighbor_addr, - 'peer_device': self.neighbors[pfc_wd_test_port]['peerdevice'], - 'test_port_id': pfc_wd_test_port_id, - 'rx_port_id': [self.pfc_wd_rx_port_id], - 'test_port_type': 'interface' - } + self.test_ports[pfc_wd_test_port] = { + 'test_neighbor_addr': pfc_wd_test_neighbor_addr, + 'rx_port': [self.pfc_wd_rx_port], + 'rx_neighbor_addr': self.pfc_wd_rx_neighbor_addr, + 'peer_device': self.neighbors.get(pfc_wd_test_port, {}).get('peerdevice', ''), + 'test_port_id': pfc_wd_test_port_id, + 'rx_port_id': [self.pfc_wd_rx_port_id], + 'test_port_type': 'interface' + } # populate info for the first port if first_pair: self.test_ports[self.pfc_wd_rx_port] = { 'test_neighbor_addr': self.pfc_wd_rx_neighbor_addr, 'rx_port': [pfc_wd_test_port], 'rx_neighbor_addr': pfc_wd_test_neighbor_addr, - 'peer_device': self.neighbors[self.pfc_wd_rx_port]['peerdevice'], + 'peer_device': self.neighbors.get(self.pfc_wd_rx_port, {}).get('peerdevice', ''), 'test_port_id': self.pfc_wd_rx_port_id, 'rx_port_id': [pfc_wd_test_port_id], 'test_port_type': 'interface' @@ -173,7 +178,7 @@ def parse_pc_list(self): self.test_ports[port] = {'test_neighbor_addr': pfc_wd_test_neighbor_addr, 'rx_port': self.pfc_wd_rx_port, 'rx_neighbor_addr': self.pfc_wd_rx_neighbor_addr, - 'peer_device': self.neighbors[port]['peerdevice'], + 'peer_device': self.neighbors.get(port, {}).get('peerdevice', ''), 'test_port_id': self.port_idx_info[port], 'rx_port_id': self.pfc_wd_rx_port_id, 'test_portchannel_members': pfc_wd_test_port_id, @@ -185,7 +190,7 @@ def parse_pc_list(self): self.test_ports[port] = {'test_neighbor_addr': self.pfc_wd_rx_neighbor_addr, 'rx_port': pfc_wd_test_port, 'rx_neighbor_addr': pfc_wd_test_neighbor_addr, - 'peer_device': self.neighbors[port]['peerdevice'], + 'peer_device': self.neighbors.get(port, {}).get('peerdevice', ''), 'test_port_id': self.port_idx_info[port], 'rx_port_id': pfc_wd_test_port_id, 'test_portchannel_members': self.pfc_wd_rx_port_id, @@ -222,7 +227,7 @@ def parse_vlan_list(self): temp_ports[item] = {'test_neighbor_addr': self.vlan_nw, 'rx_port': rx_port, 'rx_neighbor_addr': self.pfc_wd_rx_neighbor_addr, - 'peer_device': self.neighbors[item]['peerdevice'], + 'peer_device': self.neighbors.get(item, {}).get('peerdevice', ''), 'test_port_id': self.port_idx_info[item], 'rx_port_id': rx_port_id, 'test_port_type': 'vlan' @@ -267,23 +272,24 @@ def parse_vlan_sub_interface_list(self): if item['peer_addr'] == pfc_wd_test_port_addr: pfc_wd_test_neighbor_addr = item['addr'] - self.test_ports[pfc_wd_test_port] = {'test_neighbor_addr': pfc_wd_test_neighbor_addr, - 'rx_port': [self.pfc_wd_rx_port], - 'rx_neighbor_addr': self.pfc_wd_rx_neighbor_addr, - 'peer_device': self.neighbors[pfc_wd_test_port]['peerdevice'], - 'test_port_id': pfc_wd_test_port_id, - 'rx_port_id': [self.pfc_wd_rx_port_id], - 'rx_port_vlan_id': self.pfc_wd_rx_port_vlan_id, - 'test_port_vlan_id': vlan_id, - 'test_port_type': 'interface' - } + self.test_ports[pfc_wd_test_port] = { + 'test_neighbor_addr': pfc_wd_test_neighbor_addr, + 'rx_port': [self.pfc_wd_rx_port], + 'rx_neighbor_addr': self.pfc_wd_rx_neighbor_addr, + 'peer_device': self.neighbors.get(pfc_wd_test_port, {}).get('peerdevice', ''), + 'test_port_id': pfc_wd_test_port_id, + 'rx_port_id': [self.pfc_wd_rx_port_id], + 'rx_port_vlan_id': self.pfc_wd_rx_port_vlan_id, + 'test_port_vlan_id': vlan_id, + 'test_port_type': 'interface' + } # populate info for the first port if first_pair: self.test_ports[self.pfc_wd_rx_port] = { 'test_neighbor_addr': self.pfc_wd_rx_neighbor_addr, 'rx_port': [pfc_wd_test_port], 'rx_neighbor_addr': pfc_wd_test_neighbor_addr, - 'peer_device': self.neighbors[self.pfc_wd_rx_port]['peerdevice'], + 'peer_device': self.neighbors.get(self.pfc_wd_rx_port, {}).get('peerdevice', ''), 'test_port_id': self.pfc_wd_rx_port_id, 'rx_port_id': [pfc_wd_test_port_id], 'rx_port_vlan_id': vlan_id, @@ -489,6 +495,8 @@ def send_background_traffic(duthost, ptfhost, storm_hndle, selected_test_ports, selected_test_ports, test_ports_info) background_traffic_log = _send_background_traffic(ptfhost, background_traffic_params) + # Ensure the background traffic is running before moving on + time.sleep(1) yield if is_mellanox_device(duthost): _stop_background_traffic(ptfhost, background_traffic_log) @@ -510,7 +518,8 @@ def _prepare_background_traffic_params(duthost, queues, selected_test_ports, tes src_ips.append(selected_test_port_info["rx_neighbor_addr"]) router_mac = duthost.get_dut_iface_mac(selected_test_ports[0]) - pkt_count = 1000 + # Send enough packets to make sure the background traffic is running during the test + pkt_count = 100000 ptf_params = {'router_mac': router_mac, 'src_ports': src_ports, @@ -537,3 +546,81 @@ def _stop_background_traffic(ptfhost, background_traffic_log): pids = ptfhost.shell(f"pgrep -f {background_traffic_log}")["stdout_lines"] for pid in pids: ptfhost.shell(f"kill -9 {pid}", module_ignore_errors=True) + + +def has_neighbor_device(setup_pfc_test): + """ + Check if there are neighbor devices present + + Args: + setup_pfc_test (fixture): Module scoped autouse fixture for PFCwd + + Returns: + bool: True if there are neighbor devices present, False otherwise + """ + for _, details in setup_pfc_test['selected_test_ports'].items(): + # 'rx_port' and 'rx_port_id' are expected to be conjugate attributes + # if one is unset or contains None, the other should be as well + if (not details.get('rx_port') or None in details['rx_port']) or \ + (not details.get('rx_port_id') or None in details['rx_port_id']): + return False # neighbor devices are not present + return True + + +def check_pfc_storm_state(dut, port, queue, expected_state): + """ + Helper function to check if PFC storm is detected/restored on a given queue + """ + pfcwd_stat = parser_show_pfcwd_stat(dut, port, queue) + if expected_state == "storm": + if ("storm" in pfcwd_stat[0]['status']) and \ + int(pfcwd_stat[0]['storm_detect_count']) > int(pfcwd_stat[0]['restored_count']): + return True + else: + if ("storm" not in pfcwd_stat[0]['status']) and \ + int(pfcwd_stat[0]['storm_detect_count']) == int(pfcwd_stat[0]['restored_count']): + return True + return False + + +def parser_show_pfcwd_stat(dut, select_port, select_queue): + """ + CLI "show pfcwd stats" output: + admin@bjw-can-7060-1:~$ show pfcwd stats + QUEUE STATUS STORM DETECTED/RESTORED TX OK/DROP RX OK/DROP TX LAST OK/DROP RX LAST OK/DROP # noqa: E501 + ------------- -------- ------------------------- ------------ ------------ ----------------- ----------------- # noqa: E501 + Ethernet112:4 N/A 2/2 100/100 100/100 100/0 100/0 # noqa: E501 + admin@bjw-can-7060-1:~$ + """ + logger.info("port {} queue {}".format(select_port, select_queue)) + pfcwd_stat_output = dut.show_and_parse('show pfcwd stat') + + pfcwd_stat = [] + for item in pfcwd_stat_output: + port, queue = item['queue'].split(':') + if port != select_port or int(queue) != int(select_queue): + continue + storm_detect_count, restored_count = item['storm detected/restored'].split('/') + tx_ok_count, tx_drop_count = item['tx ok/drop'].split('/') + rx_ok_count, rx_drop_count = item['rx ok/drop'].split('/') + tx_last_ok_count, tx_last_drop_count = item['tx last ok/drop'].split('/') + rx_last_ok_count, rx_last_drop_count = item['rx last ok/drop'].split('/') + + parsed_dict = { + 'port': port, + 'queue': queue, + 'status': item['status'], + 'storm_detect_count': storm_detect_count, + 'restored_count': restored_count, + 'tx_ok_count': tx_ok_count, + 'tx_drop_count': tx_drop_count, + 'rx_ok_count': rx_ok_count, + 'rx_drop_count': rx_drop_count, + 'tx_last_ok_count': tx_last_ok_count, + 'tx_last_drop_count': tx_last_drop_count, + 'rx_last_ok_count': rx_last_ok_count, + 'rx_last_drop_count': rx_last_drop_count + } + pfcwd_stat.append(parsed_dict) + + return pfcwd_stat diff --git a/tests/common/helpers/portchannel_to_vlan.py b/tests/common/helpers/portchannel_to_vlan.py index 214244b97c..374a5d0cb7 100644 --- a/tests/common/helpers/portchannel_to_vlan.py +++ b/tests/common/helpers/portchannel_to_vlan.py @@ -422,3 +422,23 @@ def setup_po2vlan(duthosts, ptfhost, rand_one_dut_hostname, rand_selected_dut, p finally: config_reload(duthost, safe_reload=True) ptf_teardown(ptfhost, ptf_lag_map) + + +def has_portchannels(duthosts, rand_one_dut_hostname): + """ + Check if the ansible_facts contain non-empty portchannel interfaces and portchannels. + + Args: + duthosts: A dictionary that maps DUT hostnames to DUT instances. + rand_one_dut_hostname: A random hostname belonging to one of the DUT instances. + Returns: + bool: True if the ansible_facts contain non-empty portchannel interfaces and portchannels, False otherwise. + """ + duthost = duthosts[rand_one_dut_hostname] + # Retrieve the configuration facts from the DUT + cfg_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] + # Check if the portchannel interfaces list or portchannels dictionary is empty + if not cfg_facts.get("minigraph_portchannel_interfaces", []) or not cfg_facts.get("minigraph_portchannels", {}): + return False + + return True diff --git a/tests/common/helpers/ptf_tests_helper.py b/tests/common/helpers/ptf_tests_helper.py index 38a7a1b829..de6e8fbc5c 100644 --- a/tests/common/helpers/ptf_tests_helper.py +++ b/tests/common/helpers/ptf_tests_helper.py @@ -14,17 +14,18 @@ @pytest.fixture(scope="module") -def downstream_links(duthost, tbinfo): +def downstream_links(rand_selected_dut, tbinfo): """ Returns a dictionary of all the links that are downstream from the DUT. Args: - duthost: DUT fixture + rand_selected_dut: DUT fixture tbinfo: testbed information fixture Returns: links: Dictionary of links downstream from the DUT """ links = dict() + duthost = rand_selected_dut def filter(interface, neighbor, mg_facts, tbinfo): if ((tbinfo["topo"]["type"] == "t0" and "Server" in neighbor["name"]) @@ -41,18 +42,19 @@ def filter(interface, neighbor, mg_facts, tbinfo): @pytest.fixture(scope="module") -def upstream_links(duthost, tbinfo, nbrhosts): +def upstream_links(rand_selected_dut, tbinfo, nbrhosts): """ Returns a dictionary of all the links that are upstream from the DUT. Args: - duthost: DUT fixture + rand_selected_dut: DUT fixture tbinfo: testbed information fixture nbrhosts: neighbor host fixture Returns: links: Dictionary of links upstream from the DUT """ links = dict() + duthost = rand_selected_dut def filter(interface, neighbor, mg_facts, tbinfo): if ((tbinfo["topo"]["type"] == "t0" and "T1" in neighbor["name"]) diff --git a/tests/common/helpers/sonic_db.py b/tests/common/helpers/sonic_db.py index aa1bc854c5..f3c1ed6c96 100644 --- a/tests/common/helpers/sonic_db.py +++ b/tests/common/helpers/sonic_db.py @@ -1,3 +1,5 @@ +import ast +import ipaddress import logging import json import six @@ -93,6 +95,32 @@ def hget_key_value(self, key, field): else: return result['stdout'] + def hget_all(self, key): + """ + Executes a sonic-db-cli HGETALL command. + + Args: + key: full name of the key to get. + + Returns: + The corresponding value of the key. + Raises: + SonicDbKeyNotFound: If the key is not found. + """ + + cmd = self._cli_prefix() + "HGETALL {}".format(key) + result = self._run_and_check(cmd) + if result == {}: + raise SonicDbKeyNotFound("Key: %s not found in sonic-db cmd: %s" % (key, cmd)) + else: + v = None + if six.PY2: + v = result['stdout'].decode('unicode-escape') + else: + v = result['stdout'] + v_dict = ast.literal_eval(v) + return v_dict + def get_and_check_key_value(self, key, value, field=None): """ Executes a sonic-db CLI get or hget and validates the response against a provided field. @@ -140,7 +168,10 @@ def get_keys(self, table): if result == {}: raise SonicDbKeyNotFound("No keys for %s found in sonic-db cmd: %s" % (table, cmd)) else: - return result['stdout'].decode('unicode-escape') + if six.PY2: + return result['stdout'].decode('unicode-escape').splitlines() + else: + return result['stdout'].splitlines() def dump(self, table): """ @@ -192,6 +223,10 @@ class AsicDbCli(SonicDbCli): ASIC_LAG_MEMBER_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_LAG_MEMBER" ASIC_ROUTERINTF_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTER_INTERFACE" ASIC_NEIGH_ENTRY_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY" + ASIC_ROUTE_ENTRY_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY" + ASIC_NEXT_HOP_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP" + ASIC_ACL_ENTRY = "ASIC_STATE:SAI_OBJECT_TYPE_ACL_ENTRY" + ASIC_ACL_RANGE = "ASIC_STATE:SAI_OBJECT_TYPE_ACL_RANGE" def __init__(self, host): """ @@ -204,12 +239,19 @@ def __init__(self, host): self.system_port_key_list = [] self.port_key_list = [] self.lagid_key_list = [] + self.acl_entries = [] + self.acl_range_key_to_value = {} def get_switch_key(self): """Returns a list of keys in the switch table""" cmd = self._cli_prefix() + "KEYS %s*" % AsicDbCli.ASIC_SWITCH_TABLE return self._run_and_raise(cmd)["stdout_lines"][0] + def get_switch_key_value(self): + """Returns the value of the switch key""" + switch_key = self.get_switch_key() + return self.hget_all(switch_key) + def get_system_port_key_list(self, refresh=False): """Returns a list of keys in the system port table""" if self.system_port_key_list != [] and refresh is False: @@ -258,6 +300,166 @@ def get_neighbor_list(self): cmd = self._cli_prefix() + "KEYS %s:*" % AsicDbCli.ASIC_NEIGH_ENTRY_TABLE return self._run_and_raise(cmd)["stdout_lines"] + def get_route_entries(self): + """Returns a list of route entries""" + cmd = self._cli_prefix() + "KEYS %s:*" % AsicDbCli.ASIC_ROUTE_ENTRY_TABLE + return self._run_and_raise(cmd)['stdout_lines'] + + def get_route_entries_by_dest_ip(self, dest_ip=None): + """ + Returns a list of route entries based on destination IP address. + dest_ip can be a full IP address or partial address. With values + like '192.168.0.0/21', '192.168' or 'fc02:1000::' and so on. + """ + if dest_ip is None: + return self.get_route_entries() + + cmd = self._cli_prefix() + "KEYS %s:{\"dest\":\"%s*" % (AsicDbCli.ASIC_ROUTE_ENTRY_TABLE, dest_ip) + return self._run_and_raise(cmd)['stdout_lines'] + + def get_next_hop_entries(self): + """Returns all next hop entries""" + cmd = self._cli_prefix() + "KEYS %s:*" % AsicDbCli.ASIC_NEXT_HOP_TABLE + return self._run_and_raise(cmd)['stdout_lines'] + + def get_acl_entries(self, refresh=False): + """Returns all ACL entries""" + if self.acl_entries != [] and refresh is False: + return self.acl_entries + cmd = self._cli_prefix() + "KEYS %s:*" % AsicDbCli.ASIC_ACL_ENTRY + entry_keys = self._run_and_raise(cmd)['stdout_lines'] + for k in entry_keys: + self.acl_entries.append(self.hget_all(k)) + return self.acl_entries + + def get_acl_range_entries(self, refresh=False): + """Returns all ACL range entries""" + if self.acl_range_key_to_value and refresh is False: + return self.acl_range_key_to_value + cmd = self._cli_prefix() + "KEYS %s:*" % AsicDbCli.ASIC_ACL_RANGE + range_keys = self._run_and_raise(cmd)['stdout_lines'] + for k in range_keys: + self.acl_range_key_to_value[k] = self.hget_all(k) + return self.acl_range_key_to_value + + def find_acl_by(self, **kwargs): + """ + Returns ACL entries matching the following key values. Supported + keys include: ether_type, ip_protocol, tcp_flags, packet_action, + src_ip, dst_ip l4_src_port, l4_dst_port, range_type. + range_type recognizes 'l4_src_port' or 'l4_dst_port' values. A corresponding + l4_src_port or l4_dst_port value is expected. + + If more than one key is specified the find function assumes 'and' operation + to match an entry. + """ + + # kwargs keys to ACL_ENTRY field names + key_to_field_name = { + 'ether_type': 'SAI_ACL_ENTRY_ATTR_FIELD_ETHER_TYPE', + 'ip_protocol': 'SAI_ACL_ENTRY_ATTR_FIELD_IP_PROTOCOL', + 'ipv6_next_header': 'SAI_ACL_ENTRY_ATTR_FIELD_IPV6_NEXT_HEADER', + 'icmp_type': 'SAI_ACL_ENTRY_ATTR_FIELD_ICMP_TYPE', + 'icmp_code': 'SAI_ACL_ENTRY_ATTR_FIELD_ICMP_CODE', + 'icmpv6_type': 'SAI_ACL_ENTRY_ATTR_FIELD_ICMPV6_TYPE', + 'icmpv6_code': 'SAI_ACL_ENTRY_ATTR_FIELD_ICMPV6_CODE', + 'tcp_flags': 'SAI_ACL_ENTRY_ATTR_FIELD_TCP_FLAGS', + 'packet_action': 'SAI_ACL_ENTRY_ATTR_ACTION_PACKET_ACTION', + 'src_ip': 'SAI_ACL_ENTRY_ATTR_FIELD_SRC_IP', + 'src_ipv6': 'SAI_ACL_ENTRY_ATTR_FIELD_SRC_IPV6', + 'dst_ip': 'SAI_ACL_ENTRY_ATTR_FIELD_DST_IP', + 'dst_ipv6': 'SAI_ACL_ENTRY_ATTR_FIELD_DST_IPV6', + 'l4_src_port': 'SAI_ACL_ENTRY_ATTR_FIELD_L4_SRC_PORT', + 'l4_dst_port': 'SAI_ACL_ENTRY_ATTR_FIELD_L4_DST_PORT', + 'range_type': 'SAI_ACL_ENTRY_ATTR_FIELD_ACL_RANGE_TYPE' + } + + # ACL_RANGE keys + l4_src_port_range_key = 'SAI_ACL_RANGE_TYPE_L4_SRC_PORT_RANGE' + l4_dst_port_range_key = 'SAI_ACL_RANGE_TYPE_L4_DST_PORT_RANGE' + acl_range_attr_type_key = 'SAI_ACL_RANGE_ATTR_TYPE' + acl_range_limit_key = 'SAI_ACL_RANGE_ATTR_LIMIT' + + q = {} + for k, v in kwargs.items(): + if v is not None: + # reset key from src_ip -> src_ipv6, dst_ip -> dst_ipv6 + # based on IP version + if k == 'src_ip' or k == 'dst_ip': + ip_ver = ipaddress.ip_address(v).version + logger.debug(f'k is {k} and ip_ver = {ip_ver}') + if ip_ver == 6: + if k == 'src_ip': + q[key_to_field_name['src_ipv6']] = v + if k == 'dst_ip': + q[key_to_field_name['dst_ipv6']] = v + else: + q[key_to_field_name[k]] = v + continue + q[key_to_field_name[k]] = v + + if not q: + return [] + + if self.acl_entries == []: + logger.debug('Cannot find acl entry; empty acl_entries') + return [] + + l4_port, l4_port_key = '', '' + if key_to_field_name['range_type'] in q: + if key_to_field_name['l4_src_port'] not in q and key_to_field_name['l4_dst_port'] not in q: + raise ValueError('range type requires a src or dst port number') + if q[key_to_field_name['range_type']] == 'l4_src_port': + l4_port, l4_port_key = q[key_to_field_name['l4_src_port']], l4_src_port_range_key + del q[key_to_field_name['l4_src_port']] + elif q[key_to_field_name['range_type']] == 'l4_dst_port': + l4_port, l4_port_key = q[key_to_field_name['l4_dst_port']], l4_dst_port_range_key + del q[key_to_field_name['l4_dst_port']] + else: + raise ValueError('unknown range type') + logger.debug(f'ACL Range table {self.acl_range_key_to_value}') + + def range_matched(range_oid): + # uses l4_port and l4_port_key + k = f'ASIC_STATE:SAI_OBJECT_TYPE_ACL_RANGE:oid:{range_oid}' + v = self.acl_range_key_to_value.get(k) + if v is None: + return False + if v.get(acl_range_attr_type_key) != l4_port_key: + return False + start_port, end_port = v[acl_range_limit_key].split(',') + # l4_port can be hex starting with 0x or a decimal but not + # hex without 0x prefix + if int(l4_port, 0) >= int(start_port) and int(l4_port, 0) <= int(end_port): + return True + return False + + def entry_matched(entry): + # k - search condition, v - value passed + for k, v in q.items(): + entry_val = entry.get(k) + if entry_val is None: + return False + if k == key_to_field_name['range_type']: + range_oid = entry_val[entry_val.find('0x'):] + if range_matched(range_oid) is False: + return False + else: + # remove '&mask' + pos = entry_val.find('&') + n = len(entry_val) if pos == -1 else pos + v_clean = entry_val[:n] + if v != v_clean: + return False + return True + + entries = [] + for acl_entry in self.acl_entries: + if entry_matched(acl_entry): + entries.append(acl_entry) + + return entries + def get_neighbor_key_by_ip(self, ipaddr): """Returns the key in the neighbor table that is for a specific IP neighbor diff --git a/tests/common/helpers/syslog_helpers.py b/tests/common/helpers/syslog_helpers.py new file mode 100644 index 0000000000..d5d7351525 --- /dev/null +++ b/tests/common/helpers/syslog_helpers.py @@ -0,0 +1,194 @@ +import os +import logging +import pytest +import time +from scapy.all import rdpcap + +DUT_PCAP_FILEPATH = "/tmp/test_syslog_tcpdump.pcap" +DOCKER_TMP_PATH = "/tmp/" + +logger = logging.getLogger(__name__) + + +# Before real test, check default route on DUT: +# If DUT has no IPv4 and IPv6 default route, skip syslog test. If DUT has at least one type default route, +# tell test_syslog function to do further check +@pytest.fixture(scope="module") +def check_default_route(rand_selected_dut): + duthost = rand_selected_dut + ret = {'IPv4': False, 'IPv6': False} + + logger.info("Checking DUT default route") + result = duthost.shell("ip route show default table default | grep via", module_ignore_errors=True)['rc'] + if result == 0: + neigh_ip = duthost.shell( + "ip route show default table default | cut -d ' ' -f 3", module_ignore_errors=True)['stdout'] + result = duthost.shell( + "ip -4 neigh show {} | grep REACHABLE".format(neigh_ip), module_ignore_errors=True)['rc'] + if result == 0: + ret['IPv4'] = True + result = duthost.shell("ip -6 route show default table default | grep via", module_ignore_errors=True)['rc'] + if result == 0: + neigh_ip = duthost.shell( + "ip -6 route show default table default | cut -d ' ' -f 3", module_ignore_errors=True)['stdout'] + result = duthost.shell( + "ip -6 neigh show {} | grep REACHABLE".format(neigh_ip), module_ignore_errors=True)['rc'] + if result == 0: + ret['IPv6'] = True + + if not ret['IPv4'] and not ret['IPv6']: + pytest.skip("DUT has no default route, skiped") + + yield ret + + +# If any dummy IP type doesn't have a matching default route, skip test for this parametrize +def check_dummy_addr_and_default_route(dummy_ip_a, dummy_ip_b, has_v4_default_route, has_v6_default_route): + skip_v4 = False + skip_v6 = False + + if dummy_ip_a is not None and ":" not in dummy_ip_a and not has_v4_default_route: + skip_v4 = True + if dummy_ip_a is not None and ":" in dummy_ip_a and not has_v6_default_route: + skip_v6 = True + + if dummy_ip_b is not None and ":" not in dummy_ip_b and not has_v4_default_route: + skip_v4 = True + if dummy_ip_b is not None and ":" in dummy_ip_b and not has_v6_default_route: + skip_v6 = True + + if skip_v4 | skip_v6: + proto = "IPv4" if skip_v4 else "IPv6" + pytest.skip("DUT has no matching default route for dummy syslog ips: ({}, {}), has no {} default route" + .format(dummy_ip_a, dummy_ip_b, proto)) + + +# Check pcap file for the destination IPs +def _check_pcap(dummy_ip_a, dummy_ip_b, filepath): + is_ok_a = False + is_ok_b = False + + if dummy_ip_a is None: + is_ok_a = True + if dummy_ip_b is None: + is_ok_b = True + + packets = rdpcap(filepath) + for data in packets: + proto = "IPv6" if "IPv6" in data else "IP" + if is_ok_a is False and data[proto].dst == dummy_ip_a: + is_ok_a = True + if is_ok_b is False and data[proto].dst == dummy_ip_b: + is_ok_b = True + if is_ok_a and is_ok_b: + return True + + missed_ip = [] + if not is_ok_a: + missed_ip.append(dummy_ip_a) + if not is_ok_b: + missed_ip.append(dummy_ip_b) + logger.error("Pcap file doesn't contain dummy syslog ips: ({})".format(", ".join(missed_ip))) + return False + + +def run_syslog(rand_selected_dut, dummy_syslog_server_ip_a, dummy_syslog_server_ip_b, check_default_route): + duthost = rand_selected_dut + logger.info("Starting syslog tests") + test_message = "Basic Test Message" + + check_dummy_addr_and_default_route(dummy_syslog_server_ip_a, dummy_syslog_server_ip_b, + check_default_route['IPv4'], check_default_route['IPv6']) + + if dummy_syslog_server_ip_a: + if ":" not in dummy_syslog_server_ip_a: + duthost.command( + "sudo ip -4 rule add from all to {} pref 1 lookup default".format(dummy_syslog_server_ip_a)) + else: + duthost.command( + "sudo ip -6 rule add from all to {} pref 1 lookup default".format(dummy_syslog_server_ip_a)) + + if dummy_syslog_server_ip_b: + if ":" not in dummy_syslog_server_ip_b: + duthost.command( + "sudo ip -4 rule add from all to {} pref 2 lookup default".format(dummy_syslog_server_ip_b)) + else: + duthost.command( + "sudo ip -6 rule add from all to {} pref 2 lookup default".format(dummy_syslog_server_ip_b)) + + logger.info("Configuring the DUT") + # Add dummy rsyslog destination for testing + if dummy_syslog_server_ip_a is not None: + if "201911" in duthost.os_version and ":" in dummy_syslog_server_ip_a: + pytest.skip("IPv6 syslog server IP not supported on 201911") + duthost.shell("sudo config syslog add {}".format(dummy_syslog_server_ip_a)) + logger.debug("Added new rsyslog server IP {}".format(dummy_syslog_server_ip_a)) + if dummy_syslog_server_ip_b is not None: + if "201911" in duthost.os_version and ":" in dummy_syslog_server_ip_b: + pytest.skip("IPv6 syslog server IP not supported on 201911") + duthost.shell("sudo config syslog add {}".format(dummy_syslog_server_ip_b)) + logger.debug("Added new rsyslog server IP {}".format(dummy_syslog_server_ip_b)) + + logger.info("Start tcpdump") + # Make sure that the DUT_PCAP_FILEPATH dose not exist + duthost.shell("sudo rm -f {}".format(DUT_PCAP_FILEPATH)) + # Scapy doesn't support LINUX_SLL2 (Linux cooked v2), and tcpdump on Bullseye + # defaults to writing in that format when listening on any interface. Therefore, + # have it use LINUX_SLL (Linux cooked) instead. + tcpdump_task, tcpdump_result = duthost.shell( + "sudo timeout 20 tcpdump -y LINUX_SLL -i any -s0 -A -w {} \"udp and port 514\"" + .format(DUT_PCAP_FILEPATH), module_async=True) + # wait for starting tcpdump + time.sleep(5) + + logger.debug("Generating log message from DUT") + # Generate a syslog from the DUT + duthost.shell("logger --priority INFO {}".format(test_message)) + + # wait for stoping tcpdump + tcpdump_task.close() + tcpdump_task.join() + + # Remove the syslog configuration + if dummy_syslog_server_ip_a is not None: + duthost.shell("sudo config syslog del {}".format(dummy_syslog_server_ip_a)) + if ":" not in dummy_syslog_server_ip_a: + duthost.command( + "sudo ip -4 rule del from all to {} pref 1 lookup default".format(dummy_syslog_server_ip_a)) + else: + duthost.command( + "sudo ip -6 rule del from all to {} pref 1 lookup default".format(dummy_syslog_server_ip_a)) + + if dummy_syslog_server_ip_b is not None: + duthost.shell("sudo config syslog del {}".format(dummy_syslog_server_ip_b)) + if ":" not in dummy_syslog_server_ip_b: + duthost.command( + "sudo ip -4 rule del from all to {} pref 2 lookup default".format(dummy_syslog_server_ip_b)) + else: + duthost.command( + "sudo ip -6 rule del from all to {} pref 2 lookup default".format(dummy_syslog_server_ip_b)) + + duthost.fetch(src=DUT_PCAP_FILEPATH, dest=DOCKER_TMP_PATH) + filepath = os.path.join(DOCKER_TMP_PATH, duthost.hostname, DUT_PCAP_FILEPATH.lstrip(os.path.sep)) + + if not _check_pcap(dummy_syslog_server_ip_a, dummy_syslog_server_ip_b, filepath): + default_route_v4 = duthost.shell("ip route show default table default")['stdout'] + logger.debug("DUT's IPv4 default route:\n%s" % default_route_v4) + default_route_v6 = duthost.shell("ip -6 route show default table default")['stdout'] + logger.debug("DUT's IPv6 default route:\n%s" % default_route_v6) + syslog_config = duthost.shell("grep 'remote syslog server' -A 7 /etc/rsyslog.conf")['stdout'] + logger.debug("DUT's syslog server IPs:\n%s" % syslog_config) + + pytest.fail("Dummy syslog server IP not seen in the pcap file") + + +def is_mgmt_vrf_enabled(dut): + """ + Check mgmt vrf is enabled or not + + Args: + dut (SonicHost): The target device + Return: True or False + """ + show_mgmt_vrf = dut.command("show mgmt-vrf")["stdout"] + return "ManagementVRF : Disabled" not in show_mgmt_vrf diff --git a/tests/platform_tests/templates/ignore_boot_messages b/tests/common/helpers/tacacs/__init__.py similarity index 100% rename from tests/platform_tests/templates/ignore_boot_messages rename to tests/common/helpers/tacacs/__init__.py diff --git a/tests/tacacs/tacacs_creds.yaml b/tests/common/helpers/tacacs/tacacs_creds.yaml similarity index 100% rename from tests/tacacs/tacacs_creds.yaml rename to tests/common/helpers/tacacs/tacacs_creds.yaml diff --git a/tests/common/helpers/tacacs/tacacs_helper.py b/tests/common/helpers/tacacs/tacacs_helper.py new file mode 100644 index 0000000000..2c69667826 --- /dev/null +++ b/tests/common/helpers/tacacs/tacacs_helper.py @@ -0,0 +1,361 @@ +import os +import yaml +import pytest +import logging +import time +import crypt +import re +from tests.common.utilities import wait_until, check_skip_release, delete_running_config +from tests.common.helpers.assertions import pytest_assert +from tests.common.errors import RunAnsibleModuleFail +from contextlib import contextmanager + +# per-command accounting feature not available in following versions +per_command_accounting_skip_versions = ["201811", "201911", "202106"] +# per-command authorization feature not available in following versions +per_command_authorization_skip_versions = ["201811", "201911", "202012", "202106"] + +logger = logging.getLogger(__name__) + + +def load_tacacs_creds(): + TACACS_CREDS_FILE = 'tacacs_creds.yaml' + creds_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), TACACS_CREDS_FILE) + return yaml.safe_load(open(creds_file_path).read()) + + +def setup_local_user(duthost, tacacs_creds): + try: + duthost.shell("sudo deluser {}".format(tacacs_creds['local_user'])) + except RunAnsibleModuleFail: + logger.info("local user not exist") + + duthost.shell("sudo useradd {}".format(tacacs_creds['local_user'])) + duthost.shell('sudo echo "{}:{}" | chpasswd'.format(tacacs_creds['local_user'], tacacs_creds['local_user_passwd'])) + + +def setup_tacacs_client(duthost, tacacs_creds, tacacs_server_ip, + tacacs_server_passkey, ptfhost, authorization="local"): + """setup tacacs client""" + + # UT should failed when set reachable TACACS server with this setup_tacacs_client + retry = 5 + while retry > 0: + ping_result = duthost.shell("ping {} -c 1 -W 3".format(tacacs_server_ip), module_ignore_errors=True)['stdout'] + logger.info("TACACS server ping result: {}".format(ping_result)) + if "100% packet loss" in ping_result: + # collect more information for debug testbed network issue + duthost_interface = duthost.shell("sudo ifconfig eth0")['stdout'] + ptfhost_interface = ptfhost.shell("ifconfig mgmt")['stdout'] + logger.debug("PTF IPV6 address not reachable, dut interfaces: {}, ptfhost interfaces:{}" + .format(duthost_interface, ptfhost_interface)) + time.sleep(5) + retry -= 1 + else: + break + if retry == 0: + pytest_assert(False, "TACACS server not reachable: {}".format(ping_result)) + + # configure tacacs client + default_tacacs_servers = [] + duthost.shell("sudo config tacacs passkey %s" % tacacs_server_passkey) + + # get default tacacs servers + config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] + for tacacs_server in config_facts.get('TACPLUS_SERVER', {}): + duthost.shell("sudo config tacacs delete %s" % tacacs_server) + default_tacacs_servers.append(tacacs_server) + # setup TACACS server with port 59 + # Port 49 bind to another TACACS server for daily work and none TACACS test case + duthost.shell("sudo config tacacs add %s --port 59" % tacacs_server_ip) + duthost.shell("sudo config tacacs authtype login") + + # enable tacacs+ + duthost.shell("sudo config aaa authentication login tacacs+") + + (skip, _) = check_skip_release(duthost, per_command_authorization_skip_versions) + if not skip: + duthost.shell("sudo config aaa authorization {}".format(authorization)) + + (skip, _) = check_skip_release(duthost, per_command_accounting_skip_versions) + if not skip: + duthost.shell("sudo config aaa accounting disable") + + # setup local user + setup_local_user(duthost, tacacs_creds) + return default_tacacs_servers + + +def fix_symbolic_link_in_config(duthost, ptfhost, symbolic_link_path, path_to_be_fix=None): + """ + Fix symbolic link in tacacs config + Because tac_plus server not support regex in command name, and SONiC will send full path to tacacs server side + for authorization, so the 'python' and 'ld' path in tac_plus config file need fix. + """ + read_link_command = "readlink -f {0}".format(symbolic_link_path) + target_path = duthost.shell(read_link_command)['stdout'] + # Escape path string, will use it as regex in sed command. + + link_path_regex = re.escape(symbolic_link_path) + if path_to_be_fix is not None: + link_path_regex = re.escape(path_to_be_fix) + + target_path_regex = re.escape(target_path) + ptfhost.shell("sed -i 's|{0}|{1}|g' /etc/tacacs+/tac_plus.conf".format(link_path_regex, target_path_regex)) + + +def get_ld_path(duthost): + """ + Fix symbolic link in tacacs config + Because tac_plus server not support regex in command name, and SONiC will send full path to tacacs server side + for authorization, so the 'python' and 'ld' path in tac_plus config file need fix. + """ + find_ld_command = "find /lib/ -type f,l -regex '\/lib\/.*-linux-.*/ld-linux-.*\.so\.[0-9]*'" # noqa W605 + return duthost.shell(find_ld_command)['stdout'] + + +def fix_ld_path_in_config(duthost, ptfhost): + """ + Fix ld path in tacacs config + """ + ld_symbolic_link_path = get_ld_path(duthost) + if not ld_symbolic_link_path: + fix_symbolic_link_in_config(duthost, ptfhost, ld_symbolic_link_path, "/lib/arch-linux-abi/ld-linux-arch.so") + + +def check_all_services_status(ptfhost): + res = ptfhost.command("service --status-all") + logger.info(res["stdout_lines"]) + + +def setup_tacacs_server(ptfhost, tacacs_creds, duthost): + """setup tacacs server""" + + # configure tacacs server + extra_vars = {'tacacs_passkey': tacacs_creds[duthost.hostname]['tacacs_passkey'], + 'tacacs_rw_user': tacacs_creds['tacacs_rw_user'], + 'tacacs_rw_user_passwd': crypt.crypt(tacacs_creds['tacacs_rw_user_passwd'], 'abc'), + 'tacacs_ro_user': tacacs_creds['tacacs_ro_user'], + 'tacacs_ro_user_passwd': crypt.crypt(tacacs_creds['tacacs_ro_user_passwd'], 'abc'), + 'tacacs_authorization_user': tacacs_creds['tacacs_authorization_user'], + 'tacacs_authorization_user_passwd': crypt.crypt( + tacacs_creds['tacacs_authorization_user_passwd'], + 'abc'), + 'tacacs_jit_user': tacacs_creds['tacacs_jit_user'], + 'tacacs_jit_user_passwd': crypt.crypt(tacacs_creds['tacacs_jit_user_passwd'], 'abc'), + 'tacacs_jit_user_membership': tacacs_creds['tacacs_jit_user_membership']} + + dut_options = duthost.host.options['inventory_manager'].get_host(duthost.hostname).vars + dut_creds = tacacs_creds[duthost.hostname] + logger.debug("setup_tacacs_server: dut_options:{}".format(dut_options)) + if 'ansible_user' in dut_options and 'ansible_password' in dut_options: + duthost_admin_user = dut_options['ansible_user'] + duthost_admin_passwd = dut_options['ansible_password'] + logger.debug("setup_tacacs_server: update extra_vars with ansible_user and ansible_password.") + extra_vars['duthost_admin_user'] = duthost_admin_user + extra_vars['duthost_admin_passwd'] = crypt.crypt(duthost_admin_passwd, 'abc') + elif 'sonicadmin_user' in dut_creds and 'sonicadmin_password' in dut_creds: + logger.debug("setup_tacacs_server: update extra_vars with sonicadmin_user and sonicadmin_password.") + extra_vars['duthost_admin_user'] = dut_creds['sonicadmin_user'] + extra_vars['duthost_admin_passwd'] = crypt.crypt(dut_creds['sonicadmin_password'], 'abc') + elif 'sonicadmin_user' in dut_creds and 'ansible_altpasswords' in dut_creds: + logger.debug("setup_tacacs_server: update extra_vars with sonicadmin_user and ansible_altpasswords.") + extra_vars['duthost_admin_user'] = dut_creds['sonicadmin_user'] + extra_vars['duthost_admin_passwd'] = crypt.crypt(dut_creds['ansible_altpasswords'][0], 'abc') + else: + logger.debug("setup_tacacs_server: update extra_vars with sonic_login and sonic_password.") + extra_vars['duthost_admin_user'] = dut_creds['sonic_login'] + extra_vars['duthost_admin_passwd'] = crypt.crypt(dut_creds['sonic_password'], 'abc') + + if 'ansible_ssh_user' in dut_options and 'ansible_ssh_pass' in dut_options: + duthost_ssh_user = dut_options['ansible_ssh_user'] + duthost_ssh_passwd = dut_options['ansible_ssh_pass'] + logger.debug("setup_tacacs_server: update extra_vars with ansible_ssh_user and ansible_ssh_pass.") + extra_vars['duthost_ssh_user'] = duthost_ssh_user + extra_vars['duthost_ssh_passwd'] = crypt.crypt(duthost_ssh_passwd, 'abc') + else: + logger.debug("setup_tacacs_server: duthost options does not contains config for ansible_ssh_user.") + + ptfhost.host.options['variable_manager'].extra_vars.update(extra_vars) + ptfhost.template(src="tacacs/tac_plus.conf.j2", dest="/etc/tacacs+/tac_plus.conf") + + # Find 'python' command symbolic link target, and fix the tac_plus config file + fix_symbolic_link_in_config(duthost, ptfhost, "/usr/bin/python") + + # Find ld lib symbolic link target, and fix the tac_plus config file + fix_ld_path_in_config(duthost, ptfhost) + + # config TACACS+ to use debug flag: '-d 2058', so received data will write to /var/log/tac_plus.log + # config TACACS+ to use port 59: '-p 59', because 49 already running another tacacs server for daily work + ptfhost.lineinfile( + path="/etc/default/tacacs+", + line="DAEMON_OPTS=\"-d 2058 -l /var/log/tac_plus.log -C /etc/tacacs+/tac_plus.conf -p 59\"", + regexp='^DAEMON_OPTS=.*' + ) + + # config TACACS+ start script to check tac_plus.pid.59 + ptfhost.lineinfile( + path="/etc/init.d/tacacs_plus", + line="PIDFILE=/var/run/tac_plus.pid.59", + regexp='^PIDFILE=/var/run/tac_plus.*' + ) + check_all_services_status(ptfhost) + + # FIXME: This is a short term mitigation, we need to figure out why \nthe tacacs+ server does not start + # reliably all of a sudden. + wait_until(5, 1, 0, start_tacacs_server, ptfhost) + check_all_services_status(ptfhost) + + +def stop_tacacs_server(ptfhost): + def tacacs_not_running(ptfhost): + out = ptfhost.command("service tacacs_plus status", module_ignore_errors=True)["stdout"] + return "tacacs+ apparently not running" in out + ptfhost.shell("service tacacs_plus stop") + return wait_until(5, 1, 0, tacacs_not_running, ptfhost) + + +def remove_all_tacacs_server(duthost): + # use grep command to extract tacacs server address from tacacs config + find_server_command = 'show tacacs | grep -Po "TACPLUS_SERVER address \K.*"' # noqa W605 + server_list = duthost.shell(find_server_command, module_ignore_errors=True)['stdout_lines'] + for tacacs_server in server_list: + tacacs_server = tacacs_server.rstrip() + if tacacs_server: + duthost.shell("sudo config tacacs delete %s" % tacacs_server) + + +def cleanup_tacacs(ptfhost, tacacs_creds, duthost): + # stop tacacs server + stop_tacacs_server(ptfhost) + + # reset tacacs client configuration + remove_all_tacacs_server(duthost) + cmds = [ + "config tacacs default passkey", + "config aaa authentication login default", + "config aaa authentication failthrough default" + ] + duthost.shell_cmds(cmds=cmds) + + (skip, _) = check_skip_release(duthost, per_command_authorization_skip_versions) + if not skip: + duthost.shell("sudo config aaa authorization local") + + (skip, _) = check_skip_release(duthost, per_command_accounting_skip_versions) + if not skip: + duthost.shell("sudo config aaa accounting disable") + + duthost.user( + name=tacacs_creds['tacacs_ro_user'], state='absent', remove='yes', force='yes', module_ignore_errors=True + ) + duthost.user( + name=tacacs_creds['tacacs_rw_user'], state='absent', remove='yes', force='yes', module_ignore_errors=True + ) + duthost.user( + name=tacacs_creds['tacacs_jit_user'], state='absent', remove='yes', force='yes', module_ignore_errors=True + ) + + +def restore_tacacs_servers(duthost): + # Restore the TACACS plus server in config_db.json + config_facts = duthost.config_facts(host=duthost.hostname, source="persistent")["ansible_facts"] + for tacacs_server in config_facts.get("TACPLUS_SERVER", {}): + duthost.shell("sudo config tacacs add %s" % tacacs_server) + + cmds = [] + aaa_config = config_facts.get("AAA", {}) + if aaa_config: + cfg = aaa_config.get("authentication", {}).get("login", "") + if cfg: + cmds.append("sonic-db-cli CONFIG_DB hset 'AAA|authentication' login %s" % cfg) + + cfg = aaa_config.get("authentication", {}).get("failthrough", "") + if cfg.lower() == "true": + cmds.append("config aaa authentication failthrough enable") + elif cfg.lower() == "false": + cmds.append("config aaa authentication failthrough disable") + + cfg = aaa_config.get("authorization", {}).get("login", "") + if cfg: + cmds.append("sonic-db-cli CONFIG_DB hset 'AAA|authorization' login %s" % cfg) + + cfg = aaa_config.get("accounting", {}).get("login", "") + if cfg: + cmds.append("sonic-db-cli CONFIG_DB hset 'AAA|accounting' login %s" % cfg) + + tacplus_config = config_facts.get("TACPLUS", {}) + if tacplus_config: + cfg = tacplus_config.get("global", {}).get("auth_type", "") + if cfg: + cmds.append("config tacacs authtype %s" % cfg) + + cfg = tacplus_config.get("global", {}).get("passkey", "") + if cfg: + cmds.append("config tacacs passkey %s" % cfg) + + cfg = tacplus_config.get("global", {}).get("timeout", "") + if cfg: + cmds.append("config tacacs timeout %s" % cfg) + + # Cleanup AAA and TACPLUS config + delete_tacacs_json = [{"AAA": {}}, {"TACPLUS": {}}] + delete_running_config(delete_tacacs_json, duthost) + + # Restore AAA and TACPLUS config + duthost.shell_cmds(cmds=cmds) + + +@contextmanager +def _context_for_check_tacacs_v6(ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds): # noqa F811 + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + ptfhost_vars = ptfhost.host.options['inventory_manager'].get_host(ptfhost.hostname).vars + if 'ansible_hostv6' not in ptfhost_vars: + pytest.skip("Skip IPv6 test. ptf ansible_hostv6 not configured.") + tacacs_server_ip = ptfhost_vars['ansible_hostv6'] + tacacs_server_passkey = tacacs_creds[duthost.hostname]['tacacs_passkey'] + setup_tacacs_client(duthost, tacacs_creds, tacacs_server_ip, tacacs_server_passkey, ptfhost) + setup_tacacs_server(ptfhost, tacacs_creds, duthost) + + yield + + cleanup_tacacs(ptfhost, tacacs_creds, duthost) + restore_tacacs_servers(duthost) + + +@pytest.fixture(scope="function") +def check_tacacs_v6_func(ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds): # noqa F811 + with _context_for_check_tacacs_v6(ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds) as result: + yield result + + +def tacacs_running(ptfhost): + out = ptfhost.command("service tacacs_plus status", module_ignore_errors=True)["stdout"] + return "tacacs+ running" in out + + +def start_tacacs_server(ptfhost): + ptfhost.command("service tacacs_plus restart", module_ignore_errors=True) + return wait_until(5, 1, 0, tacacs_running, ptfhost) + + +def ssh_remote_run(localhost, remote_ip, username, password, cmd): + res = localhost.shell("sshpass -p {} ssh " + "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null " + "{}@{} {}".format(password, username, remote_ip, cmd), module_ignore_errors=True) + return res + + +def ssh_remote_run_retry(localhost, dutip, ptfhost, user, password, command, retry_count=3): + while retry_count > 0: + res = ssh_remote_run(localhost, dutip, user, + password, command) + + # TACACS server randomly crash after receive authorization request from IPV6 + if not tacacs_running(ptfhost): + start_tacacs_server(ptfhost) + retry_count -= 1 + else: + return res + + pytest_assert(False, "cat command failed because TACACS server not running") diff --git a/tests/common/helpers/telemetry_helper.py b/tests/common/helpers/telemetry_helper.py new file mode 100644 index 0000000000..2ae2611451 --- /dev/null +++ b/tests/common/helpers/telemetry_helper.py @@ -0,0 +1,128 @@ +import pytest +import logging +from contextlib import contextmanager +from tests.common.errors import RunAnsibleModuleFail +from tests.common.helpers.gnmi_utils import GNMIEnvironment +from tests.common.helpers.assertions import pytest_assert as py_assert +from tests.common.utilities import wait_until, get_mgmt_ipv6, wait_tcp_connection + +logger = logging.getLogger(__name__) + + +def check_gnmi_config(duthost): + cmd = 'sonic-db-cli CONFIG_DB HGET "GNMI|gnmi" port' + port = duthost.shell(cmd, module_ignore_errors=False)['stdout'] + return port != "" + + +def create_gnmi_config(duthost): + cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|gnmi' port 50052" + duthost.shell(cmd, module_ignore_errors=True) + cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|gnmi' client_auth true" + duthost.shell(cmd, module_ignore_errors=True) + cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|certs' "\ + "ca_crt /etc/sonic/telemetry/dsmsroot.cer" + duthost.shell(cmd, module_ignore_errors=True) + cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|certs' "\ + "server_crt /etc/sonic/telemetry/streamingtelemetryserver.cer" + duthost.shell(cmd, module_ignore_errors=True) + cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|certs' "\ + "server_key /etc/sonic/telemetry/streamingtelemetryserver.key" + duthost.shell(cmd, module_ignore_errors=True) + + +def delete_gnmi_config(duthost): + cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|gnmi' port" + duthost.shell(cmd, module_ignore_errors=True) + cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|gnmi' client_auth" + duthost.shell(cmd, module_ignore_errors=True) + cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|certs' ca_crt" + duthost.shell(cmd, module_ignore_errors=True) + cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|certs' server_crt" + duthost.shell(cmd, module_ignore_errors=True) + cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|certs' server_key" + duthost.shell(cmd, module_ignore_errors=True) + + +def setup_telemetry_forpyclient(duthost): + """ Set client_auth=false. This is needed for pyclient to successfully set up channel with gnmi server. + Restart telemetry process + """ + env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) + client_auth_out = duthost.shell('sonic-db-cli CONFIG_DB HGET "%s|gnmi" "client_auth"' % (env.gnmi_config_table), + module_ignore_errors=False)['stdout_lines'] + client_auth = str(client_auth_out[0]) + return client_auth + + +def restore_telemetry_forpyclient(duthost, default_client_auth): + env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) + client_auth_out = duthost.shell('sonic-db-cli CONFIG_DB HGET "%s|gnmi" "client_auth"' % (env.gnmi_config_table), + module_ignore_errors=False)['stdout_lines'] + client_auth = str(client_auth_out[0]) + if client_auth != default_client_auth: + duthost.shell('sonic-db-cli CONFIG_DB HSET "%s|gnmi" "client_auth" %s' + % (env.gnmi_config_table, default_client_auth), + module_ignore_errors=False) + duthost.shell("systemctl reset-failed %s" % (env.gnmi_container)) + duthost.service(name=env.gnmi_container, state="restarted") + + +@contextmanager +def _context_for_setup_streaming_telemetry(request, duthosts, enum_rand_one_per_hwsku_hostname, + localhost, ptfhost, gnxi_path): + """ + @summary: Post setting up the streaming telemetry before running the test. + """ + is_ipv6 = request.param + try: + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + has_gnmi_config = check_gnmi_config(duthost) + if not has_gnmi_config: + create_gnmi_config(duthost) + env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) + default_client_auth = setup_telemetry_forpyclient(duthost) + + if default_client_auth == "true": + duthost.shell('sonic-db-cli CONFIG_DB HSET "%s|gnmi" "client_auth" "false"' % (env.gnmi_config_table), + module_ignore_errors=False) + duthost.shell("systemctl reset-failed %s" % (env.gnmi_container)) + duthost.service(name=env.gnmi_container, state="restarted") + else: + logger.info('client auth is false. No need to restart telemetry') + + # Wait until telemetry was restarted + py_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, env.gnmi_container), + "%s not started." % (env.gnmi_container)) + logger.info("telemetry process restarted. Now run pyclient on ptfdocker") + + # Wait until the TCP port was opened + dut_ip = duthost.mgmt_ip + if is_ipv6: + dut_ip = get_mgmt_ipv6(duthost) + wait_tcp_connection(localhost, dut_ip, env.gnmi_port, timeout_s=60) + + # pyclient should be available on ptfhost. If it was not available, then fail pytest. + if is_ipv6: + cmd = "docker cp %s:/usr/sbin/gnmi_get ~/" % (env.gnmi_container) + ret = duthost.shell(cmd)['rc'] + py_assert(ret == 0) + else: + file_exists = ptfhost.stat(path=gnxi_path + "gnmi_cli_py/py_gnmicli.py") + py_assert(file_exists["stat"]["exists"] is True) + except RunAnsibleModuleFail as e: + logger.info("Error happens in the setup period of setup_streaming_telemetry, recover the telemetry.") + restore_telemetry_forpyclient(duthost, default_client_auth) + raise e + + yield + restore_telemetry_forpyclient(duthost, default_client_auth) + if not has_gnmi_config: + delete_gnmi_config(duthost) + + +@pytest.fixture(scope="function") +def setup_streaming_telemetry_func(request, duthosts, enum_rand_one_per_hwsku_hostname, localhost, ptfhost, gnxi_path): + with _context_for_setup_streaming_telemetry(request, duthosts, enum_rand_one_per_hwsku_hostname, + localhost, ptfhost, gnxi_path) as result: + yield result diff --git a/tests/platform_tests/thermal_control_test_helper.py b/tests/common/helpers/thermal_control_test_helper.py similarity index 96% rename from tests/platform_tests/thermal_control_test_helper.py rename to tests/common/helpers/thermal_control_test_helper.py index d6bf3148c2..9f1dab9e21 100644 --- a/tests/platform_tests/thermal_control_test_helper.py +++ b/tests/common/helpers/thermal_control_test_helper.py @@ -114,13 +114,12 @@ def _create_mocker(dut, mocker_name): mocker_object = None if 'mlnx' in platform or 'nvidia' in platform: - from tests.platform_tests.mellanox import mellanox_thermal_control_test_helper # noqa F401 mocker_type = BaseMocker.get_mocker_type(mocker_name) if mocker_type: mocker_object = mocker_type(dut) mockers.append(mocker_object) else: - pytest.skip("No mocker defined for this platform %s") + pytest.skip("No mocker defined for this platform {}".format(platform)) return mocker_object yield _create_mocker @@ -283,7 +282,8 @@ def restart_thermal_control_daemon(dut): 'Restarting thermal control daemon on {}...'.format(dut.hostname)) find_thermalctld_pid_cmd = 'docker exec -i pmon bash -c \'pgrep -f thermalctld\' | sort' output = dut.shell(find_thermalctld_pid_cmd) - assert output["rc"] == 0, "Run command '%s' failed" % find_thermalctld_pid_cmd + + assert output["rc"] == 0, "Run command '{}' failed".format(find_thermalctld_pid_cmd) # Usually there should be 2 thermalctld processes, but there is chance that # sonic platform API might use subprocess which creates extra thermalctld process. # For example, chassis.get_all_sfps will call sfp constructor, and sfp constructor may @@ -291,15 +291,18 @@ def restart_thermal_control_daemon(dut): # So we check here thermalcltd must have at least 2 processes. # For mellanox, it has at least two processes, but for celestica(broadcom), # it only has one thermalctld process + # For kvm, there is no thermalctld process if dut.facts["asic_type"] == "mellanox": assert len(output["stdout_lines"] ) >= 2, "There should be at least 2 thermalctld process" + elif dut.facts["asic_type"] == "vs": + assert len(output["stdout_lines"]) == 0, "There should be 0 thermalctld process" else: assert len(output["stdout_lines"] ) >= 1, "There should be at least 1 thermalctld process" restart_thermalctl_cmd = "docker exec -i pmon bash -c 'supervisorctl restart thermalctld'" - output = dut.shell(restart_thermalctl_cmd) + output = dut.shell(restart_thermalctl_cmd, module_ignore_errors=True) if output["rc"] == 0: output = dut.shell(find_thermalctld_pid_cmd) assert output["rc"] == 0, "Run command '{}' failed after restart of thermalctld on {}".format( @@ -313,6 +316,8 @@ def restart_thermal_control_daemon(dut): logging.info( "thermalctld processes restarted successfully on {}".format(dut.hostname)) return + if output["rc"] == 1 and dut.facts["asic_type"] == "vs": + return # try restore by config reload... config_reload(dut) assert 0, 'Wait thermal control daemon restart failed' diff --git a/tests/upgrade_path/upgrade_helpers.py b/tests/common/helpers/upgrade_helpers.py similarity index 81% rename from tests/upgrade_path/upgrade_helpers.py rename to tests/common/helpers/upgrade_helpers.py index a15999850c..9600462412 100644 --- a/tests/upgrade_path/upgrade_helpers.py +++ b/tests/common/helpers/upgrade_helpers.py @@ -10,7 +10,7 @@ from tests.common.reboot import get_reboot_cause, reboot_ctrl_dict from tests.common.reboot import REBOOT_TYPE_WARM, REBOOT_TYPE_COLD from tests.common.utilities import wait_until, setup_ferret -from tests.platform_tests.verify_dut_health import check_neighbors +from tests.common.platform.device_utils import check_neighbors SYSTEM_STABILIZE_MAX_TIME = 300 logger = logging.getLogger(__name__) @@ -219,3 +219,48 @@ def upgrade_test_helper(duthost, localhost, ptfhost, from_image, to_image, if enable_cpa and "warm-reboot" in reboot_type: ptfhost.shell('supervisorctl stop ferret') + + +def check_asic_and_db_consistency(pytest_config, duthost, consistency_checker_provider): + if not pytest_config.getoption("enable_consistency_checker"): + logger.info("Consistency checker is not enabled. Skipping check.") + return + + os_version = duthost.image_facts()["ansible_facts"]["ansible_image_facts"]["current"] + if not consistency_checker_provider.is_consistency_check_supported(duthost): + logger.info((f"Consistency check is not supported on this platform ({duthost.facts['platform']}) and " + f"version ({os_version})")) + return + + consistency_checker_libsairedis_url_template = pytest_config.getoption( + "consistency_checker_libsairedis_url_template") + consistency_checker_python3_pysairedis_url_template = pytest_config.getoption( + "consistency_checker_python3_pysairedis_url_template") + + if consistency_checker_libsairedis_url_template or consistency_checker_python3_pysairedis_url_template: + if "202305" in os_version: + sonic_version_template_param = "202305" + elif "202311" in os_version: + sonic_version_template_param = "202311" + else: + raise Exception(f"Unsupported OS version: {os_version}") + + libsairedis_download_url = consistency_checker_libsairedis_url_template\ + .format(sonic_version=sonic_version_template_param)\ + if consistency_checker_libsairedis_url_template else None + + python3_pysairedis_download_url = consistency_checker_python3_pysairedis_url_template\ + .format(sonic_version=sonic_version_template_param)\ + if consistency_checker_python3_pysairedis_url_template else None + + with consistency_checker_provider.get_consistency_checker(duthost, libsairedis_download_url, + python3_pysairedis_download_url) as consistency_checker: + keys = [ + "ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_POOL:*", + "ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_PROFILE:*", + "ASIC_STATE:SAI_OBJECT_TYPE_PORT:*", + "ASIC_STATE:SAI_OBJECT_TYPE_SWITCH:*", + "ASIC_STATE:SAI_OBJECT_TYPE_WRED:*", + ] + inconsistencies = consistency_checker.check_consistency(keys) + logger.warning(f"Found ASIC_DB and ASIC inconsistencies: {inconsistencies}") diff --git a/tests/voq/voq_helpers.py b/tests/common/helpers/voq_helpers.py similarity index 100% rename from tests/voq/voq_helpers.py rename to tests/common/helpers/voq_helpers.py diff --git a/tests/common/helpers/yaml_utils.py b/tests/common/helpers/yaml_utils.py new file mode 100644 index 0000000000..988422646f --- /dev/null +++ b/tests/common/helpers/yaml_utils.py @@ -0,0 +1,26 @@ +import yaml +from yaml import Dumper +from yaml.representer import Representer + +# PyYaml default dump the key with empty values to null, like: +# dict = {"key1": None, "key2": "val2"} -> +# key1: null +# key2: val2 +# If we want to keep it as the blank values rather than null, like: +# key1: +# key2: val2 +# Need to use this Representer +# refs to: https://stackoverflow.com/a/67524482/25406083 + + +class BlankNone(Representer): + """Print None as blank when used as context manager""" + def represent_none(self, *_): + return self.represent_scalar(u'tag:yaml.org,2002:null', u'') + + def __enter__(self): + self.prior = Dumper.yaml_representers[type(None)] + yaml.add_representer(type(None), self.represent_none) + + def __exit__(self, exc_type, exc_val, exc_tb): + Dumper.yaml_representers[type(None)] = self.prior diff --git a/tests/common/mellanox_data.py b/tests/common/mellanox_data.py index c7f9aa6b18..2e08915ebe 100644 --- a/tests/common/mellanox_data.py +++ b/tests/common/mellanox_data.py @@ -5,8 +5,8 @@ "ACS-MSN2010", "ACS-SN2201"] SPC2_HWSKUS = ["ACS-MSN3700", "ACS-MSN3700C", "ACS-MSN3800", "Mellanox-SN3800-D112C8", "ACS-MSN3420"] SPC3_HWSKUS = ["ACS-MSN4700", "Mellanox-SN4700-O28", "ACS-MSN4600C", "ACS-MSN4410", "ACS-MSN4600", - "Mellanox-SN4600C-D112C8", "Mellanox-SN4600C-C64"] -SPC4_HWSKUS = ["ACS-SN5600"] + "Mellanox-SN4600C-D112C8", "Mellanox-SN4600C-C64", "ACS-SN4280", "Mellanox-SN4280-O28"] +SPC4_HWSKUS = ["ACS-SN5600", "Mellanox-SN5600-V256"] SWITCH_HWSKUS = SPC1_HWSKUS + SPC2_HWSKUS + SPC3_HWSKUS + SPC4_HWSKUS PSU_CAPABILITIES = [ @@ -621,6 +621,61 @@ } } }, + "x86_64-nvidia_sn4280-r0": { + "chip_type": "spectrum3", + "reboot": { + "cold_reboot": True, + "fast_reboot": False, + "warm_reboot": True + }, + "fans": { + "number": 4, + "hot_swappable": True + }, + "psus": { + "number": 2, + "hot_swappable": True, + "capabilities": PSU_CAPABILITIES[1] + }, + "cpu_pack": { + "number": 1 + }, + "cpu_cores": { + "number": 0 + }, + "ports": { + "number": 32 + }, + "thermals": { + "cpu_core": { + "start": 0, + "number": 0 + }, + "module": { + "start": 1, + "number": 28 + }, + "psu": { + "start": 1, + "number": 2 + }, + "cpu_pack": { + "number": 1 + }, + "asic_ambient": { + "number": 1 + }, + "port_ambient": { + "number": 1 + }, + "fan_ambient": { + "number": 1 + }, + "comex_ambient": { + "number": 1 + } + } + }, "x86_64-nvidia_sn4280_simx-r0": { "chip_type": "spectrum3", "reboot": { diff --git a/tests/common/platform/args/__init__.py b/tests/common/platform/args/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/platform_tests/args/advanced_reboot_args.py b/tests/common/platform/args/advanced_reboot_args.py similarity index 74% rename from tests/platform_tests/args/advanced_reboot_args.py rename to tests/common/platform/args/advanced_reboot_args.py index 29ca7c8a1d..cf65d29901 100644 --- a/tests/platform_tests/args/advanced_reboot_args.py +++ b/tests/common/platform/args/advanced_reboot_args.py @@ -1,5 +1,5 @@ from tests.common.utilities import str2bool -from tests.platform_tests.warmboot_sad_cases import SAD_CASE_LIST +from tests.common.platform.warmboot_sad_cases import SAD_CASE_LIST def add_advanced_reboot_args(parser): @@ -154,3 +154,30 @@ def add_advanced_reboot_args(parser): "--enable_cpa", action="store_true", help="Enable control-plane assistant (only applicable for warm upgrade)") + + parser.addoption( + "--enable_consistency_checker", + action="store_true", + default=False, + help="Enables the consistency checker between the ASIC_DB and ASIC itself for the test" + ) + + parser.addoption( + "--consistency_checker_libsairedis_url_template", + default="", + help="Optional URL template for downloading and using an alternative version of libsairedis deb package " + + "during the consistency check. E.g. " + + "http://build-server.example/sonic-buildimage/{sonic_version}/debs/libsairedis_1.0.0_amd64.deb " + + "sonic_version is a template token that will be replaced with the actual sonic version of the device under " + + "test. e.g. 202311" + ) + + parser.addoption( + "--consistency_checker_python3_pysairedis_url_template", + default="", + help="Optional URL template for downloading and using an alternative version of python3-pysairedis deb " + + "package during the consistency check. E.g. " + + "http://build-server.example/sonic-buildimage/{sonic_version}/debs/python3-pysairedis_1.0.0_amd64.deb " + + "sonic_version is a template token that will be replaced with the actual sonic version of the device under " + + "test. e.g. 202311" + ) diff --git a/tests/platform_tests/args/cont_warm_reboot_args.py b/tests/common/platform/args/cont_warm_reboot_args.py similarity index 100% rename from tests/platform_tests/args/cont_warm_reboot_args.py rename to tests/common/platform/args/cont_warm_reboot_args.py diff --git a/tests/platform_tests/args/normal_reboot_args.py b/tests/common/platform/args/normal_reboot_args.py similarity index 100% rename from tests/platform_tests/args/normal_reboot_args.py rename to tests/common/platform/args/normal_reboot_args.py diff --git a/tests/common/platform/device_utils.py b/tests/common/platform/device_utils.py index 9ab56ca6d8..459631c751 100644 --- a/tests/common/platform/device_utils.py +++ b/tests/common/platform/device_utils.py @@ -1,10 +1,50 @@ import re +import time +import logging +import pytest +import traceback +import os +import json +import glob +from datetime import datetime +from collections import OrderedDict +from tests.common.utilities import wait_until +from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.dut_ports import encode_dut_port_name +from tests.common.platform.transceiver_utils import parse_transceiver_info +from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer +from tests.common.broadcom_data import is_broadcom_device +from tests.common.mellanox_data import is_mellanox_device +from tests.common.platform.reboot_timing_constants import SERVICE_PATTERNS, OTHER_PATTERNS, SAIREDIS_PATTERNS, \ + OFFSET_ITEMS, TIME_SPAN_ITEMS, REQUIRED_PATTERNS """ Helper script for fanout switch operations """ +logger = logging.getLogger(__name__) + + +LOGS_ON_TMPFS_PLATFORMS = [ + "x86_64-arista_7050_qx32", + "x86_64-arista_7050_qx32s", + "x86_64-arista_7060_cx32s", + "x86_64-arista_7260cx3_64", + "x86_64-arista_7050cx3_32s", + "x86_64-mlnx_msn2700-r0", + "x86_64-dell_s6100_c2538-r0", + "armhf-nokia_ixs7215_52x-r0" +] + +TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates") + +FMT = "%b %d %H:%M:%S.%f" +FMT_YEAR = "%Y %b %d %H:%M:%S.%f" +FMT_SHORT = "%b %d %H:%M:%S" +FMT_ALT = "%Y-%m-%dT%H:%M:%S.%f%z" + +test_report = dict() + def fanout_switch_port_lookup(fanout_switches, dut_name, dut_port): """ @@ -74,3 +114,820 @@ def list_dut_fanout_connections(dut, fanouthosts): candidates.append((dut_port, fanout, fanout_port)) return candidates + + +def watch_system_status(dut): + """ + Watch DUT's system status + + Args: + dut: DUT host object + """ + # Watch memory status + memory_output = dut.shell("show system-memory")["stdout"] + logger.info("Memory Status: %s", memory_output) + + # Watch orchagent CPU utilization + orch_cpu = dut.shell("show processes cpu | grep orchagent | awk '{print $9}'")["stdout"] + logger.info("Orchagent CPU Util: %s", orch_cpu) + + # Watch Redis Memory + redis_memory = dut.shell("redis-cli info memory | grep used_memory_human")["stdout"] + logger.info("Redis Memory: %s", redis_memory) + + +def __get_dut_if_status(dut, ifname=None): + """ + Get interface status on the DUT. + + Args: + dut: DUT host object + ifname: Interface of DUT + exp_state: State of DUT's port ('up' or 'down') + verbose: Logging port state. + + Returns: + Interface state + """ + if not ifname: + status = dut.show_interface(command='status')['ansible_facts']['int_status'] + else: + status = dut.show_interface(command='status', interfaces=[ifname])['ansible_facts']['int_status'] + return status + + +def __check_if_status(dut, dut_port, exp_state, verbose=False): + """ + Check interface status on the DUT. + + Args: + dut: DUT host object + dut_port: Port of DUT + exp_state: State of DUT's port ('up' or 'down') + verbose: Logging port state. + + Returns: + Bool value which confirm port state + """ + status = __get_dut_if_status(dut, dut_port)[dut_port] + if verbose: + logger.debug("Interface status : %s", status) + return status['oper_state'] == exp_state + + +def toggle_one_link(dut, dut_port, fanout, fanout_port, watch=False, check_status=True): + """ + Toggle one link on the fanout. + + Args: + dut: DUT host object + dut_port: Port of DUT + fanout: Fanout host object + fanout_port: Port of fanout + watch: Logging system state + """ + + sleep_time = 90 + logger.info("Testing link flap on %s", dut_port) + if check_status: + pytest_assert(__check_if_status(dut, dut_port, 'up', verbose=True), + "Fail: dut port {}: link operational down".format(dut_port)) + + logger.info("Shutting down fanout switch %s port %s connecting to %s", fanout.hostname, fanout_port, dut_port) + + need_recovery = True + try: + fanout.shutdown(fanout_port) + if check_status: + pytest_assert(wait_until(sleep_time, 1, 0, __check_if_status, dut, dut_port, 'down', True), + "dut port {} didn't go down as expected".format(dut_port)) + + if watch: + time.sleep(1) + watch_system_status(dut) + + logger.info("Bring up fanout switch %s port %s connecting to %s", fanout.hostname, fanout_port, dut_port) + fanout.no_shutdown(fanout_port) + need_recovery = False + + if check_status: + pytest_assert(wait_until(sleep_time, 1, 0, __check_if_status, dut, dut_port, 'up', True), + "dut port {} didn't go up as expected".format(dut_port)) + finally: + if need_recovery: + fanout.no_shutdown(fanout_port) + if check_status: + wait_until(sleep_time, 1, 0, __check_if_status, dut, dut_port, 'up', True) + + +class RebootHealthError(Exception): + def __init__(self, message): + self.message = message + super(RebootHealthError, self).__init__(message) + + +def handle_test_error(health_check): + def _wrapper(*args, **kwargs): + try: + health_check(*args, **kwargs) + except RebootHealthError as err: + # set result to fail + logging.error("Health check {} failed with {}".format( + health_check.__name__, err.message)) + test_report[health_check.__name__] = err.message + return + except Exception as err: + traceback.print_exc() + logging.error("Health check {} failed with unknown error: {}".format( + health_check.__name__, str(err))) + test_report[health_check.__name__] = "Unkown error" + return + # set result to pass + test_report[health_check.__name__] = True + return _wrapper + + +@handle_test_error +def check_services(duthost): + """ + Perform a health check of services + """ + logging.info("Wait until all critical services are fully started") + if not wait_until(330, 30, 0, duthost.critical_services_fully_started): + raise RebootHealthError("dut.critical_services_fully_started is False") + + logging.info("Check critical service status") + for service in duthost.critical_services: + status = duthost.get_service_props(service) + if status["ActiveState"] != "active": + raise RebootHealthError("ActiveState of {} is {}, expected: active".format( + service, status["ActiveState"])) + if status["SubState"] != "running": + raise RebootHealthError( + "SubState of {} is {}, expected: running".format(service, status["SubState"])) + + +@handle_test_error +def check_interfaces_and_transceivers(duthost, request): + """ + Perform a check of transceivers, LAGs and interfaces status + @param dut: The AnsibleHost object of DUT. + @param interfaces: DUT's interfaces defined by minigraph + """ + logging.info("Check if all the interfaces are operational") + check_interfaces = request.getfixturevalue("check_interfaces") + conn_graph_facts = request.getfixturevalue("conn_graph_facts") + results = check_interfaces() + failed = [ + result for result in results if "failed" in result and result["failed"]] + if failed: + raise RebootHealthError( + "Interface check failed, not all interfaces are up. Failed: {}".format(failed)) + + # Skip this step for virtual testbed - KVM testbed has transeivers marked as "Not present" + # and the DB returns an "empty array" for "keys TRANSCEIVER_INFO*" + if duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0': + return + + logging.info( + "Check whether transceiver information of all ports are in redis") + xcvr_info = duthost.command("redis-cli -n 6 keys TRANSCEIVER_INFO*") + parsed_xcvr_info = parse_transceiver_info(xcvr_info["stdout_lines"]) + interfaces = conn_graph_facts["device_conn"][duthost.hostname] + for intf in interfaces: + if intf not in parsed_xcvr_info: + raise RebootHealthError( + "TRANSCEIVER INFO of {} is not found in DB".format(intf)) + + +@handle_test_error +def check_neighbors(duthost, tbinfo): + """ + Perform a BGP neighborship check. + """ + logging.info("Check BGP neighbors status. Expected state - established") + bgp_facts = duthost.bgp_facts()['ansible_facts'] + mg_facts = duthost.get_extended_minigraph_facts(tbinfo) + + for value in list(bgp_facts['bgp_neighbors'].values()): + # Verify bgp sessions are established + if value['state'] != 'established': + raise RebootHealthError("BGP session not established") + # Verify locat ASNs in bgp sessions + if (value['local AS'] != mg_facts['minigraph_bgp_asn']): + raise RebootHealthError("Local ASNs not found in BGP session.\ + Minigraph: {}. Found {}".format(value['local AS'], mg_facts['minigraph_bgp_asn'])) + for v in mg_facts['minigraph_bgp']: + # Compare the bgp neighbors name with minigraph bgp neigbhors name + if (v['name'] != bgp_facts['bgp_neighbors'][v['addr'].lower()]['description']): + raise RebootHealthError("BGP neighbor's name does not match minigraph.\ + Minigraph: {}. Found {}".format(v['name'], + bgp_facts['bgp_neighbors'][v['addr'].lower()]['description'])) + # Compare the bgp neighbors ASN with minigraph + if (v['asn'] != bgp_facts['bgp_neighbors'][v['addr'].lower()]['remote AS']): + raise RebootHealthError("BGP neighbor's ASN does not match minigraph.\ + Minigraph: {}. Found {}".format(v['asn'], bgp_facts['bgp_neighbors'][v['addr'].lower()]['remote AS'])) + + +@handle_test_error +def verify_no_coredumps(duthost, pre_existing_cores): + if "20191130" in duthost.os_version: + coredumps_count = duthost.shell( + 'ls /var/core/ | grep -v python | wc -l')['stdout'] + else: + coredumps_count = duthost.shell('ls /var/core/ | wc -l')['stdout'] + if int(coredumps_count) > int(pre_existing_cores): + raise RebootHealthError("Core dumps found. Expected: {} Found: {}".format(pre_existing_cores, + coredumps_count)) + + +@pytest.fixture +def verify_dut_health(request, duthosts, rand_one_dut_hostname, tbinfo): + global test_report + test_report = {} + duthost = duthosts[rand_one_dut_hostname] + check_services(duthost) + check_interfaces_and_transceivers(duthost, request) + check_neighbors(duthost, tbinfo) + if "20191130" in duthost.os_version: + pre_existing_cores = duthost.shell( + 'ls /var/core/ | grep -v python | wc -l')['stdout'] + else: + pre_existing_cores = duthost.shell('ls /var/core/ | wc -l')['stdout'] + check_all = all([check is True for check in list(test_report.values())]) + pytest_assert(check_all, "DUT not ready for test. Health check failed before reboot: {}".format(test_report)) + + yield + + test_report = {} + check_services(duthost) + check_interfaces_and_transceivers(duthost, request) + check_neighbors(duthost, tbinfo) + verify_no_coredumps(duthost, pre_existing_cores) + check_all = all([check is True for check in list(test_report.values())]) + pytest_assert(check_all, "Health check failed after reboot: {}" + .format(test_report)) + + +def get_current_sonic_version(duthost): + return duthost.shell('sonic_installer list | grep Current | cut -f2 -d " "')['stdout'] + + +def overwrite_script_to_backup_logs(duthost, reboot_type, bgpd_log): + # find the fast/warm-reboot script path + reboot_script_path = duthost.shell('which {}'.format( + "{}-reboot".format(reboot_type)))['stdout'] + # backup original script + duthost.shell("cp {} {}".format( + reboot_script_path, reboot_script_path + ".orig")) + # find the anchor string inside fast/warm-reboot script + rebooting_log_line = "debug.*Rebooting with.*to.*" + # Create a backup log command to be inserted right after the anchor string defined above + backup_log_cmds = "cat /var/log/syslog.1 /var/log/syslog > /host/syslog.99 || true;" +\ + "cat /var/log/swss/sairedis.rec.1 /var/log/swss/sairedis.rec > /host/sairedis.rec.99 || true;" +\ + "cat /var/log/swss/swss.rec.1 /var/log/swss/swss.rec > /host/swss.rec.99 || true;" +\ + "cat {}.1 {} > /host/bgpd.log.99 || true".format(bgpd_log, bgpd_log) + # Do find-and-replace on fast/warm-reboot script to insert the backup_log_cmds string + insert_backup_command = "sed -i '/{}/a {}' {}".format( + rebooting_log_line, backup_log_cmds, reboot_script_path) + duthost.shell(insert_backup_command) + + +def _parse_timestamp(timestamp): + for format in [FMT, FMT_YEAR, FMT_SHORT, FMT_ALT]: + try: + time = datetime.strptime(timestamp, format) + return time + except ValueError: + continue + # Handling leap year FEB29 case, where year not provided causing exception + # if strptime fails for all format, check if its leap year + # ValueError exception will be raised for invalid cases for strptime + time = datetime.strptime(str(datetime.now().year) + " " + timestamp, FMT_YEAR) + return time + + +def get_kexec_time(duthost, messages, result): + reboot_pattern = re.compile( + r'.* NOTICE (?:admin|root): Rebooting with /sbin/kexec -e to.*...') + reboot_time = "N/A" + logging.info("FINDING REBOOT PATTERN") + for message in messages: + # Get timestamp of reboot - Rebooting string + if re.search(reboot_pattern, message): + logging.info( + "FOUND REBOOT PATTERN for {}".format(duthost.hostname)) + delim = "{}|{}".format(duthost.hostname, "sonic") + reboot_time = _parse_timestamp(re.split(delim, message)[ + 0].strip()).strftime(FMT) + continue + result["reboot_time"] = { + "timestamp": {"Start": reboot_time}, + } + + +def get_state_times(timestamp, state, state_times, first_after_offset=None): + time = timestamp.strftime(FMT) + state_name = state.split("|")[0].strip() + state_status = state.split("|")[1].strip() + state_dict = state_times.get(state_name, {"timestamp": {}}) + timestamps = state_dict.get("timestamp") + if state_status in timestamps: + state_dict[state_status + + " count"] = state_dict.get(state_status+" count") + 1 + # capture last occcurence - useful in calculating events end time + state_dict["last_occurence"] = time + elif first_after_offset: + state_dict[state_status+" count"] = 1 + # capture the first occurence as the one after offset timestamp and ignore the ones before + # this is useful to find time after a specific instance, for eg. - kexec time or FDB disable time. + if _parse_timestamp(first_after_offset) < _parse_timestamp(time): + timestamps[state_status] = time + else: + # only capture timestamp of first occurence of the entity. Otherwise, just increment the count above. + # this is useful in capturing start point. Eg., first neighbor entry, LAG ready, etc. + state_dict[state_status+" count"] = 1 + timestamps[state_status] = time + return {state_name: state_dict} + + +def analyze_log_file(duthost, messages, result, offset_from_kexec): + service_restart_times = dict() + derived_patterns = OTHER_PATTERNS.get("COMMON") + service_patterns = dict() + # get platform specific regexes + if is_broadcom_device(duthost): + derived_patterns.update(OTHER_PATTERNS.get("BRCM")) + elif is_mellanox_device(duthost): + derived_patterns.update(OTHER_PATTERNS.get("MLNX")) + # get image specific regexes + if "20191130" in get_current_sonic_version(duthost): + derived_patterns.update(OTHER_PATTERNS.get("201911")) + service_patterns.update(SERVICE_PATTERNS.get("201911")) + else: + derived_patterns.update(OTHER_PATTERNS.get("LATEST")) + service_patterns.update(SERVICE_PATTERNS.get("LATEST")) + + if not messages: + logging.error("Expected messages not found in syslog") + return None + + def service_time_check(message, status): + delim = "{}|{}".format(duthost.hostname, "sonic") + time = _parse_timestamp(re.split(delim, message)[0].strip()) + time = time.strftime(FMT) + service_name = message.split(status + " ")[1].split()[0] + service_name = service_name.upper() + if service_name == "ROUTER": + service_name = "RADV" + service_dict = service_restart_times.get( + service_name, {"timestamp": {}}) + timestamps = service_dict.get("timestamp") + if status in timestamps: + service_dict[status + + " count"] = service_dict.get(status+" count") + 1 + else: + service_dict[status+" count"] = 1 + timestamps[status] = time + service_restart_times.update({service_name: service_dict}) + + for message in messages: + # Get stopping to started timestamps for services (swss, bgp, etc) + for status, pattern in list(service_patterns.items()): + if re.search(pattern, message): + service_time_check(message, status) + break + # Get timestamps of all other entities + for state, pattern in list(derived_patterns.items()): + if re.search(pattern, message): + delim = "{}|{}".format(duthost.hostname, "sonic") + timestamp = _parse_timestamp( + re.split(delim, message)[0].strip()) + state_name = state.split("|")[0].strip() + if state_name + "|End" not in list(derived_patterns.keys()): + if "FDB_EVENT_OTHER_MAC_EXPIRY" in state_name or "FDB_EVENT_SCAPY_MAC_EXPIRY" in state_name: + fdb_aging_disable_start = service_restart_times.get("FDB_AGING_DISABLE", {})\ + .get("timestamp", {}).get("Start") + if not fdb_aging_disable_start: + break + first_after_offset = fdb_aging_disable_start + else: + first_after_offset = result.get("reboot_time", {}).get( + "timestamp", {}).get("Start") + state_times = get_state_times(timestamp, state, offset_from_kexec, + first_after_offset=first_after_offset) + offset_from_kexec.update(state_times) + else: + state_times = get_state_times( + timestamp, state, service_restart_times) + service_restart_times.update(state_times) + if "PORT_READY" not in state_name: + # If PORT_READY, don't break out of the for-loop here, because we want to + # try to match the other regex as well + break + # Calculate time that services took to stop/start + for _, timings in list(service_restart_times.items()): + timestamps = timings["timestamp"] + timings["stop_time"] = (_parse_timestamp(timestamps["Stopped"]) - + _parse_timestamp(timestamps["Stopping"])).total_seconds() \ + if "Stopped" in timestamps and "Stopping" in timestamps else None + + timings["start_time"] = (_parse_timestamp(timestamps["Started"]) - + _parse_timestamp(timestamps["Starting"])).total_seconds() \ + if "Started" in timestamps and "Starting" in timestamps else None + + if "Started" in timestamps and "Stopped" in timestamps: + timings["time_span"] = (_parse_timestamp(timestamps["Started"]) - + _parse_timestamp(timestamps["Stopped"])).total_seconds() + elif "Start" in timestamps and "End" in timestamps: + if "last_occurence" in timings: + timings["time_span"] = (_parse_timestamp(timings["last_occurence"]) - + _parse_timestamp(timestamps["Start"])).total_seconds() + else: + timings["time_span"] = (_parse_timestamp(timestamps["End"]) - + _parse_timestamp(timestamps["Start"])).total_seconds() + + result["time_span"].update(service_restart_times) + result["offset_from_kexec"] = offset_from_kexec + return result + + +def analyze_sairedis_rec(messages, result, offset_from_kexec): + sai_redis_state_times = dict() + for message in messages: + for state, pattern in list(SAIREDIS_PATTERNS.items()): + if re.search(pattern, message): + timestamp = datetime.strptime(message.split( + "|")[0].strip(), "%Y-%m-%d.%H:%M:%S.%f") + state_name = state.split("|")[0].strip() + reboot_time = result.get("reboot_time", {}).get( + "timestamp", {}).get("Start") + if state_name + "|End" not in list(SAIREDIS_PATTERNS.keys()): + if "FDB_EVENT_OTHER_MAC_EXPIRY" in state_name or "FDB_EVENT_SCAPY_MAC_EXPIRY" in state_name: + fdb_aging_disable_start = result.get("time_span", {}).get("FDB_AGING_DISABLE", {})\ + .get("timestamp", {}).get("Start") + if not fdb_aging_disable_start: + break + # Ignore MAC learning events before FDB aging disable, as MAC learning is still allowed + log_time = timestamp.strftime(FMT) + if _parse_timestamp(log_time) < _parse_timestamp(fdb_aging_disable_start): + break + first_after_offset = fdb_aging_disable_start + else: + first_after_offset = result.get("reboot_time", {}).get( + "timestamp", {}).get("Start") + state_times = get_state_times(timestamp, state, offset_from_kexec, + first_after_offset=first_after_offset) + offset_from_kexec.update(state_times) + else: + state_times = get_state_times(timestamp, state, sai_redis_state_times, + first_after_offset=reboot_time) + sai_redis_state_times.update(state_times) + + for _, timings in list(sai_redis_state_times.items()): + timestamps = timings["timestamp"] + if "Start" in timestamps and "End" in timestamps: + timings["time_span"] = (_parse_timestamp(timestamps["End"]) - + _parse_timestamp(timestamps["Start"])).total_seconds() + + result["time_span"].update(sai_redis_state_times) + result["offset_from_kexec"] = offset_from_kexec + + +def get_report_summary(duthost, analyze_result, reboot_type, reboot_oper, base_os_version): + time_spans = analyze_result.get("time_span", {}) + time_spans_summary = OrderedDict() + kexec_offsets = analyze_result.get("offset_from_kexec", {}) + reboot_start_time = analyze_result.get( + "reboot_time", {}).get("timestamp", {}).get("Start") + kexec_offsets_summary = OrderedDict() + for entity in OFFSET_ITEMS: + time_taken = "" + if entity in kexec_offsets: + time_taken = kexec_offsets.get(entity).get("time_taken", "") + elif entity in time_spans: + timestamp = time_spans.get(entity).get("timestamp", {}) + marker_start_time = timestamp.get( + "Start") if "Start" in timestamp else timestamp.get("Started") + if reboot_start_time and reboot_start_time != "N/A" and marker_start_time: + time_taken = (_parse_timestamp(marker_start_time) - + _parse_timestamp(reboot_start_time)).total_seconds() + kexec_offsets_summary.update({entity.lower(): str(time_taken)}) + + for entity in TIME_SPAN_ITEMS: + time_taken = "" + if entity in time_spans: + time_taken = time_spans.get(entity, {}).get("time_span", "") + elif entity in kexec_offsets: + marker_first_time = kexec_offsets.get( + entity).get("timestamp", {}).get("Start") + marker_last_time = kexec_offsets.get(entity).get("last_occurence") + if marker_first_time and marker_last_time: + time_taken = (_parse_timestamp(marker_last_time) - + _parse_timestamp(marker_first_time)).total_seconds() + time_spans_summary.update({entity.lower(): str(time_taken)}) + + lacp_sessions_dict = analyze_result.get("controlplane") + lacp_sessions_waittime = lacp_sessions_dict.pop("lacp_sessions")\ + if lacp_sessions_dict and "lacp_sessions" in lacp_sessions_dict else None + controlplane_summary = {"downtime": "", + "arp_ping": "", "lacp_session_max_wait": ""} + if duthost.facts['platform'] != 'x86_64-kvm_x86_64-r0': + if lacp_sessions_waittime and len(lacp_sessions_waittime) > 0: + # Filter out None values and then fine the maximum + filtered_lacp_sessions_waittime = [value for value in lacp_sessions_waittime.values() if value is not None] + if filtered_lacp_sessions_waittime: + max_lacp_session_wait = max(filtered_lacp_sessions_waittime) + else: + max_lacp_session_wait = None + analyze_result.get( + "controlplane", controlplane_summary).update( + {"lacp_session_max_wait": max_lacp_session_wait}) + + result_summary = { + "reboot_type": "{}-{}".format(reboot_type, reboot_oper) if reboot_oper else reboot_type, + "hwsku": duthost.facts["hwsku"], + "hostname": duthost.hostname, + "base_ver": base_os_version[0] if base_os_version and len(base_os_version) else "", + "target_ver": get_current_sonic_version(duthost), + "dataplane": analyze_result.get("dataplane", {"downtime": "", "lost_packets": ""}), + "controlplane": analyze_result.get("controlplane", controlplane_summary), + "time_span": time_spans_summary, + "offset_from_kexec": kexec_offsets_summary + } + return result_summary + + +def get_data_plane_report(analyze_result, reboot_type, log_dir, reboot_oper): + report = {"controlplane": {"arp_ping": "", "downtime": ""}, + "dataplane": {"lost_packets": "", "downtime": ""}} + files = glob.glob1(log_dir, '*reboot*-report.json') + if files: + filepath = "{}/{}".format(log_dir, files[0]) + with open(filepath) as json_file: + report = json.load(json_file) + analyze_result.update(report) + + +def verify_mac_jumping(test_name, timing_data, verification_errors): + mac_jumping_other_addr = timing_data.get("offset_from_kexec", {})\ + .get("FDB_EVENT_OTHER_MAC_EXPIRY", {}).get("Start count", 0) + mac_jumping_scapy_addr = timing_data.get("offset_from_kexec", {})\ + .get("FDB_EVENT_SCAPY_MAC_EXPIRY", {}).get("Start count", 0) + mac_expiry_start = timing_data.get("offset_from_kexec", {}).get("FDB_EVENT_OTHER_MAC_EXPIRY", {})\ + .get("timestamp", {}).get("Start") + fdb_aging_disable_start = timing_data.get("time_span", {}).get("FDB_AGING_DISABLE", {})\ + .get("timestamp", {}).get("Start") + fdb_aging_disable_end = timing_data.get("time_span", {}).get("FDB_AGING_DISABLE", {})\ + .get("timestamp", {}).get("End") + + if "mac_jump" in test_name: + # MAC jumping allowed - allow Scapy default MAC to jump + logging.info("MAC jumping is allowed. Jump count for expected mac: {}, unexpected MAC: {}" + .format(mac_jumping_scapy_addr, mac_jumping_other_addr)) + if not mac_jumping_scapy_addr: + verification_errors.append( + "MAC jumping not detected when expected for address: 00-06-07-08-09-0A") + else: + # MAC jumping not allowed - do not allow the SCAPY default MAC to jump + if mac_jumping_scapy_addr: + verification_errors.append("MAC jumping is not allowed. Jump count for scapy mac: {}, other MAC: {}" + .format(mac_jumping_scapy_addr, mac_jumping_other_addr)) + if mac_jumping_other_addr: + # In both mac jump allowed and denied cases unexpected MAC addresses should NOT jump between + # the window that starts when SAI is instructed to disable MAC learning (warmboot shutdown path) + # and ends when SAI is instructed to enable MAC learning (warmboot recovery path) + logging.info("Mac expiry for unexpected addresses started at {}".format(mac_expiry_start) + + " and FDB learning enabled at {}".format(fdb_aging_disable_end)) + if _parse_timestamp(mac_expiry_start) > _parse_timestamp(fdb_aging_disable_start) and\ + _parse_timestamp(mac_expiry_start) < _parse_timestamp(fdb_aging_disable_end): + verification_errors.append( + "Mac expiry detected during the window when FDB ageing was disabled") + + +def verify_required_events(duthost, event_counters, timing_data, verification_errors): + for key in ["time_span", "offset_from_kexec"]: + for pattern in REQUIRED_PATTERNS.get(key): + if pattern == 'PORT_READY': + observed_start_count = timing_data.get( + key, {}).get(pattern, {}).get("Start-changes-only count", 0) + else: + observed_start_count = timing_data.get( + key, {}).get(pattern, {}).get("Start count", 0) + observed_end_count = timing_data.get( + key, {}).get(pattern, {}).get("End count", 0) + expected_count = event_counters.get(pattern) + # If we're checking PORT_READY, allow any number of PORT_READY messages between 0 and the number of ports. + # Some platforms appear to have a random number of these messages, other platforms have however many ports + # are up. + if observed_start_count != expected_count and ( + pattern != 'PORT_READY' or observed_start_count > expected_count): + verification_errors.append("FAIL: Event {} was found {} times, when expected exactly {} times". + format(pattern, observed_start_count, expected_count)) + if key == "time_span" and observed_start_count != observed_end_count: + verification_errors.append("FAIL: Event {} counters did not match. ".format(pattern) + + "Started {} times, and ended {} times". + format(observed_start_count, observed_end_count)) + + +@pytest.fixture() +def advanceboot_loganalyzer(duthosts, enum_rand_one_per_hwsku_frontend_hostname, request): + """ + Advance reboot log analysis. + This fixture starts log analysis at the beginning of the test. At the end, + the collected expect messages are verified and timing of start/stop is calculated. + + Args: + duthosts : List of DUT hosts + enum_rand_one_per_hwsku_frontend_hostname: hostname of a randomly selected DUT + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + test_name = request.node.name + if "upgrade_path" in test_name: + reboot_type_source = request.config.getoption("--upgrade_type") + else: + reboot_type_source = test_name + if "warm" in reboot_type_source: + reboot_type = "warm" + elif "fast" in reboot_type_source: + reboot_type = "fast" + else: + reboot_type = "unknown" + # Currently, advanced reboot test would skip for kvm platform if the test has no device_type marker for vs. + # Doing the same skip logic in this fixture to avoid running loganalyzer without the test executed + if duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0': + device_marks = [arg for mark in request.node.iter_markers( + name='device_type') for arg in mark.args] + if 'vs' not in device_marks: + pytest.skip('Testcase not supported for kvm') + platform = duthost.facts["platform"] + logs_in_tmpfs = list() + + loganalyzer = LogAnalyzer( + ansible_host=duthost, marker_prefix="test_advanced_reboot_{}".format(test_name)) + base_os_version = list() + + def bgpd_log_handler(preboot=False): + # check current OS version post-reboot. This can be different than preboot OS version in case of upgrade + current_os_version = get_current_sonic_version(duthost) + if preboot: + if 'SONiC-OS-201811' in current_os_version: + bgpd_log = "/var/log/quagga/bgpd.log" + else: + bgpd_log = "/var/log/frr/bgpd.log" + additional_files = {'/var/log/swss/sairedis.rec': '', bgpd_log: ''} + loganalyzer.additional_files = list(additional_files.keys()) + loganalyzer.additional_start_str = list(additional_files.values()) + return bgpd_log + else: + # log_analyzer may start with quagga and end with frr, and frr.log might still have old logs. + # To avoid missing preboot log, or analyzing old logs, combine quagga and frr log into new file + duthost.shell("cat {} {} | sort -n > {}".format( + "/var/log/quagga/bgpd.log", "/var/log/frr/bgpd.log", "/var/log/bgpd.log"), module_ignore_errors=True) + loganalyzer.additional_files = [ + '/var/log/swss/sairedis.rec', '/var/log/bgpd.log'] + + def pre_reboot_analysis(): + log_filesystem = duthost.shell( + "df --output=fstype -h /var/log")['stdout'] + logs_in_tmpfs.append( + True if (log_filesystem and "tmpfs" in log_filesystem) else False) + base_os_version.append(get_current_sonic_version(duthost)) + bgpd_log = bgpd_log_handler(preboot=True) + if platform in LOGS_ON_TMPFS_PLATFORMS or (len(logs_in_tmpfs) > 0 and logs_in_tmpfs[0] is True): + # For small disk devices, /var/log in mounted in tmpfs. + # Hence, after reboot the preboot logs are lost. + # For log_analyzer to work, it needs logs from the shutdown path + # Below method inserts a step in reboot script to back up logs to /host/ + overwrite_script_to_backup_logs(duthost, reboot_type, bgpd_log) + marker = loganalyzer.init() + loganalyzer.load_common_config() + + ignore_file = os.path.join(TEMPLATES_DIR, "ignore_boot_messages") + expect_file = os.path.join(TEMPLATES_DIR, "expect_boot_messages") + ignore_reg_exp = loganalyzer.parse_regexp_file(src=ignore_file) + expect_reg_exp = loganalyzer.parse_regexp_file(src=expect_file) + + loganalyzer.ignore_regex.extend(ignore_reg_exp) + loganalyzer.expect_regex = [] + loganalyzer.expect_regex.extend(expect_reg_exp) + loganalyzer.match_regex = [] + return marker + + def post_reboot_analysis(marker, event_counters=None, reboot_oper=None, log_dir=None): + bgpd_log_handler() + if platform in LOGS_ON_TMPFS_PLATFORMS or (len(logs_in_tmpfs) > 0 and logs_in_tmpfs[0] is True): + restore_backup = "mv /host/syslog.99 /var/log/; " +\ + "mv /host/sairedis.rec.99 /var/log/swss/; " +\ + "mv /host/swss.rec.99 /var/log/swss/; " +\ + "mv /host/bgpd.log.99 /var/log/" + duthost.shell(restore_backup, module_ignore_errors=True) + # find the fast/warm-reboot script path + reboot_script_path = duthost.shell('which {}'.format( + "{}-reboot".format(reboot_type)))['stdout'] + # restore original script. If the ".orig" file does not exist (upgrade path case), ignore the error. + duthost.shell("mv {} {}".format(reboot_script_path + ".orig", reboot_script_path), + module_ignore_errors=True) + # For mac jump test, the log message we care about is uaually combined with other messages in one line, + # which makes the length of the line longer than 1000 and get dropped by Logananlyzer. So we need to increase + # the max allowed length. + # The regex library in Python 2 takes very long time (over 10 minutes) to process long lines. In our test, + # most of the combined log message for mac jump test is around 5000 characters. So we set the max allowed + # length to 6000. + result = loganalyzer.analyze(marker, fail=False, maximum_log_length=6000) + analyze_result = {"time_span": dict(), "offset_from_kexec": dict()} + offset_from_kexec = dict() + + # Parsing sairedis shall happen after parsing syslog because FDB_AGING_DISABLE is required + # when analysing sairedis.rec log, so we need to sort the keys + key_list = ["syslog", "bgpd.log", "sairedis.rec"] + for i in range(0, len(key_list)): + for message_key in list(result["expect_messages"].keys()): + if key_list[i] in message_key: + key_list[i] = message_key + break + + for key in key_list: + messages = result["expect_messages"][key] + if "syslog" in key: + get_kexec_time(duthost, messages, analyze_result) + reboot_start_time = analyze_result.get( + "reboot_time", {}).get("timestamp", {}).get("Start") + if not reboot_start_time or reboot_start_time == "N/A": + logging.error("kexec regex \"Rebooting with /sbin/kexec\" not found in syslog. " + + "Skipping log_analyzer checks..") + return + syslog_messages = messages + elif "bgpd.log" in key: + bgpd_log_messages = messages + elif "sairedis.rec" in key: + sairedis_rec_messages = messages + + # analyze_sairedis_rec() use information from syslog and must be called after analyzing syslog + analyze_log_file(duthost, syslog_messages, + analyze_result, offset_from_kexec) + analyze_log_file(duthost, bgpd_log_messages, + analyze_result, offset_from_kexec) + analyze_sairedis_rec(sairedis_rec_messages, + analyze_result, offset_from_kexec) + + for marker, time_data in list(analyze_result["offset_from_kexec"].items()): + marker_start_time = time_data.get("timestamp", {}).get("Start") + reboot_start_time = analyze_result.get( + "reboot_time", {}).get("timestamp", {}).get("Start") + if reboot_start_time and reboot_start_time != "N/A" and marker_start_time: + time_data["time_taken"] = (_parse_timestamp(marker_start_time) - + _parse_timestamp(reboot_start_time)).total_seconds() + else: + time_data["time_taken"] = "N/A" + + if reboot_oper and not isinstance(reboot_oper, str): + reboot_oper = type(reboot_oper).__name__ + get_data_plane_report(analyze_result, reboot_type, + log_dir, reboot_oper) + result_summary = get_report_summary( + duthost, analyze_result, reboot_type, reboot_oper, base_os_version) + logging.info(json.dumps(analyze_result, indent=4)) + logging.info(json.dumps(result_summary, indent=4)) + if reboot_oper: + report_file_name = request.node.name + "_" + reboot_oper + "_report.json" + summary_file_name = request.node.name + "_" + reboot_oper + "_summary.json" + else: + report_file_name = request.node.name + "_report.json" + summary_file_name = request.node.name + "_summary.json" + + report_file_dir = os.path.realpath((os.path.join(os.path.dirname(__file__), + "../logs/platform_tests/"))) + report_file_path = report_file_dir + "/" + report_file_name + summary_file_path = report_file_dir + "/" + summary_file_name + if not os.path.exists(report_file_dir): + os.makedirs(report_file_dir) + with open(report_file_path, 'w') as fp: + json.dump(analyze_result, fp, indent=4) + with open(summary_file_path, 'w') as fp: + json.dump(result_summary, fp, indent=4) + + # After generating timing data report, do some checks on the timing data + verification_errors = list() + verify_mac_jumping(test_name, analyze_result, verification_errors) + if duthost.facts['platform'] != 'x86_64-kvm_x86_64-r0': + # TBD: expand this verification to KVM - extra port events in KVM which need to be filtered + verify_required_events( + duthost, event_counters, analyze_result, verification_errors) + return verification_errors + + yield pre_reboot_analysis, post_reboot_analysis + + +@pytest.fixture() +def advanceboot_neighbor_restore(duthosts, enum_rand_one_per_hwsku_frontend_hostname, nbrhosts, tbinfo): + """ + This fixture is invoked at the test teardown for advanced-reboot SAD cases. + If a SAD case fails or crashes for some reason, the neighbor VMs can be left in + a bad state. This fixture will restore state of neighbor interfaces, portchannels + and BGP sessions that were shutdown during the test. + """ + yield + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + from tests.common.plugins.sanity_check.recover import neighbor_vm_restore + neighbor_vm_restore(duthost, nbrhosts, tbinfo) diff --git a/tests/common/platform/interface_utils.py b/tests/common/platform/interface_utils.py index f99c332a95..55d9f8e494 100644 --- a/tests/common/platform/interface_utils.py +++ b/tests/common/platform/interface_utils.py @@ -6,6 +6,7 @@ import re import logging +import json from natsort import natsorted from .transceiver_utils import all_transceivers_detected @@ -185,3 +186,20 @@ def get_physical_port_indices(duthost, logical_intfs=None): physical_port_index_dict[intf] = (int(index)) return physical_port_index_dict + + +def get_dpu_npu_ports_from_hwsku(duthost): + dpu_npu_port_list = [] + platform, hwsku = duthost.facts["platform"], duthost.facts["hwsku"] + hwsku_file = f'/usr/share/sonic/device/{platform}/{hwsku}/hwsku.json' + if duthost.shell(f"ls {hwsku_file}", module_ignore_errors=True)['rc'] != 0: + return dpu_npu_port_list + hwsku_content = duthost.shell(f"cat {hwsku_file}")["stdout"] + hwsku_dict = json.loads(hwsku_content) + dpu_npu_role_value = "Dpc" + + for intf, intf_config in hwsku_dict.get("interfaces").items(): + if intf_config.get("role") == dpu_npu_role_value: + dpu_npu_port_list.append(intf) + logging.info(f"DPU NPU ports in hwsku.json are {dpu_npu_port_list}") + return dpu_npu_port_list diff --git a/tests/platform_tests/reboot_timing_constants.py b/tests/common/platform/reboot_timing_constants.py similarity index 100% rename from tests/platform_tests/reboot_timing_constants.py rename to tests/common/platform/reboot_timing_constants.py index bc7e29aa43..186f6f6c58 100644 --- a/tests/platform_tests/reboot_timing_constants.py +++ b/tests/common/platform/reboot_timing_constants.py @@ -1,16 +1,5 @@ import re -REQUIRED_PATTERNS = { - "time_span": [ - "SAI_CREATE_SWITCH", - "APPLY_VIEW" - ], - "offset_from_kexec": [ - "LAG_READY", - "PORT_READY" - ] -} - SERVICE_PATTERNS = { "LATEST": { "Stopping": re.compile(r'.*Stopping.*(service|container).*'), @@ -106,3 +95,14 @@ 'SYNCD_CREATE_SWITCH', 'SAI_CREATE_SWITCH', 'APPLY_VIEW', 'INIT_VIEW', 'NEIGHBOR_ENTRY', 'PORT_INIT', 'PORT_READY', 'FINALIZER', 'LAG_READY', 'FPMSYNCD_RECONCILIATION', 'ROUTE_DEFERRAL_TIMER', 'FDB_RESTORE'] + +REQUIRED_PATTERNS = { + "time_span": [ + "SAI_CREATE_SWITCH", + "APPLY_VIEW" + ], + "offset_from_kexec": [ + "LAG_READY", + "PORT_READY" + ] +} diff --git a/tests/platform_tests/templates/expect_boot_messages b/tests/common/platform/templates/expect_boot_messages similarity index 100% rename from tests/platform_tests/templates/expect_boot_messages rename to tests/common/platform/templates/expect_boot_messages diff --git a/tests/common/platform/templates/ignore_boot_messages b/tests/common/platform/templates/ignore_boot_messages new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/platform_tests/warmboot_sad_cases.py b/tests/common/platform/warmboot_sad_cases.py similarity index 100% rename from tests/platform_tests/warmboot_sad_cases.py rename to tests/common/platform/warmboot_sad_cases.py diff --git a/tests/common/plugins/allure_server/__init__.py b/tests/common/plugins/allure_server/__init__.py index 140461a14d..efaede802a 100644 --- a/tests/common/plugins/allure_server/__init__.py +++ b/tests/common/plugins/allure_server/__init__.py @@ -67,28 +67,51 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): logger.info('Can not get Allure report URL. Please check logs') -def get_setup_session_info(session): - ansible_dir = get_ansible_path(session) - testbed = session.config.option.testbed - +def get_dut_info(ansible_dir, dut_host): os.chdir(ansible_dir) - cmd = "ansible -m command -i inventory {} -a 'show version'".format(testbed) + cmd = "ansible -m command -i inventory {} -a 'show version'".format(dut_host) output = subprocess.check_output(cmd, shell=True).decode('utf-8') - version = re.compile(r"sonic software version: +([^\s]+)\s", re.IGNORECASE) - platform = re.compile(r"platform: +([^\s]+)\s", re.IGNORECASE) - hwsku = re.compile(r"hwsku: +([^\s]+)\s", re.IGNORECASE) - asic = re.compile(r"asic: +([^\s]+)\s", re.IGNORECASE) + version_reg = re.compile(r"sonic software version: +([^\s]+)\s", re.IGNORECASE) + platform_reg = re.compile(r"platform: +([^\s]+)\s", re.IGNORECASE) + hwsku_reg = re.compile(r"hwsku: +([^\s]+)\s", re.IGNORECASE) + asic_reg = re.compile(r"asic: +([^\s]+)\s", re.IGNORECASE) + + version = version_reg.findall(output)[0] if version_reg.search(output) else "" + platform = platform_reg.findall(output)[0] if platform_reg.search(output) else "" + hwsku = hwsku_reg.findall(output)[0] if hwsku_reg.search(output) else "" + asic = asic_reg.findall(output)[0] if asic_reg.search(output) else "" + + return version, platform, hwsku, asic + + +def get_setup_session_info(session): + ansible_dir = get_ansible_path(session) + dut_hosts = session.config.option.ansible_host_pattern.split(",") + logger.info(f"dut hosts are:{dut_hosts}") + + version_list = [] + platform_list = [] + hwsku_list = [] + asic_list = [] + + for dut_host in dut_hosts: + version, platform, hwsku, asic = get_dut_info(ansible_dir, dut_host) + version_list.append(version) + platform_list.append(platform) + hwsku_list.append(hwsku) + asic_list.append(asic) random_seed = session.config.cache.get(RANDOM_SEED, None) result = { - "Version": version.findall(output)[0] if version.search(output) else "", - "Platform": platform.findall(output)[0] if platform.search(output) else "", - "HwSKU": hwsku.findall(output)[0] if hwsku.search(output) else "", - "ASIC": asic.findall(output)[0] if asic.search(output) else "", - "Random_seed": random_seed + "Dut_host": ", ".join(dut_hosts), + "Version": ", ".join(version_list), + "Platform": ", ".join(platform_list), + "HwSKU": ", ".join(hwsku_list), + "ASIC": ",".join(asic_list), + "Random_seed": random_seed, } return result diff --git a/tests/common/plugins/conditional_mark/README.md b/tests/common/plugins/conditional_mark/README.md index 9f2519e648..8582bed78e 100644 --- a/tests/common/plugins/conditional_mark/README.md +++ b/tests/common/plugins/conditional_mark/README.md @@ -12,8 +12,12 @@ This plugin works at the collection stage of pytest. It mainly uses two pytest h In `pytest_collection` hook function, it reads the specified conditions file and collect some basic facts that can be used in condition evaluation. The loaded information is stored in pytest object `session.config.cache`. -In `pytest_collection_modifyitems`, it checks each collected test item (test case). For each item, it searches for the longest matches test case name defined in the conditions content. If a match is found, then it will add the marks specified for this case based on conditions for each of the marks. -Different marks in multiple files are allowed. If different marks are found, all of them will be added to the test item(test case). However, when having the duplicate mark, it will choose the first one. +In `pytest_collection_modifyitems`, each collected test item (test case) is examined. +For each item, all potential matching conditions found based on the test case name are identified. +If a match is found and its mark is unique across all matches, the corresponding mark will be added to the test case. +If there are multiple matches, the mark from the longest match is used. +Different marks across multiple files are allowed. + ## How to use `--mark-conditions-files` `--mark-conditions-files` supports exactly file name such as `tests/common/plugins/conditional_mark/test_mark_conditions.yaml` or the pattern of the file name such as `tests/common/plugins/conditional_mark/test_mark_conditions*.yaml` which will collect all files under the path `tests/common/plugins/conditional_mark` named as `test_mark_conditions*.yaml`. @@ -74,9 +78,12 @@ folder3: reason: "Skip all the test scripts under subfolder 'folder3'" ``` -## Longest match rule +## Match rule -This plugin process each expanded (for parametrized test cases) test cases one by one. For each test case, the marks specified in the longest match entry in the conditions file will take precedence. +This plugin process each expanded (for parametrized test cases) test cases one by one. +For each test case, it will get all potential matches that match the pattern of test case name. +And then, for each match, if the mark in it is unique across all matches, we will add this mark to test case based on conditions. +Otherwise, we will use the mark which belongs to the longest match. Then we can easily apply a set of marks for specific test case in a script file and another set of marks for rest of the test cases in the same script file. @@ -88,8 +95,12 @@ feature_a/test_file_1.py: conditions: - "release in ['201911']" feature_a/test_file_1.py::testcase_3: + skip: + reason: "testcase_3 should be skipped for 202311 image" + conditions: + - "release in ['202311']" xfail: - reason: "testcase_i are suppose to fail because an issue" + reason: "testcase_3 are suppose to fail because an issue" conditions: - https://github.com/sonic-net/sonic-mgmt/issues/1234 ``` @@ -104,10 +115,12 @@ def testcase_2 def testcase_3 ``` -In this example, `testcase_1` and `testcase_2` will have nodeid like `feature_a/test_file_1.py::testcase_1` and `feature_a/test_file_1.py::testcase_2`. They will match entry `feature_a/test_file_1.py`. So, the `skip` mark will be added to `testcase_1` and `testcase_2` when `release in ['201911']`. -For `testcase_3`, its nodeid will be `feature_a/test_file_1.py::testcase_3`. Then it will only match `feature_a/test_file_1.py::testcase_3`. The `xfail` mark will be added to `testcase_3` when the Github issue is still open. Entry `feature_a/test_file_1.py` also matches its nodeid. But, because it is not the longest match, it will simply be ignored. +In this example, `testcase_1` and `testcase_2` will have nodeid like `feature_a/test_file_1.py::testcase_1` and `feature_a/test_file_1.py::testcase_2`. +They will match entry `feature_a/test_file_1.py`. So, the `skip` mark will be added to `testcase_1` and `testcase_2` when `release in ['201911']`. -In a summary, under such scenario, the `skip` mark will be conditionally added to `testcase_1` and `testcase_2`. The `xfail` mark will be conditionally added to `testcase_3`. +For `testcase_3`, its nodeid will be `feature_a/test_file_1.py::testcase_3`. It will match both `feature_a/test_file_1.py` and `feature_a/test_file_1.py::testcase_3`. +For mark `xfail`, it is the only mark in all matches, so it will be added to `testcase_3` when the Github issue is still open. +And for mark `skip`, it exists in multiple matches. We will use the longest match of this match, which is under the entry `feature_a/test_file_1.py::testcase_3`. If a test case is parameterized, we can even specify different mark for different parameter value combinations for the same test case. diff --git a/tests/common/plugins/conditional_mark/__init__.py b/tests/common/plugins/conditional_mark/__init__.py index 366d50cf8b..4ff2d5d1a8 100644 --- a/tests/common/plugins/conditional_mark/__init__.py +++ b/tests/common/plugins/conditional_mark/__init__.py @@ -402,31 +402,54 @@ def load_basic_facts(session): return results -def find_longest_matches(nodeid, conditions): - """Find the longest matches of the given test case name in the conditions list. - - This is similar to longest prefix match in routing table. The longest match takes precedence. +def find_all_matches(nodeid, conditions): + """Find all matches of the given test case name in the conditions list. Args: nodeid (str): Full test case name conditions (list): List of conditions Returns: - str: Longest match test case name or None if not found + list: All match test case name or None if not found """ - longest_matches = [] + all_matches = [] max_length = -1 + conditional_marks = {} + matches = [] + for condition in conditions: # condition is a dict which has only one item, so we use condition.keys()[0] to get its key. if nodeid.startswith(list(condition.keys())[0]): - length = len(condition) - if length > max_length: + all_matches.append(condition) + + for match in all_matches: + case_starting_substring = list(match.keys())[0] + length = len(case_starting_substring) + marks = match[case_starting_substring].keys() + for mark in marks: + if mark in conditional_marks: + if length >= max_length: + conditional_marks.update({ + mark: { + case_starting_substring: { + mark: match[case_starting_substring][mark]} + }}) + max_length = length + else: + conditional_marks.update({ + mark: { + case_starting_substring: { + mark: match[case_starting_substring][mark]} + }}) max_length = length - longest_matches = [] - longest_matches.append(condition) - elif length == max_length: - longest_matches.append(condition) - return longest_matches + + # We may have the same matches of different marks + # Need to remove duplicate here + for condition in list(conditional_marks.values()): + if condition not in matches: + matches.append(condition) + + return matches def update_issue_status(condition_str, session): @@ -582,12 +605,12 @@ def pytest_collection_modifyitems(session, config, items): json.dumps(basic_facts, indent=2))) dynamic_update_skip_reason = session.config.option.dynamic_update_skip_reason for item in items: - longest_matches = find_longest_matches(item.nodeid, conditions) + all_matches = find_all_matches(item.nodeid, conditions) - if longest_matches: - logger.debug('Found match "{}" for test case "{}"'.format(longest_matches, item.nodeid)) + if all_matches: + logger.debug('Found match "{}" for test case "{}"'.format(all_matches, item.nodeid)) - for match in longest_matches: + for match in all_matches: # match is a dict which has only one item, so we use match.values()[0] to get its value. for mark_name, mark_details in list(list(match.values())[0].items()): conditions_logical_operator = mark_details.get('conditions_logical_operator', 'AND').upper() diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index 1061f356e1..64cbf516b9 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -45,6 +45,12 @@ arp/test_arp_dualtor.py::test_proxy_arp_for_standby_neighbor: conditions: - "release not in ['202012']" +arp/test_neighbor_mac_noptf.py: + skip: + reason: "Not supported in standalone topologies." + conditions: + - "'standalone' in topo_name" + arp/test_unknown_mac.py: skip: reason: "Behavior on cisco-8000 & Innovium(Marvell) platform for unknown MAC is flooding rather than DROP, hence skipping." @@ -82,25 +88,34 @@ bfd/test_bfd.py::test_bfd_scale: bfd/test_bfd_static_route.py: skip: - reason: "Only supported on chassis system & cisco platform." + reason: "Only supported on multi-asic system & Cisco LCs." conditions: - - "is_multi_asic is False" - - "asic_type in ['cisco-8000']" + - "(is_multi_asic is False) or (hwsku not in ['Cisco-88-LC0-36FH-M-O36', 'Cisco-8800-LC-48H-C48', 'Cisco-88-LC0-36FH-O36'])" + +bfd/test_bfd_traffic.py: + skip: + reason: "Test only supported on Cisco 8800 platforms." + conditions: + - "asic_type not in ['cisco-8000']" ####################################### ##### bgp ##### ####################################### bgp/test_bgp_allow_list.py: skip: - reason: "Only supported on t1 topo." + reason: "Only supported on t1 topo. But Cisco 8111 T1(compute ai) platform is not supported." + conditions_logical_operator: or conditions: - "'t1' not in topo_type" + - "platform in ['x86_64-8111_32eh_o-r0']" bgp/test_bgp_bbr.py: skip: - reason: "Only supported on t1 topo." + reason: "Only supported on t1 topo. But Cisco 8111 T1(compute ai) platform is not supported." + conditions_logical_operator: or conditions: - "'t1' not in topo_type" + - "platform in ['x86_64-8111_32eh_o-r0']" bgp/test_bgp_gr_helper.py: skip: @@ -116,9 +131,9 @@ bgp/test_bgp_multipath_relax.py: bgp/test_bgp_queue.py: skip: - reason: "1. Not support on mgmt device 2. Known issue on Mellanox platform with T1-LAG topology" + reason: "Not supported on mgmt device" conditions: - - "topo_type in ['m0', 'mx'] or ((topo_name in ['t1-lag']) and (asic_type in ['mellanox']))" + - "topo_type in ['m0', 'mx']" bgp/test_bgp_slb.py: skip: @@ -130,7 +145,7 @@ bgp/test_bgp_slb.py::test_bgp_slb_neighbor_persistence_across_advanced_reboot: skip: reason: "Skip it on dual tor since it got stuck during warm reboot due to known issue on master and internal image" conditions: - - "topo_name in ['dualtor', 'dualtor-56', 'dualtor-120']" + - "topo_name in ['dualtor', 'dualtor-56', 'dualtor-120', 'dualtor-aa', 'dualtor-aa-56']" - https://github.com/sonic-net/sonic-mgmt/issues/9201 bgp/test_bgp_speaker.py: @@ -142,8 +157,10 @@ bgp/test_bgp_speaker.py: bgp/test_bgp_suppress_fib.py: skip: reason: "Not supported before release 202405." + conditions_logical_operator: or conditions: - - "release in ['201811', '201911', '202012', '202205', '202211', '202305', '202311']" + - "release in ['201811', '201911', '202012', '202205', '202211', '202305', '202311', 'master']" + - "asic_type in ['vs'] and https://github.com/sonic-net/sonic-mgmt/issues/14449" bgp/test_bgpmon.py: skip: @@ -201,29 +218,19 @@ copp/test_copp.py: skip: reason: "Topology not supported by COPP tests" conditions: - - "(topo_name not in ['ptf32', 'ptf64', 't0', 't0-64', 't0-52', 't0-116', 't1', 't1-lag', 't1-64-lag', 't1-56-lag', 't1-backend', 'm0', 'mx'] and 't2' not in topo_type)" + - "(topo_name not in ['ptf32', 'ptf64', 't0', 't0-64', 't0-52', 't0-116', 't1', 't1-lag', 't1-64-lag', 't1-56-lag', 't1-backend', 'm0', 'm0-2vlan', 'mx'] and 't2' not in topo_type)" copp/test_copp.py::TestCOPP::test_add_new_trap: skip: reason: "Copp test_add_new_trap is not yet supported on multi-asic platform" conditions: - "is_multi_asic==True" - xfail: - reason: "Can't unisntall trap on broadcom platform successfully" - conditions: - - "asic_type in ['broadcom']" - - "release in ['202305']" copp/test_copp.py::TestCOPP::test_remove_trap: skip: reason: "Copp test_remove_trap is not yet supported on multi-asic platform" conditions: - "is_multi_asic==True" - xfail: - reason: "Can't unisntall trap on broadcom platform successfully" - conditions: - - "asic_type in ['broadcom']" - - "release in ['202305']" copp/test_copp.py::TestCOPP::test_trap_config_save_after_reboot: skip: @@ -243,12 +250,6 @@ crm/test_crm.py::test_crm_fdb_entry: conditions: - "'t0' not in topo_name and topo_type not in ['m0', 'mx']" -crm/test_crm.py::test_crm_route: - skip: - reason: "Skip the test on mellanox device due to the bug of https://github.com/sonic-net/sonic-mgmt/issues/12725" - conditions: - - https://github.com/sonic-net/sonic-mgmt/issues/12725 - - "asic_type in ['mellanox']" ####################################### ##### decap ##### ####################################### @@ -267,9 +268,9 @@ decap/test_decap.py::test_decap[ttl=pipe, dscp=pipe, vxlan=set_unset]: decap/test_decap.py::test_decap[ttl=pipe, dscp=uniform, vxlan=disable]: skip: conditions_logical_operator: or - reason: "Not supported on backend, broadcom before 202012 release, innovium and x86_64-8111_32eh_o-r0 platform. Skip 7260CX3 T1 topo in 202305 release" + reason: "Not supported on backend, broadcom before 202012 release, innovium platform. Skip 7260CX3 T1 topo in 202305 release" conditions: - - "(topo_name in ['t1-backend', 't0-backend']) or (asic_type in ['broadcom'] and release in ['201811', '201911']) or asic_type in ['innovium'] or platform in ['x86_64-8111_32eh_o-r0']" + - "(topo_name in ['t1-backend', 't0-backend']) or (asic_type in ['broadcom'] and release in ['201811', '201911']) or asic_type in ['innovium']" - "'7260CX3' in hwsku and release in ['202305'] and 't1' in topo_type" decap/test_decap.py::test_decap[ttl=pipe, dscp=uniform, vxlan=set_unset]: @@ -297,6 +298,14 @@ decap/test_decap.py::test_decap[ttl=uniform, dscp=uniform, vxlan=set_unset]: skip: reason: "Not supported uniform ttl mode" +decap/test_subnet_decap.py::test_vlan_subnet_decap: + skip: + reason: "Supported only on T0 topology with KVM or broadcom td3 asic, and available for 202405 release and later" + conditions: + - "topo_type not in ['t0']" + - "asic_type not in ['vs'] or asic_gen not in ['td3']" + - "release in ['202012', '202205', '202305', '202311']" + ####################################### ##### dhcp_relay ##### ####################################### @@ -306,6 +315,30 @@ dhcp_relay/test_dhcp_relay.py: conditions: - "platform in ['x86_64-8111_32eh_o-r0']" +dhcp_relay/test_dhcp_relay.py::test_dhcp_relay_after_link_flap: + skip: + reason: "Skip test_dhcp_relay_after_link_flap on dualtor" + conditions: + - "'dualtor' in topo_name" + +dhcp_relay/test_dhcp_relay.py::test_dhcp_relay_random_sport: + skip: + reason: "Skip test_dhcp_relay_random_sport on dualtor in 201811 and 201911" + conditions: + - "'dualtor' in topo_name and release in ['201811', '201911']" + +dhcp_relay/test_dhcp_relay.py::test_dhcp_relay_start_with_uplinks_down: + skip: + reason: "Skip test_dhcp_relay_start_with_uplinks_down on dualtor" + conditions: + - "'dualtor' in topo_name" + +dhcp_relay/test_dhcp_relay.py::test_dhcp_relay_unicast_mac: + skip: + reason: "Skip test_dhcp_relay_unicast_mac on dualtor" + conditions: + - "'dualtor' in topo_name and release in ['201811', '201911']" + dhcp_relay/test_dhcpv6_relay.py: skip: reason: "Need to skip for platform x86_64-8111_32eh_o-r0" @@ -401,6 +434,13 @@ dualtor/test_standby_tor_upstream_mux_toggle.py: conditions: - "(topo_type not in ['t0']) or ('dualtor' in topo_name)" +dualtor/test_switchover_failure.py: + skip: + reason: "Test in KVM has a high failure rate, skip with Github issue." + conditions: + - "asic_type in ['vs']" + - https://github.com/sonic-net/sonic-mgmt/issues/14247 + dualtor/test_tor_ecn.py::test_dscp_to_queue_during_encap_on_standby: xfail: reason: "Testcase ignored on dualtor-aa topology and mellanox setups due to Github issue: https://github.com/sonic-net/sonic-mgmt/issues/8577" @@ -421,11 +461,11 @@ dualtor/test_tunnel_memory_leak.py::test_tunnel_memory_leak: conditions: - "https://github.com/sonic-net/sonic-mgmt/issues/11403 and 'dualtor-64' in topo_name" -dualtor_io/test_link_failure.py::test_active_link_admin_down_config_reload_downstream[active-active]: - xfail: - reason: "Testcase ignored on mellanox setups due to github issue: https://github.com/sonic-net/sonic-buildimage/issues/16085" +dualtor_io: + skip: + reason: "Testcase could only be executed on dualtor testbed." conditions: - - "https://github.com/sonic-net/sonic-buildimage/issues/16085 and asic_type in ['mellanox']" + - "'dualtor' not in topo_name" dualtor_io/test_link_failure.py::test_active_link_admin_down_config_reload_link_up_downstream_standby[active-active]: xfail: @@ -439,6 +479,10 @@ dualtor_io/test_link_failure.py::test_active_link_down_downstream_active: conditions: - https://github.com/sonic-net/sonic-mgmt/issues/8272 - "asic_type in ['mellanox']" + skip: + reason: "KVM testbed do not support shutdown fanout interface action" + conditions: + - "asic_type in ['vs']" dualtor_io/test_link_failure.py::test_active_link_down_downstream_active_soc: xfail: @@ -446,6 +490,10 @@ dualtor_io/test_link_failure.py::test_active_link_down_downstream_active_soc: conditions: - https://github.com/sonic-net/sonic-mgmt/issues/8272 - "asic_type in ['mellanox']" + skip: + reason: "KVM testbed do not support shutdown fanout interface action" + conditions: + - "asic_type in ['vs']" dualtor_io/test_link_failure.py::test_active_link_down_downstream_standby: xfail: @@ -453,12 +501,40 @@ dualtor_io/test_link_failure.py::test_active_link_down_downstream_standby: conditions: - https://github.com/sonic-net/sonic-mgmt/issues/8272 - "asic_type in ['mellanox']" + skip: + reason: "KVM testbed do not support shutdown fanout interface action" + conditions: + - "asic_type in ['vs']" -dualtor_io/test_normal_op.py::test_upper_tor_config_reload_upstream[active-active]: - xfail: - reason: "Testcase ignored on mellanox setups due to github issue: https://github.com/sonic-net/sonic-buildimage/issues/15964" +dualtor_io/test_link_failure.py::test_active_link_down_upstream: + skip: + reason: "KVM testbed do not support shutdown fanout interface action" + conditions: + - "asic_type in ['vs']" + +dualtor_io/test_link_failure.py::test_active_link_down_upstream_soc: + skip: + reason: "KVM testbed do not support shutdown fanout interface action" + conditions: + - "asic_type in ['vs']" + +dualtor_io/test_link_failure.py::test_standby_link_down_downstream_active: + skip: + reason: "KVM testbed do not support shutdown fanout interface action" + conditions: + - "asic_type in ['vs']" + +dualtor_io/test_link_failure.py::test_standby_link_down_downstream_standby: + skip: + reason: "KVM testbed do not support shutdown fanout interface action" conditions: - - "https://github.com/sonic-net/sonic-buildimage/issues/15964 and asic_type in ['mellanox']" + - "asic_type in ['vs']" + +dualtor_io/test_link_failure.py::test_standby_link_down_upstream: + skip: + reason: "KVM testbed do not support shutdown fanout interface action" + conditions: + - "asic_type in ['vs']" ####################################### ##### dut_console ##### @@ -478,9 +554,9 @@ ecmp/inner_hashing/test_inner_hashing.py: reason: "PBH introduced in 202111 and skip this test on Mellanox 2700 platform. Test does not support dualtor topology." conditions: - "branch in ['201811', '201911', '202012', '202106']" - - "platform in ['x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0']" + - "platform not in ['x86_64-mlnx_msn3800-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-kvm_x86_64-r0']" - "topo_type not in ['t0']" - - "asic_type not in ['mellanox']" + - "asic_type not in ['mellanox', 'vs']" - "'dualtor' in topo_name" ecmp/inner_hashing/test_inner_hashing_lag.py: @@ -489,9 +565,9 @@ ecmp/inner_hashing/test_inner_hashing_lag.py: reason: "PBH introduced in 202111 and skip this test on Mellanox 2700 platform. Test does not support dualtor topology." conditions: - "branch in ['201811', '201911', '202012', '202106']" - - "platform in ['x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0']" + - "platform not in ['x86_64-mlnx_msn3800-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-kvm_x86_64-r0']" - "topo_type not in ['t0']" - - "asic_type not in ['mellanox']" + - "asic_type not in ['mellanox', 'vs']" - "'dualtor' in topo_name" ecmp/inner_hashing/test_wr_inner_hashing.py: @@ -500,9 +576,9 @@ ecmp/inner_hashing/test_wr_inner_hashing.py: reason: "PBH introduced in 202111 and skip this test on Mellanox 2700 platform. Test does not support dualtor topology." conditions: - "branch in ['201811', '201911', '202012', '202106']" - - "platform in ['x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0']" + - "platform not in ['x86_64-mlnx_msn3800-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-kvm_x86_64-r0']" - "topo_type not in ['t0']" - - "asic_type not in ['mellanox']" + - "asic_type not in ['mellanox', 'vs']" - "'dualtor' in topo_name" ecmp/inner_hashing/test_wr_inner_hashing_lag.py: @@ -511,26 +587,20 @@ ecmp/inner_hashing/test_wr_inner_hashing_lag.py: reason: "PBH introduced in 202111 and skip this test on Mellanox 2700 platform. Test does not support dualtor topology." conditions: - "branch in ['201811', '201911', '202012', '202106']" - - "platform in ['x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0']" + - "platform not in ['x86_64-mlnx_msn3800-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-kvm_x86_64-r0']" - "topo_type not in ['t0']" - - "asic_type not in ['mellanox']" + - "asic_type not in ['mellanox', 'vs']" - "'dualtor' in topo_name" ecmp/test_ecmp_sai_value.py: skip: - reason: "Only support Broadcom T1/T0 topology with 20230531 image, 7050cx3 T1 doesn't enable this feature" + reason: "Only support Broadcom T1/T0 topology with 20230531 and above image, 7050cx3 T1 doesn't enable this feature" conditions_logical_operator: or conditions: - "topo_type not in ['t1', 't0']" - "asic_type not in ['broadcom']" - - "release in ['201911', '202012', '202205', '202211', 'master']" - - "'internal' in build_version" + - "release in ['201911', '202012', '202205', '202211']" - "topo_type in ['t1'] and hwsku in ['Arista-7050CX3-32S-C32']" - xfail: - reason: "This feature is not supported in 202311 as the code was not merged into 202311" - conditions: - - "release in ['202311']" - - https://github.com/sonic-net/sonic-mgmt/issues/11310 ecmp/test_fgnhg.py: skip: @@ -679,12 +749,6 @@ generic_config_updater/test_eth_interface.py::test_update_speed: - https://github.com/sonic-net/sonic-mgmt/issues/8143 - https://github.com/sonic-net/sonic-buildimage/issues/13267 -generic_config_updater/test_eth_interface.py::test_update_valid_invalid_index[33-True]: - skip: - reason: 'Skipping test on mellanox platform due to bug of https://github.com/sonic-net/sonic-mgmt/issues/7733' - conditions: - - "asic_type in ['mellanox']" - - https://github.com/sonic-net/sonic-mgmt/issues/7733 generic_config_updater/test_incremental_qos.py::test_incremental_qos_config_updates: skip: @@ -715,6 +779,108 @@ generic_config_updater/test_pg_headroom_update.py: conditions: - "topo_type in ['m0', 'mx']" +####################################### +##### hash ##### +####################################### +hash/test_generic_hash.py::test_algorithm_config: + xfail: + reason: "This is a new test cases and doesn't work for platform other than Mellanox, xfail them before the issue is addressed" + conditions: + - "asic_type not in ['mellanox']" + - https://github.com/sonic-net/sonic-mgmt/issues/14109 + +hash/test_generic_hash.py::test_backend_error_messages: + xfail: + reason: "This is a new test cases and doesn't work for platform other than Mellanox, xfail them before the issue is addressed" + conditions: + - "asic_type not in ['mellanox']" + - https://github.com/sonic-net/sonic-mgmt/issues/14109 + +hash/test_generic_hash.py::test_ecmp_and_lag_hash: + skip: + reason: 'On Mellanox SPC1 platforms, due to HW limitation, it would not support CRC_CCITT algorithm' + conditions: + - "asic_gen == 'spc1'" + +hash/test_generic_hash.py::test_ecmp_and_lag_hash[CRC-INNER_IP_PROTOCOL: + skip: + reason: "On Mellanox platforms, due to HW limitation, it would not support CRC algorithm on INNER_IP_PROTOCOL field" + conditions: + - "asic_type in ['mellanox']" + +hash/test_generic_hash.py::test_ecmp_hash[CRC-INNER_IP_PROTOCOL: + skip: + reason: "On Mellanox platforms, due to HW limitation, it would not support CRC algorithm on INNER_IP_PROTOCOL field" + conditions: + - "asic_type in ['mellanox']" + +hash/test_generic_hash.py::test_hash_capability: + xfail: + reason: "This is a new test cases and doesn't work for platform other than Mellanox, xfail them before the issue is addressed" + conditions: + - "asic_type not in ['mellanox']" + - https://github.com/sonic-net/sonic-mgmt/issues/14109 + +hash/test_generic_hash.py::test_lag_hash[CRC-INNER_IP_PROTOCOL: + skip: + reason: "On Mellanox platforms, due to HW limitation, it would not support CRC algorithm on INNER_IP_PROTOCOL field" + conditions: + - "asic_type in ['mellanox']" + +hash/test_generic_hash.py::test_lag_member_flap: + skip: + reason: 'On Mellanox SPC1 platforms, due to HW limitation, it would not support CRC_CCITT algorithm, for other platforms, skipping due to missing object in SonicHost' + conditions_logical_operator: "OR" + conditions: + - "asic_gen == 'spc1'" + - https://github.com/sonic-net/sonic-mgmt/issues/13919 + +hash/test_generic_hash.py::test_lag_member_flap[CRC-INNER_IP_PROTOCOL: + skip: + reason: "On Mellanox platforms, due to HW limitation, it would not support CRC algorithm on INNER_IP_PROTOCOL field" + conditions: + - "asic_type in ['mellanox']" + +hash/test_generic_hash.py::test_lag_member_remove_add: + skip: + reason: 'On Mellanox SPC1 platforms, due to HW limitation, it would not support CRC_CCITT algorithm, for other platforms, skipping due to missing object in SonicHost' + conditions_logical_operator: "OR" + conditions: + - "asic_gen == 'spc1'" + - https://github.com/sonic-net/sonic-mgmt/issues/13919 + +hash/test_generic_hash.py::test_lag_member_remove_add[CRC-INNER_IP_PROTOCOL: + skip: + reason: "On Mellanox platforms, due to HW limitation, it would not support CRC algorithm on INNER_IP_PROTOCOL field" + conditions: + - "asic_type in ['mellanox']" + +hash/test_generic_hash.py::test_nexthop_flap: + skip: + reason: 'On Mellanox SPC1 platforms, due to HW limitation, it would not support CRC_CCITT algorithm, for other platforms, skipping due to missing object in SonicHost' + conditions_logical_operator: "OR" + conditions: + - "asic_gen == 'spc1'" + - https://github.com/sonic-net/sonic-mgmt/issues/13919 + +hash/test_generic_hash.py::test_nexthop_flap[CRC-INNER_IP_PROTOCOL: + skip: + reason: "On Mellanox platforms, due to HW limitation, it would not support CRC algorithm on INNER_IP_PROTOCOL field" + conditions: + - "asic_type in ['mellanox']" + +hash/test_generic_hash.py::test_reboot: + skip: + reason: 'On Mellanox SPC1 platforms, due to HW limitation, it would not support CRC_CCITT algorithm' + conditions: + - "asic_gen == 'spc1'" + +hash/test_generic_hash.py::test_reboot[CRC-INNER_IP_PROTOCOL: + skip: + reason: "On Mellanox platforms, due to HW limitation, it would not support CRC algorithm on INNER_IP_PROTOCOL field" + conditions: + - "asic_type in ['mellanox']" + ####################################### ##### http ##### ####################################### @@ -830,6 +996,21 @@ link_flap/test_cont_link_flap.py: conditions: - https://github.com/sonic-net/sonic-mgmt/issues/10955 +####################################### +##### lldp ##### +####################################### +lldp/test_lldp.py::test_lldp: + skip: + reason: "Skipping LLDP test because the topology is standalone. No LLDP neighbors detected." + conditions: + - "'standalone' in topo_name" + +lldp/test_lldp.py::test_lldp_neighbor: + skip: + reason: "Skipping LLDP test because the topology is standalone. No LLDP neighbors detected." + conditions: + - "'standalone' in topo_name" + ####################################### ##### macsec ##### ####################################### @@ -878,16 +1059,6 @@ nat: conditions: - "'nat' not in feature_status" -####################################### -##### ntp ##### -####################################### -ntp/test_ntp.py::test_ntp_long_jump_disabled: - # Due to NTP code bug, long jump will still happen after disable it. - # Set xfail flag for this test case - xfail: - strict: True - reason: "Known NTP bug" - ####################################### ##### pc ##### ####################################### @@ -899,9 +1070,11 @@ pc/test_lag_2.py::test_lag_db_status_with_po_update: pc/test_lag_member.py: skip: - reason: "Not support dualtor or t0 backend topo" + reason: "Not support dualtor or t0 backend topo / Have an issue on kvm t0" + conditions_logical_operator: or conditions: - "'dualtor' in topo_name or 't0-backend' in topo_name" + - "asic_type in ['vs'] and https://github.com/sonic-net/sonic-mgmt/issues/13898" pc/test_po_cleanup.py: skip: @@ -966,6 +1139,13 @@ pfcwd/test_pfc_config.py::TestPfcConfig::test_forward_action_cfg: conditions: - "asic_type in ['cisco-8000']" +pfcwd/test_pfcwd_all_port_storm.py: + skip: + reason: "Slow pfc generation rate on 7060x6 200Gb, + pfc generation function on the Arista fanout device need to be improved by Arista" + conditions: + - "hwsku in ['Arista-7060X6-64PE-256x200G']" + pfcwd/test_pfcwd_function.py::TestPfcwdFunc::test_pfcwd_no_traffic: skip: reason: "This test is applicable only for cisco-8000" @@ -984,128 +1164,13 @@ pfcwd/test_pfcwd_warm_reboot.py: - https://github.com/sonic-net/sonic-mgmt/issues/8400 ####################################### -##### platform_tests ##### +##### process_monitoring ##### ####################################### -platform_tests/api/test_psu.py::test_temperature: - skip: - reason: "Test not supported on Mellanox Platforms." - conditions: - - "asic_type in ['mellanox']" - -platform_tests/api/test_sfp.py::TestSfpApi::test_reset: - skip: - reason: "platform does not support sfp reset" - conditions: - - "platform in ['x86_64-cel_e1031-r0']" - -platform_tests/api/test_sfp.py::TestSfpApi::test_tx_disable_channel: - skip: - reason: "platform does not support" - conditions: - - "platform in ['x86_64-cel_e1031-r0']" - -platform_tests/cli/test_show_platform.py::test_show_platform_psustatus: - skip: - reason: "Test should be skipped on DPU for it doesn't have PSUs." - conditions: "'nvda_bf' in platform" - -platform_tests/cli/test_show_platform.py::test_show_platform_psustatus_json: - skip: - reason: "Test should be skipped on DPU for it doesn't have PSUs." - conditions: "'nvda_bf' in platform" - -platform_tests/daemon/test_chassisd.py: - skip: - reason: "chassisd platform daemon introduced in 202106" - conditions: - - "release in ['201811', '201911', '202012']" - -platform_tests/mellanox/test_check_sfp_using_ethtool.py: +process_monitoring/test_critical_process_monitoring.py::test_orchagent_heartbeat: skip: - reason: "Deprecated as Mellanox do not use ethtool in release 202305 or higher" + reason: This test is intended for Orchagent freeze scenario during warm-reboot. It is not required for T1 devices. conditions: - - "asic_type in ['mellanox']" - -platform_tests/sfp/test_sfputil.py::test_check_sfputil_low_power_mode: - skip: - reason: "Get/Set low power mode is not supported in Cisco 8000 platform or in bluefield platform" - conditions_logical_operator: or - conditions: - - "platform in ['x86_64-cel_e1031-r0']" - - "asic_type in ['nvidia-bluefield']" - -platform_tests/sfp/test_sfputil.py::test_check_sfputil_reset: - skip: - reason: "platform does not support sfp reset" - conditions_logical_operator: or - conditions: - - "(platform in ['x86_64-cel_e1031-r0'] or 't2' in topo_name) and https://github.com/sonic-net/sonic-mgmt/issues/6218" - - "asic_type in ['nvidia-bluefield']" - -platform_tests/test_auto_negotiation.py: - skip: - reason: "auto negotiation test highly depends on test enviroments, file issue to track and skip for now" - conditions: https://github.com/sonic-net/sonic-mgmt/issues/5447 - -platform_tests/test_cont_warm_reboot.py: - skip: - reason: "Warm Reboot is not supported in dualtor" - conditions: - - "'dualtor' in topo_name" - -platform_tests/test_platform_info.py::test_show_platform_fanstatus_mocked: - skip: - reason: "Test not supported on Mellanox Platforms." - conditions: - - "asic_type in ['mellanox']" - -platform_tests/test_platform_info.py::test_show_platform_temperature_mocked: - skip: - reason: "Test not supported on Mellanox Platforms." - conditions: - - "asic_type in ['mellanox']" - -platform_tests/test_platform_info.py::test_thermal_control_fan_status: - skip: - reason: "Test not supported on Mellanox Platforms." - conditions: - - "asic_type in ['mellanox']" - -platform_tests/test_platform_info.py::test_thermal_control_load_invalid_format_json: - skip: - reason: "Test not supported on Mellanox Platforms." - conditions: - - "asic_type in ['mellanox']" - -platform_tests/test_platform_info.py::test_thermal_control_load_invalid_value_json: - skip: - reason: "Test not supported on Mellanox Platforms." - conditions: - - "asic_type in ['mellanox']" - -platform_tests/test_reboot.py::test_fast_reboot: - skip: - reason: "Fast reboot is broken on dualtor topology. Skipping for now." - conditions: - - "'dualtor' in topo_name and https://github.com/sonic-net/sonic-buildimage/issues/16502" - -platform_tests/test_reboot.py::test_warm_reboot: - skip: - reason: "Warm reboot is broken on dualtor topology. Skipping for now." - conditions: - - "'dualtor' in topo_name and https://github.com/sonic-net/sonic-buildimage/issues/16502" - -platform_tests/test_reload_config.py::test_reload_configuration_checks: - skip: - reason: "Skip test_reload_configuration_checks testcase due to flaky timing issue for Cisco 8000" - conditions: - - "asic_type in ['cisco-8000'] and release in ['202205', '202211', '202305']" - -platform_tests/test_secure_upgrade.py: - skip: - reason: "platform does not support secure upgrade" - conditions: - - "'sn2' in platform or 'sn3' in platform or 'sn4' in platform" + - "'t1' in topo_name" ####################################### ##### qos ##### @@ -1219,6 +1284,12 @@ qos/test_qos_sai.py::TestQosSai::testQosSaiDwrrWeightChange: conditions: - "asic_type in ['mellanox']" +qos/test_qos_sai.py::TestQosSai::testQosSaiFullMeshTrafficSanity: + skip: + reason: "Unsupported platform or testbed type." + conditions: + - "asic_type not in ['cisco-8000'] or topo_name not in ['ptf64']" + qos/test_qos_sai.py::TestQosSai::testQosSaiHeadroomPoolSize: skip: reason: "Headroom pool size not supported." @@ -1243,13 +1314,13 @@ qos/test_qos_sai.py::TestQosSai::testQosSaiLosslessVoq: skip: reason: "Lossless Voq test is not supported" conditions: - - "asic_type not in ['cisco-8000']" + - "asic_type not in ['cisco-8000'] or platform in ['x86_64-8122_64eh_o-r0']" qos/test_qos_sai.py::TestQosSai::testQosSaiLossyQueueVoq: skip: reason: "Lossy Queue Voq test is not supported" conditions: - - "asic_type not in ['cisco-8000']" + - "asic_type not in ['cisco-8000'] or platform in ['x86_64-8122_64eh_o-r0']" qos/test_qos_sai.py::TestQosSai::testQosSaiLossyQueueVoqMultiSrc: skip: @@ -1261,13 +1332,13 @@ qos/test_qos_sai.py::TestQosSai::testQosSaiPGDrop: skip: reason: "PG drop size test is not supported." conditions: - - "asic_type not in ['cisco-8000']" + - "asic_type not in ['cisco-8000'] or platform in ['x86_64-8122_64eh_o-r0']" qos/test_qos_sai.py::TestQosSai::testQosSaiPgHeadroomWatermark: skip: reason: "Priority Group Headroom Watermark is not supported on cisco asic. PG drop counter stat is covered as a part of testQosSaiPfcXoffLimit" conditions: - - "asic_type in ['cisco-8000']" + - "asic_type in ['cisco-8000'] and platform not in ['x86_64-8122_64eh_o-r0']" qos/test_qos_sai.py::TestQosSai::testQosSaiPgSharedWatermark[None-wm_pg_shared_lossy]: xfail: @@ -1285,7 +1356,7 @@ qos/test_qos_sai.py::TestQosSai::testQosSaiSharedReservationSize: skip: reason: "Shared reservation size test is not supported." conditions: - - "asic_type not in ['cisco-8000']" + - "asic_type not in ['cisco-8000'] or platform in ['x86_64-8122_64eh_o-r0']" qos/test_tunnel_qos_remap.py::test_pfc_watermark_extra_lossless_active: xfail: @@ -1334,6 +1405,12 @@ restapi/test_restapi.py: conditions: - "asic_type not in ['mellanox']" +restapi/test_restapi.py::test_create_vrf: + skip: + reason: "Only supported on Mellanox T1" + conditions: + - "'t1' not in topo_type" + restapi/test_restapi_vxlan_ecmp.py: skip: reason: "Only supported on cisco 8102 T1" @@ -1343,19 +1420,34 @@ restapi/test_restapi_vxlan_ecmp.py: ####################################### ##### route ##### ####################################### +route/test_default_route.py: + skip: + reason: "Does not apply to standalone topos." + conditions: + - "'standalone' in topo_name" + route/test_route_flap.py: skip: - reason: "Test case has issue on the t0-56-povlan and dualtor-64 topo." + reason: "Test case has issue on the t0-56-povlan and dualtor-64 topo. Does not apply to standalone topos." conditions_logical_operator: or conditions: - "https://github.com/sonic-net/sonic-mgmt/issues/11323 and 't0-56-po2vlan' in topo_name" - "https://github.com/sonic-net/sonic-mgmt/issues/11324 and 'dualtor-64' in topo_name" + - "'standalone' in topo_name" + +route/test_route_perf.py: + skip: + reason: "Does not apply to standalone topos." + conditions: + - "'standalone' in topo_name" route/test_static_route.py: skip: - reason: "Test not supported for 201911 images or older." + reason: "Test not supported for 201911 images or older. Does not apply to standalone topos." + conditions_logical_operator: OR conditions: - "release in ['201811', '201911']" + - "'standalone' in topo_name" route/test_static_route.py::test_static_route_ecmp_ipv6: # This test case may fail due to a known issue https://github.com/sonic-net/sonic-buildimage/issues/4930. @@ -1386,15 +1478,6 @@ show_techsupport/test_auto_techsupport.py::TestAutoTechSupport::test_sai_sdk_dum conditions: - "asic_type not in ['mellanox']" -show_techsupport/test_techsupport.py::test_techsupport: - xfail: - reason: "Test issue" - strict: True - conditions: - - "release in ['202305']" - - https://github.com/sonic-net/sonic-mgmt/issues/7520 - - "asic_type not in ['mellanox']" - ####################################### ##### snappi_tests ##### ####################################### @@ -1407,19 +1490,28 @@ snappi_tests/ecn/test_red_accuracy_with_snappi: ####################################### ##### snmp ##### ####################################### +snmp/test_snmp_default_route.py::test_snmp_default_route: + skip: + reason: "Skipping SNMP test because standalone topology has no default routes." + conditions: + - "'standalone' in topo_name" + snmp/test_snmp_link_local.py: skip: reason: "SNMP over IPv6 support not present in release branches." conditions: - - https://github.com/sonic-net/sonic-buildimage/issues/6108 - "is_multi_asic==False" - - "release in ['202205', '202211', '202305']" + - "release in ['202205', '202211', '202305', '202311']" snmp/test_snmp_loopback.py::test_snmp_loopback: skip: - reason: "Not supported topology backend." + reason: "1. Not supported topology backend. + 2. Skipping SNMP test because standalone topology has no neighbor VMs, + and SNMP queries will be executed from the neighbor VM." + conditions_logical_operator: OR conditions: - "'backend' in topo_name" + - "'standalone' in topo_name" snmp/test_snmp_pfc_counters.py: skip: @@ -1433,6 +1525,13 @@ snmp/test_snmp_queue.py: conditions: - "topo_type in ['m0', 'mx'] or asic_type in ['barefoot']" +snmp/test_snmp_queue_counters.py: + skip: + reason: "Have an known issue on kvm testbed" + conditions: + - asic_type in ['vs'] + - https://github.com/sonic-net/sonic-mgmt/issues/14007 + ####################################### ##### span ##### ####################################### @@ -1471,6 +1570,13 @@ sub_port_interfaces: conditions: - "is_multi_asic==True or asic_gen not in ['td2', 'spc1', 'spc2', 'spc3', 'spc4'] and asic_type not in ['barefoot','innovium']" +sub_port_interfaces/test_show_subinterface.py::test_subinterface_status[port]: + skip: + reason: "There is an image issue on kvm testbed" + conditions: + - asic_type in ['vs'] + - https://github.com/sonic-net/sonic-buildimage/issues/19735 + sub_port_interfaces/test_show_subinterface.py::test_subinterface_status[port_in_lag]: skip: reason: "Not supported port type" @@ -1497,38 +1603,33 @@ sub_port_interfaces/test_sub_port_l2_forwarding.py::test_sub_port_l2_forwarding[ ##### syslog ##### ####################################### syslog/test_syslog.py: - skip: - reason: "Testcase enhancements needed for backend topo" - conditions: - - "topo_name in ['t0-backend', 't1-backend']" - - https://github.com/sonic-net/sonic-mgmt/issues/4469 xfail: reason: "Generic internal image issue" conditions: - "branch in ['internal-202012']" - "build_version.split('.')[1].isdigit() and int(build_version.split('.')[1]) <= 33" -syslog/test_syslog_rate_limit.py: - xfail: - reason: "Testcase xfail, raised issue to track" +syslog/test_syslog_source_ip.py: + skip: + reason: "Vs setup doesn't work when creating mgmt vrf" conditions: - - https://github.com/sonic-net/sonic-mgmt/issues/11181 + - "asic_type in ['vs']" -syslog/test_syslog_source_ip.py: +syslog/test_syslog_source_ip.py::TestSSIP::test_syslog_config_work_after_reboot: skip: reason: "Testcase consistent failed, raised issue to track" conditions: - - https://github.com/sonic-net/sonic-mgmt/issues/6479 + - https://github.com/sonic-net/sonic-buildimage/issues/19638 -####################################### -##### system_health ##### -####################################### -system_health/test_system_health.py::test_device_checker: +syslog/test_syslog_source_ip.py::TestSSIP::test_syslog_protocol_filter_severity: skip: - reason: "Temporary skip for Mellanox platforms" + reason: "Testcase consistent failed, raised issue to track" conditions: - - "asic_type in ['mellanox']" + - https://github.com/sonic-net/sonic-mgmt/issues/14493 +####################################### +##### system_health ##### +####################################### system_health/test_system_health.py::test_service_checker_with_process_exit: xfail: strict: True @@ -1553,6 +1654,11 @@ telemetry/test_events.py: reason: "Skip telemetry test events for 202211 and older branches" conditions: - "release in ['201811', '201911', '202012', '202205', '202211']" + xfail: + reason: "Test events is flaky in PR test, xfail until issue resolved" + conditions: + - "asic_type in ['vs']" + - https://github.com/sonic-net/sonic-buildimage/issues/19943 telemetry/test_telemetry.py: skip: @@ -1560,27 +1666,15 @@ telemetry/test_telemetry.py: conditions: - "(is_multi_asic==True) and (release in ['201811', '201911'])" -telemetry/test_telemetry.py::test_osbuild_version: - skip: - reason: "Testcase ignored due to Github issue: https://github.com/sonic-net/sonic-mgmt/issues/12021" - conditions: - - https://github.com/sonic-net/sonic-mgmt/issues/12021 - -####################################### -##### neighbor health ##### -####################################### -test_nbr_health.py: - skip: - reason: "Testcase ignored due to sonic-mgmt issue: https://github.com/sonic-net/sonic-mgmt/issues/7883" - conditions: - - "https://github.com/sonic-net/sonic-mgmt/issues/7883" - ####################################### ##### pktgen ##### ####################################### test_pktgen.py: skip: - reason: "have known issue, skip for now" + reason: "Have known issue, only running for 'cisco-8000', skipping for all other ASIC types" + conditions: + - "asic_type not in ['cisco-8000']" + - "https://github.com/sonic-net/sonic-mgmt/issues/13804" ####################################### ##### vlan ##### @@ -1589,19 +1683,24 @@ vlan/test_vlan.py::test_vlan_tc7_tagged_qinq_switch_on_outer_tag: skip: reason: "Unsupported platform." conditions: - - "asic_type not in ['mellanox', 'barefoot']" + - "asic_type not in ['mellanox', 'barefoot', 'cisco-8000']" vlan/test_vlan_ping.py: skip: - reason: "test_vlan_ping doesn't work on Broadcom platform. Ignored on dualtor topo and mellanox and Cisco-8000 setups due to Github issue: https://github.com/sonic-net/sonic-mgmt/issues/9642." + reason: "test_vlan_ping doesn't work on Broadcom platform. Ignored on dualtor topo and mellanox setups due to Github issue: https://github.com/sonic-net/sonic-mgmt/issues/9642." conditions_logical_operator: OR conditions: - "asic_type in ['broadcom']" - - "https://github.com/sonic-net/sonic-mgmt/issues/9642 and 'dualtor' in topo_name and asic_type in ['mellanox', 'cisco-8000']" ####################################### ##### voq ##### ####################################### +voq: + skip: + reason: "Cisco 8800 doesn't support voq tests" + conditions: + - "asic_type in ['cisco-8000']" + voq/test_fabric_cli_and_db.py: skip: reason: "Skip test_fabric_cli_and_db on unsupported testbed." @@ -1659,32 +1758,104 @@ vxlan/test_vnet_vxlan.py: 2. Test skipped due to issue #8374" conditions_logical_operator: OR conditions: - - "asic_type not in ['mellanox', 'barefoot']" + - "asic_type not in ['mellanox', 'barefoot', 'vs']" - https://github.com/sonic-net/sonic-mgmt/issues/8374 vxlan/test_vxlan_bfd_tsa.py: skip: - reason: "VxLAN ECMP BFD TSA test is not yet supported on multi-ASIC platform. Also this test can only run on 4600c, 2700 and 8102." + reason: "VxLAN ECMP BFD TSA test is not yet supported on multi-ASIC platform. Also this test can only run on some platforms." conditions: - - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0'])" + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0'])" + +vxlan/test_vxlan_bfd_tsa.py::Test_VxLAN_BFD_TSA::test_tsa_case4: + skip: + reason: "VxLAN ECMP BFD TSA test is not yet supported on multi-ASIC platform. Also this test can only run on 4600c, 2700 and 8102. For KVM platform, this test is not supported due to system check not supported currently." + conditions_logical_operator: OR + conditions: + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0'])" + - "(asic_type in ['vs']) and https://github.com/sonic-net/sonic-buildimage/issues/19879" + +vxlan/test_vxlan_bfd_tsa.py::Test_VxLAN_BFD_TSA::test_tsa_case5: + skip: + reason: "VxLAN ECMP BFD TSA test is not yet supported on multi-ASIC platform. Also this test can only run on 4600c, 2700 and 8102. For KVM platform, this test is not supported due to system check not supported currently." + conditions_logical_operator: OR + conditions: + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0'])" + - "(asic_type in ['vs']) and https://github.com/sonic-net/sonic-buildimage/issues/19879" + +vxlan/test_vxlan_bfd_tsa.py::Test_VxLAN_BFD_TSA::test_tsa_case6: + skip: + reason: "VxLAN ECMP BFD TSA test is not yet supported on multi-ASIC platform. Also this test can only run on 4600c, 2700 and 8102. For KVM platform, this test is not supported due to system check not supported currently." + conditions_logical_operator: OR + conditions: + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0'])" + - "(asic_type in ['vs']) and https://github.com/sonic-net/sonic-buildimage/issues/19879" vxlan/test_vxlan_crm.py: skip: - reason: "VxLAN crm test is not yet supported on multi-ASIC platform. Also this test can only run on 4600c, 2700 and 8102." + reason: "VxLAN crm test is not yet supported on multi-ASIC platform. Also this test can only run on some platforms." + conditions: + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0'])" + +vxlan/test_vxlan_crm.py::Test_VxLAN_Crm::test_crm_128_group_members[v4_in_v6]: + skip: + reason: "VxLAN crm test is not yet supported on multi-ASIC platform. Also this test can only run on some platforms. On Mellanox spc1 platform, due to HW limitation, vxlan ipv6 tunnel is not supported" + conditions_logical_operator: OR + conditions: + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0'])" + - "asic_gen == 'spc1'" + +vxlan/test_vxlan_crm.py::Test_VxLAN_Crm::test_crm_128_group_members[v6_in_v6]: + skip: + reason: "VxLAN crm test is not yet supported on multi-ASIC platform. Also this test can only run on some platforms. On Mellanox spc1 platform, due to HW limitation, vxlan ipv6 tunnel is not supported" + conditions_logical_operator: OR + conditions: + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0'])" + - "asic_gen == 'spc1'" + +vxlan/test_vxlan_crm.py::Test_VxLAN_Crm::test_crm_16k_routes[v4_in_v6]: + skip: + reason: "VxLAN crm test is not yet supported on multi-ASIC platform. Also this test can only run on some platforms. On Mellanox spc1 platform, due to HW limitation, vxlan ipv6 tunnel is not supported" + conditions_logical_operator: OR conditions: - - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0'])" + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0'])" + - "asic_gen == 'spc1'" + +vxlan/test_vxlan_crm.py::Test_VxLAN_Crm::test_crm_16k_routes[v6_in_v6]: + skip: + reason: "VxLAN crm test is not yet supported on multi-ASIC platform. Also this test can only run on some platforms. On Mellanox spc1 platform, due to HW limitation, vxlan ipv6 tunnel is not supported" + conditions_logical_operator: OR + conditions: + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0'])" + - "asic_gen == 'spc1'" + +vxlan/test_vxlan_crm.py::Test_VxLAN_Crm::test_crm_512_nexthop_groups[v4_in_v6]: + skip: + reason: "VxLAN crm test is not yet supported on multi-ASIC platform. Also this test can only run on some platforms. On Mellanox spc1 platform, due to HW limitation, vxlan ipv6 tunnel is not supported" + conditions_logical_operator: OR + conditions: + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0'])" + - "asic_gen == 'spc1'" + +vxlan/test_vxlan_crm.py::Test_VxLAN_Crm::test_crm_512_nexthop_groups[v6_in_v6]: + skip: + reason: "VxLAN crm test is not yet supported on multi-ASIC platform. Also this test can only run on some platforms. On Mellanox spc1 platform, due to HW limitation, vxlan ipv6 tunnel is not supported" + conditions_logical_operator: OR + conditions: + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0'])" + - "asic_gen == 'spc1'" vxlan/test_vxlan_ecmp.py: skip: - reason: "VxLAN ECMP test is not yet supported on multi-ASIC platform. Also this test can only run on 4600c, 2700 and 8102." + reason: "VxLAN ECMP test is not yet supported on multi-ASIC platform. Also this test can only run on some platforms." conditions: - - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0'])" + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0'])" vxlan/test_vxlan_ecmp_switchover.py: skip: - reason: "VxLAN ECMP switchover test is not yet supported on multi-ASIC platform. Also this test can only run on 4600c, 2700 and 8102." + reason: "VxLAN ECMP switchover test is not yet supported on multi-ASIC platform. Also this test can only run on some platforms." conditions: - - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0'])" + - "(is_multi_asic == True) or (platform not in ['x86_64-8102_64h_o-r0', 'x86_64-8101_32fh_o-r0', 'x86_64-mlnx_msn4600c-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2700a1-r0', 'x86_64-kvm_x86_64-r0', 'x86_64-mlnx_msn4700-r0', 'x86_64-nvidia_sn4280-r0'])" ####################################### ##### wan_lacp ##### diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions_acl.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions_acl.yaml index bfd3bf5b19..69853395a2 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions_acl.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions_acl.yaml @@ -250,12 +250,22 @@ acl/test_acl.py::TestAclWithPortToggle::test_udp_source_ip_match_dropped[ipv6-eg - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 +acl/test_acl.py::TestAclWithReboot: + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" + acl/test_acl.py::TestAclWithReboot::test_dest_ip_match_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: reason: "Egress issue in Nokia" conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_dest_ip_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -263,6 +273,10 @@ acl/test_acl.py::TestAclWithReboot::test_dest_ip_match_dropped[ipv6-egress-uplin conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_dest_ip_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: @@ -270,6 +284,10 @@ acl/test_acl.py::TestAclWithReboot::test_dest_ip_match_dropped[ipv6-egress-uplin conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_icmp_match_forwarded[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: @@ -277,6 +295,10 @@ acl/test_acl.py::TestAclWithReboot::test_icmp_match_forwarded[ipv6-egress-uplink conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_icmp_match_forwarded[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -284,6 +306,10 @@ acl/test_acl.py::TestAclWithReboot::test_icmp_match_forwarded[ipv6-egress-uplink conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_icmp_match_forwarded[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: @@ -291,6 +317,10 @@ acl/test_acl.py::TestAclWithReboot::test_icmp_match_forwarded[ipv6-egress-uplink conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_icmp_source_ip_match_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: @@ -298,6 +328,10 @@ acl/test_acl.py::TestAclWithReboot::test_icmp_source_ip_match_dropped[ipv6-egres conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_icmp_source_ip_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -305,6 +339,10 @@ acl/test_acl.py::TestAclWithReboot::test_icmp_source_ip_match_dropped[ipv6-egres conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_icmp_source_ip_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: @@ -312,6 +350,10 @@ acl/test_acl.py::TestAclWithReboot::test_icmp_source_ip_match_dropped[ipv6-egres conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_ip_proto_match_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: @@ -319,6 +361,10 @@ acl/test_acl.py::TestAclWithReboot::test_ip_proto_match_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_ip_proto_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -326,6 +372,10 @@ acl/test_acl.py::TestAclWithReboot::test_ip_proto_match_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_ip_proto_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: @@ -333,6 +383,10 @@ acl/test_acl.py::TestAclWithReboot::test_ip_proto_match_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_dport_match_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: @@ -340,6 +394,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_dport_match_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_dport_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -347,6 +405,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_dport_match_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_dport_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: @@ -354,6 +416,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_dport_match_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_dport_range_match_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: @@ -361,6 +427,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_dport_range_match_dropped[ipv6-egres conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_dport_range_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -368,7 +438,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_dport_range_match_dropped[ipv6-egres conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 - + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_dport_range_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: @@ -376,6 +449,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_dport_range_match_dropped[ipv6-egres conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_sport_match_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: @@ -383,6 +460,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_sport_match_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_sport_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -390,6 +471,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_sport_match_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_sport_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: @@ -397,6 +482,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_sport_match_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_sport_range_match_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: @@ -404,6 +493,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_sport_range_match_dropped[ipv6-egres conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_sport_range_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -411,6 +504,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_sport_range_match_dropped[ipv6-egres conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_l4_sport_range_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: @@ -418,6 +515,10 @@ acl/test_acl.py::TestAclWithReboot::test_l4_sport_range_match_dropped[ipv6-egres conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_rules_priority_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: @@ -425,6 +526,10 @@ acl/test_acl.py::TestAclWithReboot::test_rules_priority_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_rules_priority_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -432,6 +537,10 @@ acl/test_acl.py::TestAclWithReboot::test_rules_priority_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_rules_priority_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: @@ -439,6 +548,10 @@ acl/test_acl.py::TestAclWithReboot::test_rules_priority_dropped[ipv6-egress-upli conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_source_ip_match_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: @@ -446,6 +559,10 @@ acl/test_acl.py::TestAclWithReboot::test_source_ip_match_dropped[ipv6-egress-upl conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_source_ip_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -453,12 +570,21 @@ acl/test_acl.py::TestAclWithReboot::test_source_ip_match_dropped[ipv6-egress-upl conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" + acl/test_acl.py::TestAclWithReboot::test_source_ip_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: reason: "Egress issue in Nokia" conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_tcp_flags_match_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: @@ -466,6 +592,10 @@ acl/test_acl.py::TestAclWithReboot::test_tcp_flags_match_dropped[ipv6-egress-upl conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_tcp_flags_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -473,6 +603,10 @@ acl/test_acl.py::TestAclWithReboot::test_tcp_flags_match_dropped[ipv6-egress-upl conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_tcp_flags_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: @@ -480,6 +614,10 @@ acl/test_acl.py::TestAclWithReboot::test_tcp_flags_match_dropped[ipv6-egress-upl conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_udp_source_ip_match_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: @@ -487,6 +625,10 @@ acl/test_acl.py::TestAclWithReboot::test_udp_source_ip_match_dropped[ipv6-egress conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_udp_source_ip_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan1000]: xfail: @@ -494,7 +636,10 @@ acl/test_acl.py::TestAclWithReboot::test_udp_source_ip_match_dropped[ipv6-egress conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 - + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestAclWithReboot::test_udp_source_ip_match_dropped[ipv6-egress-uplink->downlink-m0_vlan_scenario-Vlan2000]: xfail: @@ -502,6 +647,10 @@ acl/test_acl.py::TestAclWithReboot::test_udp_source_ip_match_dropped[ipv6-egress conditions: - "platform in ['armhf-nokia_ixs7215_52x-r0']" - https://github.com/sonic-net/sonic-mgmt/issues/8639 + skip: + reason: "Skip in t1-lag KVM test due to test time too long and t0 would cover this testbeds" + conditions: + - "topo_type in ['t1'] and asic_type in ['vs']" acl/test_acl.py::TestBasicAcl::test_dest_ip_match_dropped[ipv6-egress-uplink->downlink-default-Vlan1000]: xfail: diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions_drop_packets.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions_drop_packets.yaml index fdc315fe15..b1bd625186 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions_drop_packets.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions_drop_packets.yaml @@ -3,12 +3,27 @@ #################################################### #Link local address(169.254.xxx.xxx) as a source address as IPv4 header is not invalid in all the cases #Hence, it is not dropped by default in Cisco-8000. For dropping link local address, it should be done through security/DATA ACL -drop_packets/test_configurable_drop_counters.py::test_sip_link_local: +drop_packets/test_configurable_drop_counters.py::test_dip_link_local: + skip: + reason: "Cisco 8000 platform and some mlx platforms does not drop DIP link local packets" + conditions_logical_operator: or + conditions: + - "'Mellanox' in hwsku" + - asic_type=='cisco-8000' + +drop_packets/test_configurable_drop_counters.py::test_neighbor_link_down: skip: - reason: "Cisco 8000 platform does not drop SIP link local packets" + reason: "This test case requires a T0 topology because it is mocking a server within VLAN." conditions: - - asic_type=="cisco-8000" + - "topo_type not in ['t0']" +drop_packets/test_configurable_drop_counters.py::test_sip_link_local: + skip: + reason: "Cisco 8000 platform and some MLX platforms does not drop SIP link local packets" + conditions_logical_operator: or + conditions: + - asic_type=="cisco-8000" + - "'Mellanox' in hwsku" ####################################### ##### test_drop_counters.py ##### ####################################### @@ -54,9 +69,11 @@ drop_packets/test_drop_counters.py::test_dst_ip_is_loopback_addr[vlan_members]: drop_packets/test_drop_counters.py::test_dst_ip_link_local: skip: - reason: "Cisco 8000 and broadcom DNX platforms do not drop DIP linklocal packets" + reason: "Cisco 8000 broadcom DNX platforms and some MLX platforms do not drop DIP linklocal packets" + conditions_logical_operator: or conditions: - "(asic_type=='cisco-8000') or (asic_subtype in ['broadcom-dnx'])" + - "'Mellanox' in hwsku" drop_packets/test_drop_counters.py::test_equal_smac_dmac_drop: skip: @@ -72,36 +89,49 @@ drop_packets/test_drop_counters.py::test_ip_is_zero_addr: drop_packets/test_drop_counters.py::test_ip_is_zero_addr[vlan_members-ipv4-dst]: skip: - reason: "Image issue on Boradcom dualtor testbeds. Cisco 8000 platform does not drop packets with 0.0.0.0 source or destination IP address" + reason: + - "Image issue on Broadcom dualtor testbeds. Cisco 8000 platform does not drop packets with 0.0.0.0 source or destination IP address" + - "Test case requires topology type t0 for vlan testing" strict: True + conditions_logical_operator: or conditions: - - "asic_type in ['broadcom', 'cisco-8000']" - - "topo_name in ['dualtor', 'dualtor-56', 'dualtor-120']" + - "asic_type in ['broadcom', 'cisco-8000'] and topo_name in ['dualtor', 'dualtor-56', 'dualtor-120']" + - topo_type not in ['t0'] drop_packets/test_drop_counters.py::test_ip_is_zero_addr[vlan_members-ipv4-src]: skip: - reason: "Image issue on Boradcom dualtor testbeds. Cisco 8000 platform does not drop packets with 0.0.0.0 source or destination IP address" + reason: + - "Image issue on Broadcom dualtor testbeds. Cisco 8000 platform does not drop packets with 0.0.0.0 source or destination IP address" + - "Test case requires topology type t0 for vlan testing" strict: True conditions: - - "asic_type in ['broadcom', 'cisco-8000']" - - "topo_name in ['dualtor', 'dualtor-56', 'dualtor-120']" + - "asic_type in ['broadcom', 'cisco-8000'] and topo_name in ['dualtor', 'dualtor-56', 'dualtor-120']" + - topo_type not in ['t0'] drop_packets/test_drop_counters.py::test_ip_is_zero_addr[vlan_members-ipv6-dst]: skip: - reason: "Image issue on Boradcom dualtor testbeds. Cisco 8000 platform does not drop packets with 0.0.0.0 source or destination IP address" + reason: + - "Image issue on Broadcom dualtor testbeds. Cisco 8000 platform does not drop packets with 0.0.0.0 source or destination IP address" + - "Test case requires topology type t0 for vlan testing" strict: True conditions: - - "asic_type in ['broadcom', 'cisco-8000']" - - "topo_name in ['dualtor', 'dualtor-56', 'dualtor-120']" + - "asic_type in ['broadcom', 'cisco-8000'] and topo_name in ['dualtor', 'dualtor-56', 'dualtor-120']" + - topo_type not in ['t0'] drop_packets/test_drop_counters.py::test_ip_is_zero_addr[vlan_members-ipv6-src]: skip: - reason: "Image issue on Boradcom dualtor testbeds. Cisco 8000 platform does not drop packets with 0.0.0.0 source or destination IP address" + reason: "Image issue on Broadcom dualtor testbeds. Cisco 8000 platform does not drop packets with 0.0.0.0 source or destination IP address" strict: True conditions: - "asic_type in ['broadcom', 'cisco-8000']" - "topo_name in ['dualtor', 'dualtor-56', 'dualtor-120']" +drop_packets/test_drop_counters.py::test_ip_pkt_with_expired_ttl: + skip: + reason: "Not supported on Mellanox devices" + conditions: + - "asic_type in ['mellanox']" + drop_packets/test_drop_counters.py::test_loopback_filter: # Test case is skipped, because SONiC does not have a control to adjust loop-back filter settings. # Default SONiC behavior is to forward the traffic, so loop-back filter does not triggers for IP packets. @@ -111,6 +141,12 @@ drop_packets/test_drop_counters.py::test_loopback_filter: skip: reason: "SONiC can't enable loop-back filter feature" +drop_packets/test_drop_counters.py::test_no_egress_drop_on_down_link: + skip: + reason: "VS platform do not support fanout configuration" + conditions: + - "asic_type in ['vs']" + drop_packets/test_drop_counters.py::test_not_expected_vlan_tag_drop[vlan_members]: skip: reason: "Image issue on Boradcom dualtor testbeds" @@ -163,6 +199,8 @@ drop_packets/test_drop_counters.py::test_src_ip_is_multicast_addr[vlan_members-i drop_packets/test_drop_counters.py::test_src_ip_link_local: skip: - reason: "Cisco 8000 platform does not drop SIP link local packets" + reason: "Cisco 8000 broadcom DNX platforms and some MLX platforms do not drop SIP linklocal packets" + conditions_logical_operator: or conditions: - - "asic_type=='cisco-8000'" + - "(asic_type=='cisco-8000') or (asic_subtype in ['broadcom-dnx'])" + - "'Mellanox' in hwsku" diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions_platform_tests.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions_platform_tests.yaml index 0a3c671986..ea366c11a4 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions_platform_tests.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions_platform_tests.yaml @@ -371,6 +371,12 @@ platform_tests/api/test_psu.py::TestPsuApi::test_temperature: ####################################### ##### api/test_psu_fans.py ##### ####################################### +platform_tests/api/test_psu.py::test_temperature: + skip: + reason: "Test not supported on Mellanox Platforms." + conditions: + - "asic_type in ['mellanox']" + platform_tests/api/test_psu_fans.py::TestPsuFans::test_get_error_description: xfail: reason: "Testcase consistently fails, raised issue to track" @@ -526,8 +532,10 @@ platform_tests/api/test_sfp.py::TestSfpApi::test_power_override: platform_tests/api/test_sfp.py::TestSfpApi::test_reset: skip: reason: "Unsupported platform API" + conditions_logical_operator: or conditions: - "'sw_to3200k' in hwsku or asic_type in ['nvidia-bluefield']" + - "platform in ['x86_64-cel_e1031-r0']" platform_tests/api/test_sfp.py::TestSfpApi::test_thermals: skip: @@ -545,7 +553,7 @@ platform_tests/api/test_sfp.py::TestSfpApi::test_tx_disable_channel: skip: reason: "Unsupported platform API" conditions: - - "asic_type in ['mellanox'] or (asic_type in ['barefoot'] and hwsku in ['newport']) or platform in ['armhf-nokia_ixs7215_52x-r0']" + - "asic_type in ['mellanox'] or (asic_type in ['barefoot'] and hwsku in ['newport']) or platform in ['armhf-nokia_ixs7215_52x-r0', 'x86_64-cel_e1031-r0']" ####################################### ##### api/test_thermal.py ##### @@ -693,6 +701,9 @@ platform_tests/cli/test_show_platform.py::test_show_platform_psustatus: conditions: - "hwsku in ['Celestica-DX010-C32']" - https://github.com/sonic-net/sonic-mgmt/issues/6518 + skip: + reason: "Test should be skipped on DPU for it doesn't have PSUs." + conditions: "'nvda_bf' in platform" platform_tests/cli/test_show_platform.py::test_show_platform_psustatus_json: xfail: @@ -700,6 +711,9 @@ platform_tests/cli/test_show_platform.py::test_show_platform_psustatus_json: conditions: - "hwsku in ['Celestica-DX010-C32']" - https://github.com/sonic-net/sonic-mgmt/issues/6518 + skip: + reason: "Test should be skipped on DPU for it doesn't have PSUs." + conditions: "'nvda_bf' in platform" platform_tests/cli/test_show_platform.py::test_show_platform_syseeprom: xfail: @@ -718,8 +732,15 @@ platform_tests/counterpoll/test_counterpoll_watermark.py::test_counterpoll_queue - "topo_type in ['m0', 'mx']" ####################################### -##### daemon/test_syseepromd.py ##### +##### daemon ##### ####################################### +platform_tests/daemon/test_chassisd.py: + skip: + reason: "chassisd platform daemon introduced in 202106" + conditions: + - "release in ['201811', '201911', '202012']" + + platform_tests/daemon/test_ledd.py::test_pmon_ledd_kill_and_start_status: skip: reason: "LEDD daemon auto restart not included in 201911" @@ -749,11 +770,36 @@ platform_tests/mellanox: conditions: - "asic_type not in ['mellanox', 'nvidia-bluefield']" +platform_tests/mellanox/test_check_sfp_using_ethtool.py: + skip: + reason: "Deprecated as Mellanox do not use ethtool in release 202305 or higher" + conditions: + - "asic_type in ['mellanox']" + platform_tests/mellanox/test_reboot_cause.py: skip: - reason: "Does not support platform_tests/mellanox/test_reboot_cause.py" + reason: "Does not support platform_tests/mellanox/test_reboot_cause.py. sn4280 driver doesn't support reset_from_asic and reset_reload_bios" conditions: - - "platform in ['x86_64-mlnx_msn2010-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2100-r0', 'x86_64-mlnx_msn2410-r0', 'x86_64-nvidia_sn2201-r0']" + - "platform in ['x86_64-mlnx_msn2010-r0', 'x86_64-mlnx_msn2700-r0', 'x86_64-mlnx_msn2100-r0', 'x86_64-mlnx_msn2410-r0', 'x86_64-nvidia_sn2201-r0', 'x86_64-nvidia_sn4280-r0']" + +####################################### +##### sfp ##### +####################################### +platform_tests/sfp/test_sfputil.py::test_check_sfputil_low_power_mode: + skip: + reason: "Get/Set low power mode is not supported in Cisco 8000 platform or in bluefield platform" + conditions_logical_operator: or + conditions: + - "platform in ['x86_64-cel_e1031-r0']" + - "asic_type in ['nvidia-bluefield']" + +platform_tests/sfp/test_sfputil.py::test_check_sfputil_reset: + skip: + reason: "platform does not support sfp reset" + conditions_logical_operator: or + conditions: + - "(platform in ['x86_64-cel_e1031-r0'] or 't2' in topo_name) and https://github.com/sonic-net/sonic-mgmt/issues/6218" + - "asic_type in ['nvidia-bluefield']" ####################################### #### test_advanced_reboot ##### @@ -772,6 +818,15 @@ platform_tests/test_advanced_reboot.py::test_fast_reboot_from_other_vendor: conditions: - https://github.com/sonic-net/sonic-mgmt/issues/11007 +####################################### +##### test_cont_warm_reboot.py ##### +####################################### +platform_tests/test_cont_warm_reboot.py: + skip: + reason: "Warm Reboot is not supported in dualtor" + conditions: + - "'dualtor' in topo_name" + ####################################### #### test_kdump.py ##### ####################################### @@ -795,11 +850,25 @@ platform_tests/test_memory_exhaustion.py: ####################################### ##### test_platform_info.py ##### ####################################### +platform_tests/test_platform_info.py::test_show_platform_fanstatus_mocked: + skip: + reason: "Test disabled on Mellanox platforms as it could be un-stable and interfered by the thermal algorithm." + conditions: + - "asic_type in ['mellanox']" + +platform_tests/test_platform_info.py::test_show_platform_temperature_mocked: + skip: + reason: "Test not supported on Mellanox Platforms." + conditions: + - "asic_type in ['mellanox']" + platform_tests/test_platform_info.py::test_thermal_control_fan_status: skip: - reason: "Thermal control daemon is not present" + reason: "Thermal control daemon is not present / Test disabled on Mellanox platforms as it could be un-stable and interfered by the thermal algorithm." + conditions_logical_operator: or conditions: - "is_multi_asic==True and release in ['201911']" + - "asic_type in ['mellanox']" platform_tests/test_platform_info.py::test_thermal_control_load_invalid_format_json: #Thermal policies are implemented as part of BSP layer in Cisco 8000 platform, so there is no need for loading JSON file, @@ -808,9 +877,11 @@ platform_tests/test_platform_info.py::test_thermal_control_load_invalid_format_j # Cisco platforms use different mechanism to generate thermal policy, current method is not applicable # Multi ASIC platform running 201911 release doesn't have thermalctld # s6100 has not supported get_thermal_manager yet - reason: "Skip on Cisco platform and multi-asic platform running 201911 release" + reason: "Skip on Cisco platform and multi-asic platform running 201911 release / Test not required on Mellanox Platforms as all thermal policies have been removed from thermalctld" + conditions_logical_operator: or conditions: - "asic_type=='cisco-8000' or (is_multi_asic==True and release in ['201911']) or ('dell_s6100' in platform) or ('sw_to3200k' in hwsku)" + - "asic_type in ['mellanox']" platform_tests/test_platform_info.py::test_thermal_control_load_invalid_value_json: #Thermal policies are implemented as part of BSP layer in Cisco 8000 platform, so there is no need for loading JSON file, @@ -819,9 +890,11 @@ platform_tests/test_platform_info.py::test_thermal_control_load_invalid_value_js # Cisco platforms use different mechanism to generate thermal policy, current method is not applicable # Multi ASIC platform running 201911 release doesn't have thermalctld # s6100 has not supported get_thermal_manager yet - reason: "Skip on Cisco platform and multi-asic platform running 201911 release" + reason: "Skip on Cisco platform and multi-asic platform running 201911 release / Test not required on Mellanox Platforms as all thermal policies have been removed from thermalctld" + conditions_logical_operator: or conditions: - "asic_type=='cisco-8000' or (is_multi_asic==True and release in ['201911']) or ('dell_s6100' in platform) or ('sw_to3200k' in hwsku)" + - "asic_type in ['mellanox']" platform_tests/test_platform_info.py::test_turn_on_off_psu_and_check_psustatus: xfail: @@ -849,9 +922,11 @@ platform_tests/test_reboot.py::test_cold_reboot: platform_tests/test_reboot.py::test_fast_reboot: skip: - reason: "Skip test_fast_reboot for m0/mx" + reason: "Skip test_fast_reboot for m0/mx/t1/t2 / Fast reboot is broken on dualtor topology. Skipping for now." + conditions_logical_operator: or conditions: - "topo_type in ['m0', 'mx', 't1', 't2']" + - "'dualtor' in topo_name and https://github.com/sonic-net/sonic-buildimage/issues/16502" xfail: reason: "case failed and waiting for fix" conditions: @@ -860,9 +935,9 @@ platform_tests/test_reboot.py::test_fast_reboot: platform_tests/test_reboot.py::test_soft_reboot: skip: - reason: "Skip test_soft_reboot for m0/mx" + reason: "Skip test_soft_reboot for m0/mx and test is supported only on S6100 hwsku" conditions: - - "topo_type in ['m0', 'mx']" + - "topo_type in ['m0', 'mx'] or hwsku not in ['Force10-S6100']" xfail: reason: "case failed and waiting for fix" conditions: @@ -871,9 +946,11 @@ platform_tests/test_reboot.py::test_soft_reboot: platform_tests/test_reboot.py::test_warm_reboot: skip: - reason: "Skip test_warm_reboot for m0/mx" + reason: "Skip test_warm_reboot for m0/mx/t1/t2 / Warm reboot is broken on dualtor topology. Skipping for now." + conditions_logical_operator: or conditions: - "topo_type in ['m0', 'mx', 't1', 't2']" + - "'dualtor' in topo_name and https://github.com/sonic-net/sonic-buildimage/issues/16502" xfail: reason: "case failed and waiting for fix" conditions: @@ -890,21 +967,33 @@ platform_tests/test_reboot.py::test_watchdog_reboot: ####################################### ##### test_reload_config.py ##### ####################################### -platform_tests/test_reload_config.py: +platform_tests/test_reload_config.py::test_reload_configuration: skip: - reason: "Unsupported platform API" + reason: "Unsupported platform API / Service watchdog-control can not load on kvm testbed." + conditions_logical_operator: or conditions: - "asic_type in ['barefoot'] and hwsku in ['newport']" + - "https://github.com/sonic-net/sonic-buildimage/issues/19879 and asic_type in ['vs']" +platform_tests/test_reload_config.py::test_reload_configuration_checks: + skip: + reason: "Unsupported platform API / Service watchdog-control can not load on kvm testbed / Skip test_reload_configuration_checks testcase due to flaky timing issue for Cisco 8000." + conditions_logical_operator: or + conditions: + - "asic_type in ['barefoot'] and hwsku in ['newport']" + - "https://github.com/sonic-net/sonic-buildimage/issues/19879 and asic_type in ['vs']" + - "asic_type in ['cisco-8000'] and release in ['202205', '202211', '202305', '202405']" ####################################### ###### test_secure_upgrade.py ####### ####################################### platform_tests/test_secure_upgrade.py: skip: - reason: "Skip test_secure_upgrade for m0/mx with 202305 release" + reason: "Skip test_secure_upgrade for m0/mx with 202305 release / platform does not support secure upgrade" + conditions_logical_operator: or conditions: - "topo_type in ['m0', 'mx'] and release in ['202305']" + - "'sn2' in platform or 'sn3' in platform or 'sn4' in platform" ####################################### ######### test_sensors.py ########### @@ -925,13 +1014,13 @@ platform_tests/test_sequential_restart.py::test_restart_syncd: reason: "Restarting syncd is not supported yet" ####################################### -##### test_service_warm_restart.py #### +#####test_service_warm_restart.py ##### ####################################### platform_tests/test_service_warm_restart.py: skip: - reason: "Skip test_service_warm_restart on mellanox platform" + reason: "Testcase ignored due to sonic-mgmt issue: https://github.com/sonic-net/sonic-mgmt/issues/10362" conditions: - - "asic_type in ['mellanox']" + - "https://github.com/sonic-net/sonic-mgmt/issues/10362" ####################################### ##### test_xcvr_info_in_db.py ##### diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions_skip_traffic_test.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions_skip_traffic_test.yaml new file mode 100644 index 0000000000..c9e35f2788 --- /dev/null +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions_skip_traffic_test.yaml @@ -0,0 +1,340 @@ +####################################### +##### acl ##### +####################################### +acl/custom_acl_table/test_custom_acl_table.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +acl/null_route/test_null_route_helper.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +acl/test_acl.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +acl/test_acl_outer_vlan.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +acl/test_stress_acl.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### arp ##### +####################################### +arp/test_stress_arp.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +arp/test_unknown_mac.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +arp/test_wr_arp.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### copp ##### +####################################### +copp/test_copp.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### decap ##### +####################################### +decap/test_decap.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +decap/test_subnet_decap.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### drop_packets ##### +####################################### +drop_packets/test_drop_counters.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### dualtor ##### +####################################### +dualtor/test_ipinip.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +dualtor/test_orchagent_active_tor_downstream.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +dualtor/test_orchagent_mac_move.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +dualtor/test_orchagent_slb.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +dualtor/test_orchagent_standby_tor_downstream.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +dualtor/test_standby_tor_upstream_mux_toggle.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +dualtor/test_tor_ecn.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +dualtor/test_tunnel_memory_leak.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### dualtor_io ##### +####################################### +dualtor_io/test_heartbeat_failure.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +dualtor_io/test_link_drop.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +dualtor_io/test_link_failure.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +dualtor_io/test_normal_op.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +dualtor_io/test_tor_bgp_failure.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### dualtor_mgmt ##### +####################################### +dualtor_mgmt/test_ingress_drop.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### ecmp ##### +####################################### +ecmp/inner_hashing/test_inner_hashing.py: + skip_traffic_test: + conditions_logical_operator: or + reason: "Skip traffic test if not in mellanox platform." + conditions: + - "platform not in ['x86_64-mlnx_msn3800-r0', 'x86_64-mlnx_msn4600c-r0']" + - "asic_type not in ['mellanox']" + +ecmp/inner_hashing/test_inner_hashing_lag.py: + skip_traffic_test: + conditions_logical_operator: or + reason: "Skip traffic test if not in mellanox platform." + conditions: + - "platform not in ['x86_64-mlnx_msn3800-r0', 'x86_64-mlnx_msn4600c-r0']" + - "asic_type not in ['mellanox']" + +ecmp/inner_hashing/test_wr_inner_hashing.py: + skip_traffic_test: + conditions_logical_operator: or + reason: "Skip traffic test if not in mellanox platform." + conditions: + - "platform not in ['x86_64-mlnx_msn3800-r0', 'x86_64-mlnx_msn4600c-r0']" + - "asic_type not in ['mellanox']" + +ecmp/inner_hashing/test_wr_inner_hashing_lag.py: + skip_traffic_test: + conditions_logical_operator: or + reason: "Skip traffic test if not in mellanox platform." + conditions: + - "platform not in ['x86_64-mlnx_msn3800-r0', 'x86_64-mlnx_msn4600c-r0']" + - "asic_type not in ['mellanox']" + +####################################### +##### everflow ##### +####################################### +everflow/test_everflow_ipv6.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +everflow/test_everflow_per_interface.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +everflow/test_everflow_testbed.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### fib ##### +####################################### +fib/test_fib.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### generic_config_updater ##### +####################################### +generic_config_updater/test_dynamic_acl.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### hash ##### +####################################### +hash/test_generic_hash.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### ip ##### +####################################### +ip/test_ip_packet.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### ipfwd ##### +####################################### +ipfwd/test_dir_bcast.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### span ##### +####################################### +span/test_port_mirroring.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### sub_port_interfaces ##### +####################################### +sub_port_interfaces/test_sub_port_interfaces.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### vlan ##### +####################################### +vlan/test_vlan_ping.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +####################################### +##### vxlan ##### +####################################### +vxlan/test_vnet_vxlan.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +vxlan/test_vxlan_bfd_tsa.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +vxlan/test_vxlan_decap.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +vxlan/test_vxlan_ecmp.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + +vxlan/test_vxlan_ecmp_switchover.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" diff --git a/tests/common/plugins/custom_fixtures/check_dut_asic_type.py b/tests/common/plugins/custom_fixtures/check_dut_asic_type.py index b1ffe45fc5..f4caf13a9f 100644 --- a/tests/common/plugins/custom_fixtures/check_dut_asic_type.py +++ b/tests/common/plugins/custom_fixtures/check_dut_asic_type.py @@ -3,8 +3,8 @@ @pytest.fixture(scope="function") -def check_dut_asic_type(request, duthosts, rand_one_dut_hostname): - duthost = duthosts[rand_one_dut_hostname] +def check_dut_asic_type(request, duthosts, rand_one_dut_front_end_hostname): + duthost = duthosts[rand_one_dut_front_end_hostname] asic_marks = [mark for mark in request.node.iter_markers(name="asic")] if not asic_marks: return diff --git a/tests/common/plugins/memory_utilization/README.md b/tests/common/plugins/memory_utilization/README.md new file mode 100644 index 0000000000..306216a676 --- /dev/null +++ b/tests/common/plugins/memory_utilization/README.md @@ -0,0 +1,722 @@ +### Scope +This document provides a high level description of the memory utilization verification design and instructions on using the "memory_utilization" fixture in SONiC testing. + +### Overview +During testing, the memory usage of the DUT can vary due to different configurations, environment setups, and test steps. To ensure the safe use of memory resources, it is necessary to check the memory usage after the test to confirm that it has not exceeded the high memory usage threshold and that no memory leaks have occurred. + +The purpose of the current feature is to verify that memory resources do not increase during test runs and do not exceed the high memory usage threshold. + +### Module Design +Newly introduced a plugin "memory_utilization" and config files. + +#### Config files +Config files, including both common file and platform dependence file. +- **memory_utilization_common.json**: Common configurations for memory utilization. + - The file is in JSON format and is defined in the public branch. + - It includes "COMMON" key which holds all publicly defined common memory items. +- **memory_utilization_dependence.json**: Dependency configruations for memory utilization, currently supporting special configurations for specific HwSku devices. + - The dependency file is also in JSON format and is used in internal branch. + - It also includes a 'COMMON' key which holds all internal common memory items. If an internal memory item has the same name as an item in the public common configuration file, it will overwrite the public common item. + - The dependency file also could includes a "HWSKU" key for all hwsku related memory items. If the hwsku memory item is same as a common one, it will overwrite it. + - This dependency file should be left empty in the public branch, with special configurations added in internal branch. + +Memory utilization config files include memory items which need to check. +Each memory item include "name", "cmd", "memory_params" and "memory_check". +- **name**: The name of the memory check item. +- **cmd**: The shell command is run on the DUT to collect memory information. +- **memory_params**: The items and thresholds for memory usage, defined as a Dict type in the configuration JSON file. It could be modified in the test case. +- **memory_check**: The function used to parse the output of the shell command takes two input parameters: cmd's output string and memory_params. It returns the parsered memory inforamtion, which will be compared with "memory_params" to check for memory threshold. + +#### Workflow +1. Collect memory information based on memory utilization config files before running a test case. + - Execute the "cmd" and use the "memory_check" function to collect the memory information before running the test case. +2. Collect memory information based on memory utilization config files after the test case is completed. + - Execute the "cmd" and use the "memory_check" function to collect the memory information after running the test case. +3. Compare the memory information from before and after the test run. + - Compare the collected memory information with the thresholds in "memory_params". +4. Raise an alarm if there is any memory leak or if the memory usage exceeds the high memory threshold based on the memory utilization config files. + ``` + > pytest.fail(message) + E Failed: [ALARM]: monit:memory_usage memory usage 77.8 exceeds high threshold 70.0 + ``` + + +### Memory Utilization usage example + +Below is a description of the possible uses for the "memory_utilization" fixture/module. + +##### memory_utilization fixture +In the root conftest there is an implemented "memory_utilization" pytest fixture that starts automatically for all test cases. +The main flow of the fixture is as follows: +- memory_utilization collects memory information before the test case starts. +- memory_utilization collects memory information after the test case finishes. +- memory_utilization compares DUT memory usage and displays the results. +- if memory_utilization finds any exceeded thresholds for high memory usage or memory increase, it will display the result and pytest will generate an 'error'. + +#### To skip memory_utilization for: + +memory_utilization is enabled by default, if you want to skip the memory_utilization, please follow below steps +- For all test cases - use pytest command line option ```--disable_memory_utilization``` +- Per test case: mark test case with ```@pytest.mark.disable_memory_utilization``` decorator. Example is shown below. + ```python + pytestmark = [ + pytest.mark.disable_memory_utilization + ] + ``` + +#### Example of memory items configuration in json file +Current we support the "monit", "free", "docker" and "free" memory items. "monit" is already defined in common file, we can also define "top", "free" or "docker" in internal branch with below example. +The value in the following examples are for demonstration purposes only and are not actual values. +Please adjust the corrrsponding values according to your specific platform. + +##### "monit" +"monit" uses the command "sudo monit status" to get the memory item's information. +"monit" has three configurations in the example below, the first in the common file, the second in the dependency common configuration, and the third in the hwsku configuration. Therefore, the configuration from the hwsku should be use. +The threshold for high memory is 80%, and for an increase is 5%. +The function "parse_monit_status_output" parses the output of the command "sudo monit status" and returns the memory information 'monit':{'memory_usage':41.2}. +The Memory utilization fixture uses the function "parse_monit_status_output" to parse the output of "sudo monit status" before and after the test case. It then compares the value with the threshold. If the value exceeds the threshold, an 'error' will be raised. + + +###### "monit" configuration in memory_utilization_common.json +```json + "COMMON": [ + { + "name": "monit", + "cmd": "sudo monit status", + "memory_params": { + "memory_usage": { + "memory_increase_threshold": 5, + "memory_high_threshold": 70 + } + }, + "memory_check": "parse_monit_status_output" + } + ] +``` +###### "monit" configuration in memory_utilization_dependence.json +```json + "HWSKU" : { + "Arista-7050QX": ["Arista-7050-QX-32S", "Arista-7050QX32S-Q32"] + }, + "COMMON": [ + { + "name": "monit", + "cmd": "sudo monit status", + "memory_params": { + "memory_usage": { + "memory_increase_threshold": 5, + "memory_high_threshold": 60 + } + }, + "memory_check": "parse_monit_status_output" + } + ], + "Arista-7050QX": [ + { + "name": "monit", + "cmd": "sudo monit status", + "memory_params": { + "memory_usage": { + "memory_increase_threshold": 5, + "memory_high_threshold": 80 + } + }, + "memory_check": "parse_monit_status_output" + } + ] +``` + +###### "sudo monit status" output +```shell +System 'sonic' + status Running + monitoring status Monitored + monitoring mode active + on reboot start + load average [1.44] [1.10] [1.04] + cpu 22.7%us 3.3%sy 0.0%wa + memory usage 3.2 GB [41.2%] + swap usage 0 B [0.0%] + uptime 4d 3h 55m + boot time Thu, 11 Jul 2024 06:41:46 + data collected Mon, 15 Jul 2024 10:36:45 +``` +###### "parse_monit_status_output" return value +```shell + 'monit': { + 'memory_usage': 41.2 + } +``` + +##### "free" +"free" uses the command "free -m" to get the memory item's information. +The threshold for high memory is 1500, and for an increase is 100. +The function "parse_free_output" parses the output of the command "free -m" and returns the memory information 'free':{'used':2533}. +The Memory utilization fixture uses the function "parse_free_output" to parse the output of "free -m" before and after the test case. It then compares the value with the threshold. If the value exceeds the threshold, an 'error' will be raised. + +###### "free" configuration +```json + { + "name": "free", + "cmd": "free -m", + "memory_params": { + "used": { + "memory_increase_threshold": 100, + "memory_high_threshold": 3000 + } + }, + "memory_check": "parse_free_output" + } +``` +###### "free -m" output +```shell + total used free shared buff/cache available +Mem: 3897 2533 256 183 1108 956 +Swap: 0 0 0 +``` +###### "parse_free_output" return value +```shell + 'free': { + 'used': 2533 + } +``` + +##### "docker" +"docker" uses the command "docker stats --no-stream" to get the memory item's information. +30The memory_params could have several sub memory items, the example below is "snmp", "pmon" and other dockers. +The "snmp" threshold for high memory is 30, and for an increase is 3. +The function "parse_docker_stats_output" parses the output of the command "docker stats --no-stream" and returns the memory information {'snmp': '2.07', 'pmon': '5.64', 'lldp': '1.74', 'gnmi': '3.46', 'radv': '1.02', 'syncd': '13.16', 'teamd': '1.67', 'bgp': '8.98', 'swss': '3.75', 'acms': '2.22', 'database': '3.54'} +The Memory utilization fixture uses the function "parse_docker_stats_output" to parse the output of "docker stats --no-stream" before and after the test case. It then compares the value with the threshold. If the value exceeds the threshold, an 'error' will be raised. + +###### "docker" configuration +```json + { + "name": "docker", + "cmd": "docker stats --no-stream", + "memory_params": { + "snmp": { + "memory_increase_threshold": 3, + "memory_high_threshold": 30 + }, + "pmon": { + "memory_increase_threshold": 3, + "memory_high_threshold": 30 + }, + "lldp": { + "memory_increase_threshold": 3, + "memory_high_threshold": 30 + }, + "gnmi": { + "memory_increase_threshold": 3, + "memory_high_threshold": 30 + }, + "radv": { + "memory_increase_threshold": 3, + "memory_high_threshold": 30 + }, + "syncd": { + "memory_increase_threshold": 3, + "memory_high_threshold": 30 + }, + "bgp": { + "memory_increase_threshold": 3, + "memory_high_threshold": 30 + }, + "teamd": { + "memory_increase_threshold": 3, + "memory_high_threshold": 30 + }, + "swss": { + "memory_increase_threshold": 3, + "memory_high_threshold": 30 + }, + "acms": { + "memory_increase_threshold": 3, + "memory_high_threshold": 30 + }, + "database": { + "memory_increase_threshold": 3, + "memory_high_threshold": 30 + } + }, + "memory_check": "parse_docker_stats_output" + } +``` + +###### "docker stats --no-stream" output +```shell +CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS +52958334481e snmp 5.27% 80.8MiB / 3.807GiB 2.07% 0B / 0B 4.76MB / 180kB 10 +5acbd9b57da7 pmon 2.51% 220MiB / 3.807GiB 5.64% 0B / 0B 11.6MB / 123kB 19 +dc281fd005f8 lldp 0.36% 67.84MiB / 3.807GiB 1.74% 0B / 0B 5.9MB / 98.3kB 13 +9f7c67031b80 gnmi 2.05% 134.7MiB / 3.807GiB 3.46% 0B / 0B 22.8MB / 823kB 28 +e5b96437643b radv 0.42% 39.84MiB / 3.807GiB 1.02% 0B / 0B 713kB / 65.5kB 8 +48de97a88314 syncd 16.26% 513.2MiB / 3.807GiB 13.16% 0B / 0B 273MB / 778kB 48 +26a72f50a90a teamd 1.38% 65.09MiB / 3.807GiB 1.67% 0B / 0B 1.4MB / 90.1kB 22 +2995cc0130a7 bgp 16.07% 350.2MiB / 3.807GiB 8.98% 0B / 0B 20.2MB / 537MB 27 +45c759cb9770 swss 0.66% 146.2MiB / 3.807GiB 3.75% 0B / 0B 27.2MB / 209kB 41 +91903b37d1cd acms 0.35% 86.38MiB / 3.807GiB 2.22% 0B / 0B 266kB / 1.06MB 11 +5ffa57081cb8 database 35.64% 138.1MiB / 3.807GiB 3.54% 0B / 0B 50.2MB / 73.7kB 13 +``` +###### "parse_docker_stats_output" return value +```shell + 'docker': { + 'snmp': '2.07', + 'pmon': '5.64', + 'lldp': '1.74', + 'gnmi': '3.46', + 'radv': '1.02', + 'syncd': '13.16', + 'teamd': '1.67', + 'bgp': '8.98', + 'swss': '3.75', + 'acms': '2.22', + 'database': '3.54' + } +``` + + +##### "top" +"top" uses the command "top -b -n 1" to get the memory item's information. +The memory_params could have several sub memory items, the example below is "bgpd" and "zebra". +The "bgpd" threshold for high memory is 200000, and for an increase is 10000. +The "zebra" threshold for high memory is 200000, and for an increase is 10000. +The function "parse_top_output" parses the output of the command "top -b -n 1" and returns the memory information 'top':{'zebra':67780,'bgpd':197416}. +The Memory utilization fixture uses the function "parse_top_output" to parse the output of "top -b -n 1" before and after the test case. It then compares the value with the threshold. If the value exceeds the threshold, an 'error' will be raised. + +###### "json" configuration +```json + { + "name": "top", + "cmd": "top -b -n 1", + "memory_params": { + "bgpd": { + "memory_increase_threshold": 10000, + "memory_high_threshold": 200000 + }, + "zebra": { + "memory_increase_threshold": 10000, + "memory_high_threshold": 200000 + } + }, + "memory_check": "parse_top_output" + } +``` + +###### "top -b -n 1" output +```shell +top - 03:01:19 up 4 days, 11 min, 0 users, load average: 1.56, 1.38, 1.16 +Tasks: 252 total, 2 running, 246 sleeping, 0 stopped, 4 zombie +%Cpu(s): 23.5 us, 8.6 sy, 0.0 ni, 66.7 id, 0.0 wa, 0.0 hi, 1.2 si, 0.0 st +MiB Mem : 3897.9 total, 254.8 free, 2534.7 used, 1108.4 buff/cache +MiB Swap: 0.0 total, 0.0 free, 0.0 used. 954.9 avail Mem + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND + 1063 message+ 20 0 125476 73676 8184 R 35.3 1.8 2254:29 redis-s+ + 6205 root 20 0 60392 41044 17348 S 29.4 1.0 553:32.09 python3 +2049149 root 20 0 11276 4116 3504 R 17.6 0.1 0:00.05 top + 2808 root 20 0 2701756 639364 283328 S 11.8 16.0 1041:03 syncd + 2529 root 20 0 95860 11568 10196 S 5.9 0.3 36:50.14 tlm_tea+ + 2536 root 20 0 19208 3824 2476 S 5.9 0.1 5:08.00 teamd + 5588 root 20 0 129120 32012 15944 S 5.9 0.8 19:47.47 python3 + 5730 root 20 0 1417896 78608 28512 S 5.9 2.0 54:33.37 telemet+ + 6082 root 20 0 38956 31736 10536 S 5.9 0.8 3:07.13 supervi+ + 6193 root 20 0 282292 48244 17000 S 5.9 1.2 90:43.80 python3 + 1 root 20 0 166628 13728 10192 S 0.0 0.3 10:58.80 systemd + 2 root 20 0 0 0 0 S 0.0 0.0 0:00.12 kthreadd + 3 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 rcu_gp + 4 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 rcu_par+ + 6 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kworker+ + 8 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 mm_perc+ + 9 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcu_tas+ + 10 root 20 0 0 0 0 S 0.0 0.0 0:00.00 rcu_tas+ + 11 root 20 0 0 0 0 S 0.0 0.0 0:26.79 ksoftir+ + 12 root 20 0 0 0 0 I 0.0 0.0 15:09.90 rcu_sch+ + 13 root rt 0 0 0 0 S 0.0 0.0 0:02.01 migrati+ + 15 root 20 0 0 0 0 S 0.0 0.0 0:00.00 cpuhp/0 + 16 root 20 0 0 0 0 S 0.0 0.0 0:00.00 cpuhp/1 + 17 root rt 0 0 0 0 S 0.0 0.0 0:02.30 migrati+ + 18 root 20 0 0 0 0 S 0.0 0.0 0:26.84 ksoftir+ + 20 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kworker+ + 21 root 20 0 0 0 0 S 0.0 0.0 0:00.00 cpuhp/2 + 22 root rt 0 0 0 0 S 0.0 0.0 0:02.20 migrati+ + 23 root 20 0 0 0 0 S 0.0 0.0 0:24.26 ksoftir+ + 25 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kworker+ + 26 root 20 0 0 0 0 S 0.0 0.0 0:00.00 cpuhp/3 + 27 root rt 0 0 0 0 S 0.0 0.0 0:03.03 migrati+ + 28 root 20 0 0 0 0 S 0.0 0.0 0:25.04 ksoftir+ + 30 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kworker+ + 33 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kdevtmp+ + 34 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 netns + 35 root 20 0 0 0 0 S 0.0 0.0 0:00.07 kauditd + 36 root 20 0 0 0 0 S 0.0 0.0 0:00.23 khungta+ + 37 root 20 0 0 0 0 S 0.0 0.0 0:00.00 oom_rea+ + 38 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 writeba+ + 39 root 20 0 0 0 0 S 0.0 0.0 0:17.86 kcompac+ + 40 root 25 5 0 0 0 S 0.0 0.0 0:00.00 ksmd + 41 root 39 19 0 0 0 S 0.0 0.0 0:14.38 khugepa+ + 60 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kintegr+ + 61 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kblockd + 62 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 blkcg_p+ + 63 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 edac-po+ + 64 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 devfreq+ + 66 root 0 -20 0 0 0 I 0.0 0.0 0:04.09 kworker+ + 67 root 20 0 0 0 0 S 0.0 0.0 0:00.59 kswapd0 + 68 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kthrotld + 69 root -51 0 0 0 0 S 0.0 0.0 0:00.00 irq/24-+ + 70 root -51 0 0 0 0 S 0.0 0.0 0:00.00 irq/25-+ + 71 root -51 0 0 0 0 S 0.0 0.0 0:00.00 irq/26-+ + 72 root -51 0 0 0 0 S 0.0 0.0 0:00.00 irq/27-+ + 73 root -51 0 0 0 0 S 0.0 0.0 0:00.00 irq/28-+ + 75 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 acpi_th+ + 76 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 ipv6_ad+ + 85 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kstrp + 88 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 zswap-s+ + 89 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 kworker+ + 111 root 0 -20 0 0 0 I 0.0 0.0 0:04.06 kworker+ + 141 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 ata_sff + 142 root 20 0 0 0 0 S 0.0 0.0 0:00.00 scsi_eh+ + 143 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 scsi_tm+ + 144 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 sdhci + 145 root 20 0 0 0 0 S 0.0 0.0 0:00.00 scsi_eh+ + 146 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 scsi_tm+ + 147 root -51 0 0 0 0 S 0.0 0.0 0:00.00 irq/16-+ + 151 root 0 -20 0 0 0 I 0.0 0.0 0:04.33 kworker+ + 168 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 nvme-wq + 169 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 nvme-re+ + 170 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 nvme-de+ + 184 root 20 0 0 0 0 S 0.0 0.0 0:00.00 scsi_eh+ + 185 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 scsi_tm+ + 186 root 20 0 0 0 0 S 0.0 0.0 0:19.38 usb-sto+ + 187 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 uas + 191 root 0 -20 0 0 0 I 0.0 0.0 0:04.10 kworker+ + 263 root 20 0 0 0 0 S 0.0 0.0 0:11.26 jbd2/sd+ + 264 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 ext4-rs+ + 278 root 0 -20 0 0 0 S 0.0 0.0 0:08.12 loop0 + 360 root 20 0 48024 17748 14244 S 0.0 0.4 0:13.98 systemd+ + 361 root 20 0 22932 7344 5764 S 0.0 0.2 0:01.14 systemd+ + 412 root 16 -4 91408 3132 2200 S 0.0 0.1 0:00.53 auditd + 417 root 16 -4 7244 5104 4432 S 0.0 0.1 0:00.08 audisp-+ + 479 root 20 0 8180 6796 1752 S 0.0 0.2 0:13.09 haveged + 494 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 cryptd + 495 root 20 0 7372 2968 2708 S 0.0 0.1 0:01.75 cron + 496 message+ 20 0 7988 4416 3848 S 0.0 0.1 0:08.29 dbus-da+ + 523 root 20 0 11152 5320 4376 S 0.0 0.1 0:00.13 smartd + 532 root 20 0 14552 9028 8048 S 0.0 0.2 0:04.42 systemd+ + 534 root 20 0 1579288 65448 33076 S 0.0 1.6 16:40.26 contain+ + 629 root 20 0 84096 3416 2388 S 0.0 0.1 3:46.33 monit + 650 root 20 0 1836628 100396 44680 S 0.0 2.5 41:36.89 dockerd + 654 root 20 0 6472 1732 1624 S 0.0 0.0 0:00.00 agetty + 655 root 20 0 6104 1976 1864 S 0.0 0.0 0:00.00 agetty + 658 root 20 0 14368 8700 7500 S 0.0 0.2 0:00.34 sshd + 780 root 20 0 720760 17092 6500 S 0.0 0.4 2:13.22 contain+ + 801 root 20 0 32148 28064 10316 S 0.0 0.7 2:47.49 supervi+ + 859 root 20 0 37176 27352 10344 S 0.0 0.7 0:43.02 arista + 1062 root 20 0 125080 27668 15768 S 0.0 0.7 18:21.61 python3 + 1152 root 20 0 12040 6920 6128 S 0.0 0.2 0:00.01 databas+ + 1157 root 20 0 12304 7200 6240 S 0.0 0.2 0:00.02 databas+ + 1160 root 20 0 1254740 33820 16584 S 0.0 0.8 0:17.96 docker + 1202 root 20 0 222220 6212 3664 S 0.0 0.2 0:00.06 rsyslogd + 1240 root 20 0 236476 61884 20724 S 0.0 1.6 12:58.16 healthd + 1311 root 20 0 720504 17896 6312 S 0.0 0.4 2:13.54 contain+ + 1333 root 20 0 36256 32032 10276 S 0.0 0.8 3:13.31 supervi+ + 1339 root 20 0 386980 48952 8248 S 0.0 1.2 0:59.09 healthd + 1344 root 20 0 175300 52568 10300 S 0.0 1.3 0:34.17 healthd + 1347 root 20 0 180060 52160 11056 S 0.0 1.3 0:01.20 healthd + 1350 root 20 0 170468 47420 7020 S 0.0 1.2 0:53.54 healthd + 1411 admin 20 0 12172 6484 5720 S 0.0 0.2 0:00.02 acms.sh + 1412 root 20 0 121940 27460 15048 S 0.0 0.7 0:40.33 caclmgrd + 1416 root 20 0 46296 25168 14756 S 0.0 0.6 8:52.39 procdoc+ + 1422 admin 20 0 69388 42444 18400 S 0.0 1.1 0:00.95 python3 + 1730 root 20 0 129144 31712 15616 S 0.0 0.8 19:57.12 python3 + 1732 root 20 0 44784 29552 15108 S 0.0 0.7 0:00.47 start.py + 1734 root 20 0 45820 29696 14840 S 0.0 0.7 0:00.49 CA_cert+ + 1735 root 20 0 44572 27912 13496 S 0.0 0.7 0:01.74 cert_co+ + 1869 root 20 0 222220 4188 3684 S 0.0 0.1 0:00.81 rsyslogd + 1960 root 20 0 720760 16332 6500 S 0.0 0.4 2:11.96 contain+ + 1981 root 20 0 36224 32124 10320 S 0.0 0.8 6:54.24 supervi+ + 2051 root 20 0 292104 5940 3028 S 0.0 0.1 0:07.35 rsyslogd + 2088 root 20 0 12304 7180 6200 S 0.0 0.2 0:00.03 swss.sh + 2272 root 20 0 720504 17260 6500 S 0.0 0.4 2:15.91 contain+ + 2299 root 20 0 720504 17048 6500 S 0.0 0.4 2:13.86 contain+ + 2307 root 20 0 36292 32128 10340 S 0.0 0.8 2:59.93 supervi+ + 2332 root 20 0 720760 18280 6500 S 0.0 0.5 2:15.46 contain+ + 2346 root 20 0 36292 32076 10268 S 0.0 0.8 3:23.37 supervi+ + 2361 root 20 0 35612 31380 10236 S 0.0 0.8 4:19.78 supervi+ + 2398 root 20 0 12172 7128 6212 S 0.0 0.2 0:00.02 syncd.sh + 2403 root 20 0 12172 6656 5888 S 0.0 0.2 0:00.02 syncd.sh + 2408 root 20 0 69388 42304 18268 S 0.0 1.1 0:00.94 python3 + 2411 admin 20 0 12172 6288 5520 S 0.0 0.2 0:00.02 teamd.sh + 2421 admin 20 0 12172 6452 5684 S 0.0 0.2 0:00.02 teamd.sh + 2423 admin 20 0 69388 42348 18312 S 0.0 1.1 0:00.93 python3 + 2430 admin 20 0 12040 6572 5804 S 0.0 0.2 0:00.02 bgp.sh + 2436 admin 20 0 12172 6440 5672 S 0.0 0.2 0:00.02 bgp.sh + 2439 admin 20 0 69388 42252 18216 S 0.0 1.1 0:00.93 python3 + 2443 root 20 0 125104 27596 15668 S 0.0 0.7 18:41.56 python3 + 2488 root 20 0 222220 6160 3660 S 0.0 0.2 0:04.46 rsyslogd + 2493 root 20 0 129144 31796 15700 S 0.0 0.8 18:19.70 python3 + 2500 root 20 0 129144 32524 15628 S 0.0 0.8 20:00.72 python3 + 2503 root 20 0 92180 11028 9824 S 0.0 0.3 0:30.90 portsyn+ + 2520 root 20 0 222220 4216 3716 S 0.0 0.1 0:00.84 rsyslogd + 2524 root 20 0 222220 6348 3716 S 0.0 0.2 3:18.10 rsyslogd + 2528 root 20 0 92608 11256 9776 S 0.0 0.3 0:35.48 teammgrd + 2571 root 20 0 19212 3804 2444 S 0.0 0.1 4:55.60 teamd + 2580 root 20 0 19212 3828 2480 S 0.0 0.1 5:02.54 teamd + 2600 root 20 0 19216 3908 2560 S 0.0 0.1 4:56.09 teamd + 2602 root 20 0 620592 87972 22340 S 0.0 2.2 16:54.01 orchage+ + 2611 root 20 0 19212 3800 2444 S 0.0 0.1 5:04.60 teamd + 2621 root 20 0 19212 3884 2532 S 0.0 0.1 4:56.33 teamd + 2632 root 20 0 87704 1540 1372 S 0.0 0.0 0:00.02 dsserve + 2633 root 20 0 92992 11580 9716 S 0.0 0.3 0:35.32 teamsyn+ + 2643 root 20 0 19212 3832 2484 S 0.0 0.1 5:10.30 teamd + 2655 root 20 0 19212 3912 2564 S 0.0 0.1 5:27.09 teamd + 2664 root 20 0 720760 18556 6436 S 0.0 0.5 2:14.23 contain+ + 2691 root 20 0 36224 32148 10320 S 0.0 0.8 2:37.63 supervi+ + 2709 admin 20 0 12040 7024 6232 S 0.0 0.2 0:00.02 radv.sh + 2718 admin 20 0 12172 6640 5872 S 0.0 0.2 0:00.02 radv.sh + 2725 admin 20 0 69388 42464 18424 S 0.0 1.1 0:00.94 python3 + 2742 root 20 0 293620 42828 18256 S 0.0 1.1 0:00.93 python3 + 2755 root 20 0 92336 10048 8928 S 0.0 0.3 0:31.47 coppmgrd + 2777 root 20 0 125104 27796 15796 S 0.0 0.7 20:01.54 python3 + 2813 root 20 0 222220 4224 3720 S 0.0 0.1 0:00.23 rsyslogd + 2826 300 20 0 665588 67780 6916 S 0.0 1.7 0:23.12 zebra + 2864 300 20 0 44084 13616 5320 S 0.0 0.3 0:07.68 staticd + 2882 root 20 0 125080 27604 15700 S 0.0 0.7 20:04.30 python3 + 2895 ntp 20 0 74488 3648 2912 S 0.0 0.1 0:43.96 ntpd + 2907 root 20 0 92172 9892 8768 S 0.0 0.2 0:16.61 neighsy+ + 2908 root 20 0 92336 11124 9904 S 0.0 0.3 0:31.36 vlanmgrd + 2909 root 20 0 92496 11268 9936 S 0.0 0.3 0:32.85 intfmgrd + 2910 root 20 0 92308 11128 9864 S 0.0 0.3 0:31.84 portmgrd + 2911 root 20 0 92544 11068 9768 S 0.0 0.3 0:34.04 bufferm+ + 2914 root 20 0 92328 11044 9772 S 0.0 0.3 0:31.67 vrfmgrd + 2916 root 20 0 92216 9988 8868 S 0.0 0.3 0:31.45 nbrmgrd + 2917 root 20 0 92360 11112 9876 S 0.0 0.3 0:31.86 vxlanmg+ + 2920 root 20 0 92108 10136 9016 S 0.0 0.3 0:15.75 fdbsyncd + 2924 root 20 0 92308 11024 9812 S 0.0 0.3 0:32.53 tunnelm+ + 3034 root 20 0 222220 4208 3708 S 0.0 0.1 0:00.06 rsyslogd + 3327 root 20 0 50524 30004 15580 S 0.0 0.8 0:47.96 featured + 3328 root 20 0 61100 40044 16576 S 0.0 1.0 0:31.69 hostcfgd + 3331 root 20 0 5472 3020 2772 S 0.0 0.1 0:00.02 rasdaem+ + 4346 300 20 0 498388 197416 8156 S 0.0 4.9 3:00.32 bgpd + 4351 root 20 0 123448 33264 16476 S 0.0 0.8 0:40.03 bgpcfgd + 4359 root 20 0 37472 22380 15156 S 0.0 0.6 2:10.32 bgpmon + 4361 root 20 0 93460 11936 9548 S 0.0 0.3 0:18.78 fpmsyncd + 4377 root 20 0 37664 22160 14580 S 0.0 0.6 0:35.31 staticr+ + 5413 root 20 0 32180 19580 10260 S 0.0 0.5 0:00.29 python3 + 5465 root 20 0 720760 17684 6372 S 0.0 0.4 3:19.35 contain+ + 5486 root 20 0 36284 32056 10232 S 0.0 0.8 3:03.55 supervi+ + 5499 admin 20 0 12040 6984 6192 S 0.0 0.2 0:00.02 gnmi.sh + 5501 admin 20 0 12172 6396 5632 S 0.0 0.2 0:00.02 gnmi.sh + 5503 admin 20 0 69388 42216 18168 S 0.0 1.1 0:00.87 python3 + 5591 root 20 0 222220 6212 3688 S 0.0 0.2 0:00.07 rsyslogd + 5614 root 20 0 720760 16616 6372 S 0.0 0.4 2:08.40 contain+ + 5640 root 20 0 36224 32148 10328 S 0.0 0.8 3:28.26 supervi+ + 5668 admin 20 0 12040 6440 5676 S 0.0 0.2 0:00.01 lldp.sh + 5671 admin 20 0 12172 6540 5772 S 0.0 0.2 0:00.02 lldp.sh + 5673 admin 20 0 69388 42228 18176 S 0.0 1.1 0:00.89 python3 + 5738 root 20 0 1418152 73376 28400 S 0.0 1.8 54:30.22 telemet+ + 5877 root 20 0 125104 27600 15608 S 0.0 0.7 19:59.83 python3 + 5884 root 20 0 721016 16468 6308 S 0.0 0.4 2:19.75 contain+ + 5915 root 20 0 36232 32132 10336 S 0.0 0.8 4:06.02 supervi+ + 5938 admin 20 0 12172 6568 5808 S 0.0 0.2 0:00.02 pmon.sh + 5940 admin 20 0 69388 42344 18304 S 0.0 1.1 0:00.88 python3 + 5963 root 20 0 222220 4128 3624 S 0.0 0.1 0:00.08 rsyslogd + 6035 tcpdump 20 0 20252 8844 7644 S 0.0 0.2 0:00.31 lldpd + 6037 tcpdump 20 0 20384 3720 2388 S 0.0 0.1 4:04.87 lldpd + 6061 root 20 0 720760 17832 6500 S 0.0 0.4 3:15.76 contain+ + 6089 root 20 0 115064 28844 14628 S 0.0 0.7 32:49.56 python3 + 6098 root 20 0 12120 7108 6324 S 0.0 0.2 0:00.02 snmp.sh + 6100 root 20 0 12252 7096 6304 S 0.0 0.2 0:00.02 snmp.sh + 6103 root 20 0 69468 42572 18512 S 0.0 1.1 0:00.89 python3 + 6128 root 20 0 42324 25320 15492 S 0.0 0.6 0:04.51 python3 + 6164 root 20 0 125104 27568 15636 S 0.0 0.7 18:36.36 python3 + 6166 root 20 0 132408 33104 15988 S 0.0 0.8 19:49.98 python3 + 6176 root 20 0 222224 4016 3516 S 0.0 0.1 0:01.39 rsyslogd + 6180 root 20 0 222220 8280 3712 S 0.0 0.2 0:00.18 rsyslogd + 6188 tcpdump 20 0 26616 15068 9736 S 0.0 0.4 6:39.60 snmpd + 6191 root 20 0 52408 35208 16700 S 0.0 0.9 0:36.22 python3 + 6194 root 20 0 59796 42796 16824 S 0.0 1.1 26:20.79 python3 + 6195 root 20 0 60552 43180 16556 S 0.0 1.1 0:02.77 python3 + 6197 root 20 0 61708 44256 16576 S 0.0 1.1 5:20.56 python3 + 6198 root 20 0 72120 55252 16952 S 0.0 1.4 1:32.29 pcied + 6204 root 20 0 55332 1140 0 S 0.0 0.0 0:22.51 sensord + 6206 root 20 0 61708 33988 6308 S 0.0 0.9 3:52.65 python3 + 110653 root 20 0 4160 3392 2896 S 0.0 0.1 0:00.06 bash +2002226 root 20 0 0 0 0 I 0.0 0.0 0:01.10 kworker+ +2016391 root 20 0 0 0 0 I 0.0 0.0 0:00.40 kworker+ +2026996 root 20 0 0 0 0 I 0.0 0.0 0:00.44 kworker+ +2030519 root 20 0 0 0 0 I 0.0 0.0 0:00.00 kworker+ +2034114 root 20 0 0 0 0 I 0.0 0.0 0:00.24 kworker+ +2036072 root 20 0 0 0 0 I 0.0 0.0 0:00.00 kworker+ +2036243 root 20 0 0 0 0 I 0.0 0.0 0:00.06 kworker+ +2041185 root 20 0 0 0 0 I 0.0 0.0 0:00.17 kworker+ +2044722 root 20 0 0 0 0 I 0.0 0.0 0:00.10 kworker+ +2044751 root 20 0 0 0 0 I 0.0 0.0 0:00.00 kworker+ +2047210 root 20 0 0 0 0 I 0.0 0.0 0:00.00 kworker+ +2048525 root 20 0 0 0 0 I 0.0 0.0 0:00.00 kworker+ +2048528 root 20 0 14852 9136 7744 S 0.0 0.2 0:00.22 sshd +2048534 admin 20 0 15308 6584 4672 S 0.0 0.2 0:00.18 sshd +2048690 root 20 0 0 0 0 Z 0.0 0.0 0:01.06 python3 +2048691 root 20 0 0 0 0 Z 0.0 0.0 0:00.19 memory_+ +2048692 root 20 0 0 0 0 Z 0.0 0.0 0:00.94 python3 +2048693 root 20 0 0 0 0 Z 0.0 0.0 0:00.95 python3 +2049145 admin 20 0 2480 512 448 S 0.0 0.0 0:00.02 sh +2049146 root 20 0 15008 8240 7260 S 0.0 0.2 0:00.03 sudo +2049147 root 20 0 2480 516 448 S 0.0 0.0 0:00.00 sh +2049148 root 20 0 33600 23920 11384 S 0.0 0.6 0:00.43 python +``` + +###### "parse_top_output" return value +```shell + 'top': { + 'zebra': 67780, + 'bgpd': 197416 + } +``` + +#### Example Use Cases for memory items + +##### Define a memory item globally +- We can define a global memory item in "memory_utilization_common.json" which is usually used in public branch. This memory item will apply to all test cases. +- If we prefer not to define it in the public branch, we can alternatively define it in "memory_utilization_dependence.json" which is usually used in the internal branch. This memory item will also apply to all test cases. +- If a memory item is defined with the same name in both "memory_utilization_common.json" and "memory_utilization_dependence.json" under their "COMMON" sections, the definition in the "memory_utilization_dependence.json" will take priority. + + + + ```python + # define in "memory_utilization_common.json" + "COMMON": [ + { + "name": "monit", + "cmd": "sudo monit status", + "memory_params": { + "memory_usage": { + "memory_increase_threshold": 5, + "memory_high_threshold": 70 + } + }, + "memory_check": "parse_monit_status_output" + } + ] + ``` + + ```python + # define in memory_utilization_dependence.json + # "memory_high_threshold" overwrited to 60 + "COMMON": [ + { + "name": "monit", + "cmd": "sudo monit status", + "memory_params": { + "memory_usage": { + "memory_increase_threshold": 5, + "memory_high_threshold": 60 + } + }, + "memory_check": "parse_monit_status_output" + } + ] + ``` + + +##### Define a memory item per HWSKU +We can define a memory item per HwSku in "memory_utilization_dependence.json". +- First, define a dict named "HWSKU" to manage all HwSku collections. +- Second, specify each HwSku collections. use the collection name as the key and a list of HwSku names included in that HwSku collection as the value. +- Finally, define the memory items for each HwSku collection, the configuration will take priority. + + ```python + # memory_utilization_dependence.json + "HWSKU" : { + "Arista-7050QX": ["Arista-7050-QX-32S", "Arista-7050QX32S-Q32"] + }, + "COMMON": [ + { + "name": "monit", + "cmd": "sudo monit status", + "memory_params": { + "memory_usage": { + "memory_increase_threshold": 5, + "memory_high_threshold": 60 + } + }, + "memory_check": "parse_monit_status_output" + } + ], + # the "memory_high_threshold" value would be overwrite to "80" for HwSku "Arista-7050-QX-32S", "Arista-7050QX32S-Q32" + "Arista-7050QX": [ + { + "name": "monit", + "cmd": "sudo monit status", + "memory_params": { + "memory_usage": { + "memory_increase_threshold": 5, + "memory_high_threshold": 80 + } + }, + "memory_check": "parse_monit_status_output" + } + ] + ``` + + +##### Define a memory item per test case +We can modify the threshold of existing memory items within the test case, but we cannot change the cmd and function of the memory items. +However, we can add new memory items within the test case by using memory_utilization fixture and then registering them. + +This functionality has not been verified yet, the following examples are provided for reference only. +Updates will be made once the verification process is complete. + + ```python + # memory item config per test case + per_test_case_config = [ + # exist memory item + { + "name": "monit", + "cmd": "sudo monit status", + "memory_params": { + "memory_usage": { + "memory_increase_threshold": 10, + "memory_high_threshold": 90 + } + }, + "memory_check": "parse_monit_status_output" + }, + # new memory item per test case + { + "name": "memory_item_per_test_case", + "cmd": "cmd per test case", + "memory_params": { + "used": { + "memory_increase_threshold": 100, + "memory_high_threshold": 1500 + } + }, + "memory_check": "parse_output_per_test_case" + } + ] + + # use the fixture memory_utilization + def test_case_example(duthosts, enum_frontend_dut_hostname, memory_utilization): + ... + for memory_item in per_test_case_config: + is_exist = False + # for exist memory item + for i, exist_commands in enumerate(memory_monitor.commands): + exist_name, exist_cmd, exist_memory_params, exist_memory_check = exist_commands + if memory_item["name"] == exist_name: + memory_monitor.commands[i] = ( + exist_name, + exist_cmd, + memory_item["memory_params"], + exist_memory_check + ) + is_exist = True + break + # if memory item not exist, register a new memory item. + if not is_exist: + memory_monitor.register_command(memory_item["name"], memory_item["cmd"], memory_item["memory_params"], memory_item["memory_check"]) + output = memory_monitor.execute_command(memory_item["cmd"]) + + initial_memory_value[memory_item["name"]] = memory_monitor.run_command_parser_function(memory_item["name"], output) + ``` diff --git a/tests/common/plugins/memory_utilization/__init__.py b/tests/common/plugins/memory_utilization/__init__.py new file mode 100644 index 0000000000..91c5f9661f --- /dev/null +++ b/tests/common/plugins/memory_utilization/__init__.py @@ -0,0 +1,96 @@ +import logging +import pytest +from tests.common.plugins.memory_utilization.memory_utilization import MemoryMonitor + + +def pytest_addoption(parser): + parser.addoption( + "--disable_memory_utilization", + action="store_true", + default=False, + help="Disable memory utilization analysis for the 'memory_utilization' fixture" + ) + + +@pytest.fixture(scope="function", autouse=True) +def store_fixture_values(request, duthosts, memory_utilization): + logging.info("store memory_utilization {}".format(request.node.name)) + request.config.store_duthosts = duthosts + request.config.store_memory_utilization = memory_utilization + + +@pytest.hookimpl(trylast=True) +def pytest_runtest_setup(item): + logging.info("collect memory before test {}".format(item.name)) + + duthosts = getattr(item.config, 'store_duthosts', None) + memory_utilization = getattr(item.config, 'store_memory_utilization', None) + if duthosts is None and memory_utilization is None: + return + + memory_monitors, memory_values = memory_utilization + + logging.debug("memory_values {} ".format(memory_values)) + + for duthost in duthosts: + if duthost.topo_type == 't2': + continue + + # Initial memory check for all registered commands + for name, cmd, memory_params, memory_check in memory_monitors[duthost.hostname].commands: + output = memory_monitors[duthost.hostname].execute_command(cmd) + memory_values["before_test"][duthost.hostname][name] = memory_check(output, memory_params) + + logging.info("Before test: collected memory_values {}".format(memory_values)) + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_teardown(item, nextitem): + logging.info("collect memory after test {}".format(item.name)) + + duthosts = getattr(item.config, 'store_duthosts', None) + memory_utilization = getattr(item.config, 'store_memory_utilization', None) + if duthosts is None and memory_utilization is None: + return + + memory_monitors, memory_values = memory_utilization + + logging.debug("memory_values {} ".format(memory_values)) + + for duthost in duthosts: + if duthost.topo_type == 't2': + continue + + # memory check for all registered commands + for name, cmd, memory_params, memory_check in memory_monitors[duthost.hostname].commands: + output = memory_monitors[duthost.hostname].execute_command(cmd) + memory_values["after_test"][duthost.hostname][name] = memory_check(output, memory_params) + + memory_monitors[duthost.hostname].check_memory_thresholds( + memory_values["after_test"][duthost.hostname], memory_values["before_test"][duthost.hostname]) + + logging.info("After test: collected memory_values {}".format(memory_values)) + + +@pytest.fixture(autouse=True) +def memory_utilization(duthosts, request): + if request.config.getoption("--disable_memory_utilization") or "disable_memory_utilization" in request.keywords: + logging.info("Memory utilization is disabled") + yield None, None + return + + memory_monitors = {} + memory_values = {"before_test": {}, "after_test": {}} + + for duthost in duthosts: + if duthost.topo_type == 't2': + continue + memory_monitor = MemoryMonitor(ansible_host=duthost) + memory_values["before_test"][duthost.hostname] = {} + memory_values["after_test"][duthost.hostname] = {} + logging.info("Hostname: {}, Hwsku: {}, Platform: {}".format( + duthost.hostname, duthost.sonichost._facts["hwsku"], duthost.sonichost._facts["platform"])) + memory_monitor.parse_and_register_commands(hwsku=duthost.sonichost._facts["hwsku"]) + memory_monitors[duthost.hostname] = memory_monitor + + yield memory_monitors, memory_values diff --git a/tests/common/plugins/memory_utilization/memory_utilization.py b/tests/common/plugins/memory_utilization/memory_utilization.py new file mode 100755 index 0000000000..c431d70a6d --- /dev/null +++ b/tests/common/plugins/memory_utilization/memory_utilization.py @@ -0,0 +1,253 @@ +import logging +import re +import json +from os.path import join, split +import pytest + +logger = logging.getLogger(__name__) + +MEMORY_UTILIZATION_COMMON_JSON_FILE = join(split(__file__)[0], "memory_utilization_common.json") +MEMORY_UTILIZATION_DEPENDENCE_JSON_FILE = join(split(__file__)[0], "memory_utilization_dependence.json") + + +class MemoryMonitor: + def __init__(self, ansible_host): + self.ansible_host = ansible_host + self.commands = [] + self.memory_values = {} + + def register_command(self, name, cmd, memory_params, memory_check_fn): + """Register a command with its associated memory parameters and check function.""" + self.commands.append((name, cmd, memory_params, memory_check_fn)) + self.memory_values[name] = {} + + def execute_command(self, cmd): + """Execute a shell command and return its output.""" + response = self.ansible_host.command(cmd, module_ignore_errors=True) + stdout = response.get('stdout', None) + # logger.debug("Command '{}' response: {}".format(cmd, stdout)) + return stdout + + def check_memory_thresholds(self, current_values, previous_values): + """Check memory usage against thresholds. """ + logger.debug("Previous values: {}".format(previous_values)) + logger.debug("Current values: {}".format(current_values)) + + for name, cmd, memory_params, memory_check_fn in self.commands: + for mem_item, thresholds in memory_params.items(): + current_value = float(current_values.get(name, {}).get(mem_item, 0)) + previous_value = float(previous_values.get(name, {}).get(mem_item, 0)) + + if current_value == 0 or previous_value == 0: + logger.warning("Skipping memory check for {}-{} due to zero value".format(name, mem_item)) + continue + + high_threshold = float(thresholds.get("memory_high_threshold", float('inf'))) + increase_threshold = float(thresholds.get("memory_increase_threshold", float('inf'))) + + if previous_value > high_threshold: + self.handle_memory_threshold_exceeded( + name, mem_item, previous_value, high_threshold, + previous_values, current_values, is_current=False + ) + + if current_value > high_threshold: + self.handle_memory_threshold_exceeded( + name, mem_item, current_value, high_threshold, + previous_values, current_values, is_current=True + ) + + increase = current_value - previous_value + if increase > increase_threshold: + self.handle_memory_threshold_exceeded( + name, mem_item, increase, increase_threshold, + previous_values, current_values, is_increase=True + ) + + def handle_memory_threshold_exceeded(self, name, mem_item, value, threshold, + previous_values, current_values, is_current=False, is_increase=False): + + """Handle memory threshold or increase exceeded.""" + logger.info("{}:{}, previous_values: {}".format(name, mem_item, previous_values)) + logger.info("{}:{}, current_values: {}".format(name, mem_item, current_values)) + + if is_increase: + message = ( + "[ALARM]: {}:{} memory usage increased by {}, " + "exceeds increase threshold {}".format( + name, mem_item, value, threshold + ) + ) + else: + message = ( + "[ALARM]: {}:{}, {} memory usage {} exceeds " + "high threshold {}".format( + name, mem_item, "Current" if is_current else "Previous", value, threshold + ) + ) + + logger.warning(message) + pytest.fail(message) + + def parse_and_register_commands(self, hwsku=None): + """Initialize the MemoryMonitor by reading commands from JSON files and registering them.""" + + parameter_dict = {} + with open(MEMORY_UTILIZATION_COMMON_JSON_FILE, 'r') as file: + data = json.load(file) + memory_items = data.get("COMMON", []) + for item in memory_items: + name = item["name"] + command = item["cmd"] + memory_params = item["memory_params"] + memory_check_fn = item["memory_check"] + parameter_dict[name] = { + 'name': name, + 'cmd': command, + 'memory_params': memory_params, + 'memory_check_fn': memory_check_fn + } + + with open(MEMORY_UTILIZATION_DEPENDENCE_JSON_FILE, 'r') as file: + data = json.load(file) + memory_items = data.get("COMMON", []) + for item in memory_items: + name = item["name"] + command = item["cmd"] + memory_params = item["memory_params"] + memory_check_fn = item["memory_check"] + parameter_dict[name] = { + 'name': name, + 'cmd': command, + 'memory_params': memory_params, + 'memory_check_fn': memory_check_fn + } + + if hwsku: + hwsku_found = any(hwsku in sku_list for sku_list in data.get("HWSKU", {}).values()) + if hwsku_found: + for key, value in data["HWSKU"].items(): + if hwsku in value: + for item in data[key]: + logger.info("#### CMD {} ".format(item)) + name = item["name"] + command = item["cmd"] + memory_params = item["memory_params"] + memory_check_fn = item["memory_check"] + parameter_dict[name] = { + 'name': name, + 'cmd': command, + 'memory_params': memory_params, + 'memory_check_fn': memory_check_fn + } + + for param in parameter_dict.values(): + logger.debug( + "Registering command: name={}, cmd={}, memory_params={}, " + "memory_check={}".format( + param['name'], param['cmd'], param['memory_params'], param['memory_check_fn'] + ) + ) + self.register_command(param['name'], param['cmd'], param['memory_params'], eval(param['memory_check_fn'])) + + +def parse_top_output(output, memory_params): + """Parse the 'top' command output to extract memory usage information.""" + memory_values = {} + headers = [] + length = 0 + for line in output.split('\n'): + if "PID" in line and "USER" in line and "RES" in line and "COMMAND" in line: + headers = line.split() + length = len(headers) + continue + + parts = line.split() + if length != 0 and len(parts) == length: + process_info = {headers[i]: parts[i] for i in range(length)} + + for mem_item, thresholds in memory_params.items(): + if mem_item in process_info["COMMAND"]: + if mem_item in memory_values: + memory_values[mem_item] += int(process_info["RES"]) + else: + memory_values[mem_item] = int(process_info["RES"]) + + logger.debug("Parsed memory values: {}".format(memory_values)) + return memory_values + + +def parse_free_output(output, memory_params): + """Parse the 'free' command output to extract memory usage information.""" + memory_values = {} + headers, Mem, Swap = [], [], [] + for line in output.split('\n'): + if "total" in line: + headers = line.split() + if "Mem:" in line: + Mem = line.split()[1:] + if "Swap:" in line: + Swap = line.split()[1:] + + mem_info = {headers[i]: int(Mem[i]) for i in range(len(Mem))} + swap_info = {headers[i]: int(Swap[i]) for i in range(len(Swap))} + + for mem_item, _ in memory_params.items(): + memory_values[mem_item] = mem_info.get(mem_item, 0) + swap_info.get(mem_item, 0) + + logger.debug("Parsed memory values: {}".format(memory_values)) + return memory_values + + +def parse_monit_status_output(output, memory_params): + """Parse the 'monit status' command output to extract memory usage information.""" + memory_values = {} + memory_pattern = r"memory usage\s+([\d\.]+ \w+)\s+\[(\d+\.\d+)%\]" + swap_pattern = r"swap usage\s+([\d\.]+ \w+)\s+\[(\d+\.\d+)%\]" + + for line in output.split('\n'): + if "memory usage" in line: + match = re.search(memory_pattern, line) + if match: + used_memory = match.group(1) # noqa F841 + memory_percentage = match.group(2) + memory_values['memory_usage'] = float(memory_percentage) + else: + logger.error("Failed to parse memory usage from line: {}".format(line)) + if "swap usage" in line: + match = re.search(swap_pattern, line) + if match: + used_swap = match.group(1) # noqa F841 + swap_percentage = match.group(2) # noqa F841 + else: + logger.debug("Failed to parse swap usage from line: {}".format(line)) + + logger.debug("Parsed memory values: {}".format(memory_values)) + return memory_values + + +def parse_docker_stats_output(output, memory_params): + memory_values = {} + length = 0 + pattern = r"(\d+\.\d+)%.*?(\d+\.\d+)%" + + for line in output.split('\n'): + if "NAME" in line and "CPU" in line and "MEM" in line: + headers = line.split() + length = len(headers) + continue + + if length != 0: + for mem_item, thresholds in memory_params.items(): + if mem_item in line: + match = re.search(pattern, line) + if match: + mem_usage = match.group(2) + memory_values[mem_item] = mem_usage + else: + logger.error("Failed to parse memory usage from line: {}".format(line)) + else: + continue + + logger.debug("Parsed memory values: {}".format(memory_values)) + return memory_values diff --git a/tests/common/plugins/memory_utilization/memory_utilization_common.json b/tests/common/plugins/memory_utilization/memory_utilization_common.json new file mode 100755 index 0000000000..b4137abcb0 --- /dev/null +++ b/tests/common/plugins/memory_utilization/memory_utilization_common.json @@ -0,0 +1,15 @@ +{ + "COMMON": [ + { + "name": "monit", + "cmd": "sudo monit status", + "memory_params": { + "memory_usage": { + "memory_increase_threshold": 50, + "memory_high_threshold": 90 + } + }, + "memory_check": "parse_monit_status_output" + } + ] +} diff --git a/tests/common/plugins/memory_utilization/memory_utilization_dependence.json b/tests/common/plugins/memory_utilization/memory_utilization_dependence.json new file mode 100755 index 0000000000..0967ef424b --- /dev/null +++ b/tests/common/plugins/memory_utilization/memory_utilization_dependence.json @@ -0,0 +1 @@ +{} diff --git a/tests/common/plugins/sanity_check/__init__.py b/tests/common/plugins/sanity_check/__init__.py index c94809b667..2d77106769 100644 --- a/tests/common/plugins/sanity_check/__init__.py +++ b/tests/common/plugins/sanity_check/__init__.py @@ -10,13 +10,22 @@ from tests.common.plugins.sanity_check import constants from tests.common.plugins.sanity_check import checks from tests.common.plugins.sanity_check.checks import * # noqa: F401, F403 -from tests.common.plugins.sanity_check.recover import recover +from tests.common.plugins.sanity_check.recover import recover, recover_chassis from tests.common.plugins.sanity_check.constants import STAGE_PRE_TEST, STAGE_POST_TEST from tests.common.helpers.assertions import pytest_assert as pt_assert logger = logging.getLogger(__name__) SUPPORTED_CHECKS = checks.CHECK_ITEMS +DUT_CHEK_LIST = ['core_dump_check_pass', 'config_db_check_pass'] +CACHE_LIST = ['core_dump_check_pass', 'config_db_check_pass', + 'pre_sanity_check_failed', 'post_sanity_check_failed', + 'pre_sanity_recovered', 'post_sanity_recovered'] + + +def reset_cache_list(request): + for item in CACHE_LIST: + request.config.cache.set(item, None) def pytest_sessionfinish(session, exitstatus): @@ -28,6 +37,8 @@ def pytest_sessionfinish(session, exitstatus): session.config.cache.set("pre_sanity_check_failed", None) if post_sanity_failed: session.config.cache.set("post_sanity_check_failed", None) + for key in CACHE_LIST: + session.config.cache.set(key, None) if pre_sanity_failed and not post_sanity_failed: session.exitstatus = constants.PRE_SANITY_CHECK_FAILED_RC @@ -120,12 +131,50 @@ def do_checks(request, check_items, *args, **kwargs): @pytest.fixture(scope="module", autouse=True) -def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): - if request.config.option.skip_sanity: - logger.info("Skip sanity check according to command line argument") - yield - return - +def log_custom_msg(request): + yield + module_name = request.node.name + items = request.session.items + for item in items: + if item.module.__name__ + ".py" == module_name.split("/")[-1]: + customMsgDict = {} + dutChekResults = {} + for key in DUT_CHEK_LIST: + if request.config.cache.get(key, None) is False: + dutChekResults[key] = False + if dutChekResults: + customMsgDict['DutChekResult'] = dutChekResults + + # Check pre_sanity_checks results + preSanityCheckResults = {} + if request.config.cache.get("pre_sanity_check_failed", None): + preSanityCheckResults['pre_sanity_check_failed'] = True + # pre_sanity_recovered should be None in healthy case, record either True/False + if request.config.cache.get("pre_sanity_recovered", None) is not None: + preSanityCheckResults['pre_sanity_recovered'] = request.config.cache.get("pre_sanity_recovered", None) + if preSanityCheckResults: + customMsgDict['PreSanityCheckResults'] = preSanityCheckResults + + # Check post_sanity_checks results + postSanityCheckResults = {} + if request.config.cache.get("post_sanity_check_failed", None): + postSanityCheckResults['post_sanity_check_failed'] = True + # post_sanity_recovered should be None in healthy case, record either True/False + if request.config.cache.get("post_sanity_recovered", None) is not None: + preSanityCheckResults['post_sanity_recovered'] = request.config.cache.get("post_sanity_recovered", None) + if postSanityCheckResults: + customMsgDict['PostSanityCheckResults'] = postSanityCheckResults + + # if we have any custom message to log, append it to user_properties + if customMsgDict: + logger.debug("customMsgDict: {}".format(customMsgDict)) + item.user_properties.append(('CustomMsg', json.dumps(customMsgDict))) + + reset_cache_list(request) + + +@pytest.fixture(scope="module") +def sanity_check_full(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): logger.info("Prepare sanity check") skip_sanity = False @@ -237,46 +286,8 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): pt_assert(False, "!!!!!!!!!!!!!!!!Pre-test sanity check failed: !!!!!!!!!!!!!!!!\n{}" .format(json.dumps(failed_results, indent=4, default=fallback_serializer))) else: - try: - dut_failed_results = defaultdict(list) - infra_recovery_actions = [] - for failed_result in failed_results: - if 'host' in failed_result: - dut_failed_results[failed_result['host']].append(failed_result) - if 'hosts' in failed_result: - for hostname in failed_result['hosts']: - dut_failed_results[hostname].append(failed_result) - if failed_result['check_item'] in constants.INFRA_CHECK_ITEMS: - if 'action' in failed_result and failed_result['action'] is not None \ - and callable(failed_result['action']): - infra_recovery_actions.append(failed_result['action']) - for action in infra_recovery_actions: - action() - for dut_name, dut_results in list(dut_failed_results.items()): - # Attempt to restore DUT state - recover(duthosts[dut_name], localhost, fanouthosts, nbrhosts, tbinfo, dut_results, - recover_method) - - except BaseException as e: - request.config.cache.set("pre_sanity_check_failed", True) - logger.error("Recovery of sanity check failed with exception: ") - pt_assert( - False, - "!!!!!!!!!!!!!!!! Recovery of sanity check failed !!!!!!!!!!!!!!!!" - "Exception: {}".format(repr(e)) - ) - - logger.info("Run sanity check again after recovery") - new_check_results = do_checks(request, pre_check_items, stage=STAGE_PRE_TEST, after_recovery=True) - logger.debug("Pre-test sanity check after recovery results:\n%s" % - json.dumps(new_check_results, indent=4, default=fallback_serializer)) - - new_failed_results = [result for result in new_check_results if result['failed']] - if new_failed_results: - request.config.cache.set("pre_sanity_check_failed", True) - pt_assert(False, - "!!!!!!!!!!!!!!!! Pre-test sanity check after recovery failed: !!!!!!!!!!!!!!!!\n{}" - .format(json.dumps(new_failed_results, indent=4, default=fallback_serializer))) + recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, localhost, nbrhosts, + pre_check_items, recover_method, request, tbinfo, STAGE_PRE_TEST) logger.info("Done pre-test sanity check") else: @@ -296,10 +307,82 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): post_failed_results = [result for result in post_check_results if result['failed']] if post_failed_results: - request.config.cache.set("post_sanity_check_failed", True) - pt_assert(False, "!!!!!!!!!!!!!!!! Post-test sanity check failed: !!!!!!!!!!!!!!!!\n{}" - .format(json.dumps(post_failed_results, indent=4, default=fallback_serializer))) + if not allow_recover: + request.config.cache.set("post_sanity_check_failed", True) + pt_assert(False, "!!!!!!!!!!!!!!!! Post-test sanity check failed: !!!!!!!!!!!!!!!!\n{}" + .format(json.dumps(post_failed_results, indent=4, default=fallback_serializer))) + else: + recover_on_sanity_check_failure(duthosts, post_failed_results, fanouthosts, localhost, nbrhosts, + post_check_items, recover_method, request, tbinfo, STAGE_POST_TEST) logger.info("Done post-test sanity check") else: logger.info('No post-test sanity check item, skip post-test sanity check.') + + +def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, localhost, nbrhosts, check_items, + recover_method, request, tbinfo, sanity_check_stage: str): + cache_key = "pre_sanity_check_failed" + recovery_cache_key = "pre_sanity_recovered" + if sanity_check_stage == STAGE_POST_TEST: + cache_key = "post_sanity_check_failed" + recovery_cache_key = "post_sanity_recovered" + + try: + dut_failed_results = defaultdict(list) + infra_recovery_actions = [] + for failed_result in failed_results: + if 'host' in failed_result: + dut_failed_results[failed_result['host']].append(failed_result) + if 'hosts' in failed_result: + for hostname in failed_result['hosts']: + dut_failed_results[hostname].append(failed_result) + if failed_result['check_item'] in constants.INFRA_CHECK_ITEMS: + if 'action' in failed_result and failed_result['action'] is not None \ + and callable(failed_result['action']): + infra_recovery_actions.append(failed_result['action']) + for action in infra_recovery_actions: + action() + + is_modular_chassis = duthosts[0].get_facts().get("modular_chassis") + if is_modular_chassis: + recover_chassis(duthosts) + else: + for dut_name, dut_results in list(dut_failed_results.items()): + # Attempt to restore DUT state + recover(duthosts[dut_name], localhost, fanouthosts, nbrhosts, tbinfo, dut_results, + recover_method) + + except BaseException as e: + request.config.cache.set(cache_key, True) + request.config.cache.set(recovery_cache_key, False) + + logger.error(f"Recovery of sanity check failed with exception: {repr(e)}") + pt_assert( + False, + f"!!!!!!!!!!!!!!!! Recovery of sanity check failed !!!!!!!!!!!!!!!!" + f"Exception: {repr(e)}" + ) + logger.info("Run sanity check again after recovery") + new_check_results = do_checks(request, check_items, stage=sanity_check_stage, after_recovery=True) + logger.debug(f"{sanity_check_stage} sanity check after recovery results: \n%s" % + json.dumps(new_check_results, indent=4, default=fallback_serializer)) + new_failed_results = [result for result in new_check_results if result['failed']] + if new_failed_results: + request.config.cache.set(cache_key, True) + request.config.cache.set(recovery_cache_key, False) + pt_assert(False, + f"!!!!!!!!!!!!!!!! {sanity_check_stage} sanity check after recovery failed: !!!!!!!!!!!!!!!!\n" + f"{json.dumps(new_failed_results, indent=4, default=fallback_serializer)}") + # Record recovery success + request.config.cache.set(recovery_cache_key, True) + + +# make sure teardown of log_custom_msg happens after sanity_check +@pytest.fixture(scope="module", autouse=True) +def sanity_check(request, log_custom_msg): + if request.config.option.skip_sanity: + logger.info("Skip sanity check according to command line argument") + yield + else: + yield request.getfixturevalue('sanity_check_full') diff --git a/tests/common/plugins/sanity_check/checks.py b/tests/common/plugins/sanity_check/checks.py index e7239bdcdc..61d4d8bb2f 100644 --- a/tests/common/plugins/sanity_check/checks.py +++ b/tests/common/plugins/sanity_check/checks.py @@ -14,6 +14,7 @@ from tests.common.helpers.parallel import parallel_run, reset_ansible_local_tmp from tests.common.dualtor.mux_simulator_control import _probe_mux_ports from tests.common.fixtures.duthost_utils import check_bgp_router_id +from tests.common.errors import RunAnsibleModuleFail logger = logging.getLogger(__name__) SYSTEM_STABILIZE_MAX_TIME = 300 @@ -29,7 +30,9 @@ 'check_monit', 'check_secureboot', 'check_neighbor_macsec_empty', - 'check_mux_simulator'] + 'check_ipv6_mgmt', + 'check_mux_simulator', + 'check_orchagent_usage'] __all__ = CHECK_ITEMS @@ -64,25 +67,6 @@ def _find_down_ip_ports(dut, ip_interfaces): return down_ip_ports -def _parse_bfd_output(output): - data_rows = output[3:] - data_dict = {} - for data in data_rows: - data = data.split() - data_dict[data[0]] = {} - data_dict[data[0]]['Interface'] = data[1] - data_dict[data[0]]['Vrf'] = data[2] - data_dict[data[0]]['State'] = data[3] - data_dict[data[0]]['Type'] = data[4] - data_dict[data[0]]['Local Addr'] = data[5] - data_dict[data[0]]['TX Interval'] = data[6] - data_dict[data[0]]['RX Interval'] = data[7] - data_dict[data[0]]['Multiplier'] = data[8] - data_dict[data[0]]['Multihop'] = data[9] - data_dict[data[0]]['Local Discriminator'] = data[10] - return data_dict - - def _find_down_ports(dut, phy_interfaces, ip_interfaces): """Finds the ports which are operationally down @@ -107,7 +91,7 @@ def check_interfaces(duthosts): def _check(*args, **kwargs): result = parallel_run(_check_interfaces_on_dut, args, kwargs, duthosts.frontend_nodes, - timeout=600, init_result=init_result) + timeout=1200, init_result=init_result) return list(result.values()) @reset_ansible_local_tmp @@ -118,6 +102,8 @@ def _check_interfaces_on_dut(*args, **kwargs): networking_uptime = dut.get_networking_uptime().seconds timeout = max((SYSTEM_STABILIZE_MAX_TIME - networking_uptime), 0) + if dut.get_facts().get("modular_chassis"): + timeout = max(timeout, 900) interval = 20 logger.info("networking_uptime=%d seconds, timeout=%d seconds, interval=%d seconds" % (networking_uptime, timeout, interval)) @@ -171,7 +157,7 @@ def check_bgp(duthosts, tbinfo): def _check(*args, **kwargs): result = parallel_run(_check_bgp_on_dut, args, kwargs, duthosts.frontend_nodes, - timeout=600, init_result=init_result) + timeout=1200, init_result=init_result) return list(result.values()) @reset_ansible_local_tmp @@ -244,6 +230,8 @@ def _check_bgp_status_helper(): max_timeout = 500 else: max_timeout = SYSTEM_STABILIZE_MAX_TIME - networking_uptime + 480 + if dut.get_facts().get("modular_chassis"): + max_timeout = max(max_timeout, 900) timeout = max(max_timeout, 1) interval = 20 wait_until(timeout, interval, 0, _check_bgp_status_helper) @@ -274,17 +262,20 @@ def _is_db_omem_over_threshold(command_output): total_omem = 0 re_omem = re.compile(r"omem=(\d+)") result = False + non_zero_output = [] for line in command_output: m = re_omem.search(line) if m: omem = int(m.group(1)) total_omem += omem + if omem > 0: + non_zero_output.append(line) logger.debug('total_omen={}, OMEM_THRESHOLD_BYTES={}'.format(total_omem, OMEM_THRESHOLD_BYTES)) if total_omem > OMEM_THRESHOLD_BYTES: result = True - return result, total_omem + return result, total_omem, non_zero_output @pytest.fixture(scope="module") @@ -305,11 +296,13 @@ def _check_dbmemory_on_dut(*args, **kwargs): # check the db memory on the redis instance running on each instance for asic in dut.asics: res = asic.run_redis_cli_cmd(redis_cmd)['stdout_lines'] - result, total_omem = _is_db_omem_over_threshold(res) + result, total_omem, non_zero_output = _is_db_omem_over_threshold(res) check_result["total_omem"] = total_omem if result: check_result["failed"] = True logging.info("{} db memory over the threshold ".format(str(asic.namespace or ''))) + logging.info("{} db memory omem non-zero output: \n{}" + .format(str(asic.namespace or ''), "\n".join(non_zero_output))) break logger.info("Done checking database memory on %s" % dut.hostname) results[dut.hostname] = check_result @@ -618,10 +611,14 @@ def _check_mux_status_helper(): return False, err_msg, {} if not check_success: - if len(dut_wrong_mux_status_ports) == 0: + if len(dut_wrong_mux_status_ports) != 0: # NOTE: Let's probe here to see if those inconsistent mux ports could be # restored before using the recovery method. - _probe_mux_ports(duthosts, dut_wrong_mux_status_ports) + port_index_map = duts_minigraph_facts[duthosts[0].hostname][0][1]['minigraph_port_indices'] + dut_wrong_mux_status_ports = list(set(dut_wrong_mux_status_ports)) + inconsistent_mux_ports = [port for port, port_index in port_index_map.items() + if port_index in dut_wrong_mux_status_ports] + _probe_mux_ports(duthosts, inconsistent_mux_ports) if not wait_until(60, 10, 0, _check_mux_status_helper): if err_msg_from_mux_status: err_msg = err_msg_from_mux_status[-1] @@ -997,3 +994,58 @@ def _check(*args, **kwargs): return init_check_result return _check + + +# check ipv6 neighbor reachability +@pytest.fixture(scope="module") +def check_ipv6_mgmt(duthosts, localhost): + # check ipv6 mgmt interface reachability for debugging purpose only. + # No failure will be trigger for this sanity check. + def _check(*args, **kwargs): + init_result = {"failed": False, "check_item": "ipv6_mgmt"} + result = parallel_run(_check_ipv6_mgmt_to_dut, args, kwargs, duthosts, timeout=30, init_result=init_result) + return list(result.values()) + + def _check_ipv6_mgmt_to_dut(*args, **kwargs): + dut = kwargs['node'] + results = kwargs['results'] + + logger.info("Checking ipv6 mgmt interface reachability on %s..." % dut.hostname) + check_result = {"failed": False, "check_item": "ipv6_mgmt", "host": dut.hostname} + + # most of the testbed should reply within 10 ms, Set the timeout to 2 seconds to reduce the impact of delay. + try: + shell_result = localhost.shell("ping6 -c 2 -W 2 " + dut.mgmt_ipv6) + logging.info("ping6 output: %s" % shell_result["stdout"]) + except RunAnsibleModuleFail as e: + # set to False for now to avoid blocking the test + check_result["failed"] = False + logging.info("Failed to ping ipv6 mgmt interface on %s, exception: %s" % (dut.hostname, repr(e))) + except Exception as e: + logger.info("Exception while checking ipv6_mgmt reachability for %s: %s" % (dut.hostname, repr(e))) + finally: + logger.info("Done checking ipv6 management reachability on %s" % dut.hostname) + results[dut.hostname] = check_result + return _check + + +@pytest.fixture(scope="module") +def check_orchagent_usage(duthosts): + def _check(*args, **kwargs): + init_result = {"failed": False, "check_item": "orchagent_usage"} + result = parallel_run(_check_orchagent_usage_on_dut, args, kwargs, duthosts, + timeout=600, init_result=init_result) + + return list(result.values()) + + def _check_orchagent_usage_on_dut(*args, **kwargs): + dut = kwargs['node'] + results = kwargs['results'] + logger.info("Checking orchagent CPU usage on %s..." % dut.hostname) + check_result = {"failed": False, "check_item": "orchagent_usage", "host": dut.hostname} + res = dut.shell("COLUMNS=512 show processes cpu | grep orchagent | awk '{print $9}'")["stdout_lines"] + check_result["orchagent_usage"] = res + logger.info("Done checking orchagent CPU usage on %s" % dut.hostname) + results[dut.hostname] = check_result + + return _check diff --git a/tests/common/plugins/sanity_check/recover.py b/tests/common/plugins/sanity_check/recover.py index 97d2f9af5e..de894b9a2a 100644 --- a/tests/common/plugins/sanity_check/recover.py +++ b/tests/common/plugins/sanity_check/recover.py @@ -9,6 +9,7 @@ from tests.common.reboot import reboot from tests.common.utilities import wait from . import constants +from ...helpers.parallel_utils import config_reload_parallel_compatible logger = logging.getLogger(__name__) @@ -195,3 +196,12 @@ def recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_results, recove reboot_dut(dut, localhost, method["cmd"]) else: _recover_with_command(dut, method['cmd'], wait_time) + + +def recover_chassis(duthosts): + logger.warning(f"Try to recover chassis {[dut.hostname for dut in duthosts]} using config reload") + return parallel_run(config_reload_parallel_compatible, (), + { + 'config_source': 'running_golden_config', 'safe_reload': True, + 'check_intf_up_ports': True, 'wait_for_bgp': True}, + duthosts, timeout=1200) diff --git a/tests/common/port_toggle.py b/tests/common/port_toggle.py index b24bfa1871..890288394b 100644 --- a/tests/common/port_toggle.py +++ b/tests/common/port_toggle.py @@ -80,7 +80,7 @@ def __get_down_ports(expect_up=True): if not startup_ok: down_ports = __get_down_ports() - startup_err_msg = "Some ports did not come up as expected: {}".format(str(down_ports)) + startup_err_msg = "{}: Some ports did not come up as expected: {}".format(duthost.hostname, str(down_ports)) except Exception as e: startup_err_msg = "Startup ports failed with exception: {}".format(repr(e)) @@ -128,5 +128,8 @@ def default_port_toggle_wait_time(duthost, port_count): port_count_factor = port_count / BASE_PORT_COUNT port_down_wait_time = int(port_down_wait_time * port_count_factor) port_up_wait_time = int(port_up_wait_time * port_count_factor) + elif duthost.get_facts().get("modular_chassis"): + port_down_wait_time = 300 + port_up_wait_time = 300 return port_down_wait_time, port_up_wait_time diff --git a/tests/common/reboot.py b/tests/common/reboot.py index 99cb053ec6..559646c4fe 100644 --- a/tests/common/reboot.py +++ b/tests/common/reboot.py @@ -256,9 +256,12 @@ def reboot(duthost, localhost, reboot_type='cold', delay=10, timeout = plt_reboot_ctrl.get('timeout', timeout) if warmboot_finalizer_timeout == 0 and 'warmboot_finalizer_timeout' in reboot_ctrl: warmboot_finalizer_timeout = reboot_ctrl['warmboot_finalizer_timeout'] + if duthost.get_facts().get("modular_chassis"): + wait = max(wait, 900) + timeout = max(timeout, 600) except KeyError: raise ValueError('invalid reboot type: "{} for {}"'.format(reboot_type, hostname)) - + logger.info('Reboot {}: wait[{}], timeout[{}]'.format(hostname, wait, timeout)) # Create a temporary file in tmpfs before reboot logger.info('DUT {} create a file /dev/shm/test_reboot before rebooting'.format(hostname)) duthost.command('sudo touch /dev/shm/test_reboot') @@ -276,19 +279,18 @@ def reboot(duthost, localhost, reboot_type='cold', delay=10, wait_for_startup(duthost, localhost, delay, timeout) logger.info('waiting for switch {} to initialize'.format(hostname)) - if safe_reboot: # The wait time passed in might not be guaranteed to cover the actual # time it takes for containers to come back up. Therefore, add 5 # minutes to the maximum wait time. If it's ready sooner, then the # function will return sooner. pytest_assert(wait_until(wait + 400, 20, 0, duthost.critical_services_fully_started), - "All critical services should be fully started!") + "{}: All critical services should be fully started!".format(hostname)) wait_critical_processes(duthost) if check_intf_up_ports: - pytest_assert(wait_until(300, 20, 0, check_interface_status_of_up_ports, duthost), - "Not all ports that are admin up on are operationally up") + pytest_assert(wait_until(wait + 300, 20, 0, check_interface_status_of_up_ports, duthost), + "{}: Not all ports that are admin up on are operationally up".format(hostname)) else: time.sleep(wait) @@ -340,6 +342,14 @@ def get_reboot_cause(dut): output = dut.shell('show reboot-cause') cause = output['stdout'] + # For kvm testbed, the expected output of command `show reboot-cause` + # is such like "User issued 'xxx' command [User: admin, Time: Sun Aug 4 06:43:19 PM UTC 2024]" + # So, use the above pattern to get real reboot cause + if dut.facts["asic_type"] == "vs": + match = re.search("User issued '(.*)' command", cause) + if match: + cause = match.groups()[0] + for type, ctrl in list(reboot_ctrl_dict.items()): if re.search(ctrl['cause'], cause): return type @@ -452,6 +462,11 @@ def check_reboot_cause_history(dut, reboot_type_history_queue): logger.debug("dut {} reboot-cause history {}. reboot type history queue is {}".format( dut.hostname, reboot_cause_history_got, reboot_type_history_queue)) + # For kvm testbed, command `show reboot-cause history` will return None + # So, return in advance if this check is running on kvm. + if dut.facts["asic_type"] == "vs": + return True + logger.info("Verify reboot-cause history title") if reboot_cause_history_got: if not set(REBOOT_CAUSE_HISTORY_TITLE) == set(reboot_cause_history_got[0].keys()): diff --git a/tests/common/snappi_tests/common_helpers.py b/tests/common/snappi_tests/common_helpers.py index cc5a9f15cc..d7f79642b4 100644 --- a/tests/common/snappi_tests/common_helpers.py +++ b/tests/common/snappi_tests/common_helpers.py @@ -18,6 +18,8 @@ from tests.common.mellanox_data import is_mellanox_device as isMellanoxDevice from ipaddress import IPv6Network, IPv6Address from random import getrandbits +from tests.common.portstat_utilities import parse_portstat +from collections import defaultdict def increment_ip_address(ip, incr=1): @@ -32,7 +34,7 @@ def increment_ip_address(ip, incr=1): ipaddress = ipaddr.IPv4Address(ip) ipaddress = ipaddress + incr return_value = ipaddress._string_from_ip_int(ipaddress._ip) - return(return_value) + return return_value def ansible_stdout_to_str(ansible_stdout): @@ -105,15 +107,30 @@ def get_lossless_buffer_size(host_ans): Returns: total switch buffer size in byte (int) """ - config_facts = host_ans.config_facts(host=host_ans.hostname, + dut_asic = host_ans.asic_instance() + config_facts = dut_asic.config_facts(host=host_ans.hostname, source="running")['ansible_facts'] + is_cisco8000_platform = True if 'cisco-8000' in host_ans.facts['platform_asic'] else False + # Checking if platform is Broadcom-DNX. + is_broadcom_dnx = ( + True + if "platform_asic" in host_ans.facts and host_ans.facts["platform_asic"] == "broadcom-dnx" + else False + ) + if "BUFFER_POOL" not in list(config_facts.keys()): return None buffer_pools = config_facts['BUFFER_POOL'] - profile_name = 'ingress_lossless_pool' if is_cisco8000_platform else 'egress_lossless_pool' + + # Added check to select ingress_lossles_pool for Nokia7250 platform. + profile_name = ( + 'ingress_lossless_pool' + if (is_cisco8000_platform or is_broadcom_dnx) + else 'egress_lossless_pool' + ) if profile_name not in list(buffer_pools.keys()): return None @@ -122,7 +139,7 @@ def get_lossless_buffer_size(host_ans): return int(lossless_pool['size']) -def get_pg_dropped_packets(duthost, phys_intf, prio): +def get_pg_dropped_packets(duthost, phys_intf, prio, asic_value=None): """ Get number of ingress packets dropped on a specific priority of a physical interface @@ -133,14 +150,22 @@ def get_pg_dropped_packets(duthost, phys_intf, prio): Returns: total number of dropped packets (int) """ - oid_cmd = "sonic-db-cli COUNTERS_DB HGET COUNTERS_QUEUE_NAME_MAP " + phys_intf + ":" + str(prio) + if asic_value is None: + oid_cmd = "sonic-db-cli COUNTERS_DB HGET COUNTERS_QUEUE_NAME_MAP " + phys_intf + ":" + str(prio) + else: + oid_cmd = "sudo ip netns exec {} sonic-db-cli COUNTERS_DB HGET COUNTERS_QUEUE_NAME_MAP ".format(asic_value) \ + + phys_intf + ":" + str(prio) oid_out = duthost.command(oid_cmd) oid_str = str(oid_out["stdout_lines"][0] or 1) if oid_str == "1": return None - cmd = "sonic-db-cli COUNTERS_DB HGET COUNTERS:" + oid_str + " SAI_QUEUE_STAT_DROPPED_PACKETS" + if asic_value is None: + cmd = "sonic-db-cli COUNTERS_DB HGET COUNTERS:" + oid_str + " SAI_QUEUE_STAT_DROPPED_PACKETS" + else: + cmd = "sudo ip netns exec {} sonic-db-cli COUNTERS_DB HGET COUNTERS:".format(asic_value) \ + + oid_str + " SAI_QUEUE_STAT_DROPPED_PACKETS" out = duthost.command(cmd) dropped_packets = int(out["stdout_lines"][0] or -1) @@ -229,7 +254,13 @@ def get_peer_snappi_chassis(conn_data, dut_hostname): if len(peer_devices) == 1: return peer_devices[0] else: - return None + # in case there are other fanout devices (Arista, SONiC, etc) defined in the inventory file, + # try to filter out the other device based on the name for now. + peer_snappi_devices = list(filter(lambda dut_name: ('ixia' in dut_name), peer_devices)) + if len(peer_snappi_devices) == 1: + return peer_snappi_devices[0] + else: + return None def get_peer_port(conn_data, dut_hostname, dut_intf): @@ -439,34 +470,43 @@ def config_wred(host_ans, kmin, kmax, pmax, profile=None, asic_value=None): if profile is not None and profile not in profiles: return False + color = 'green' + + # Broadcom ASIC only supports RED. + if asic_type == 'broadcom': + color = 'red' + + kmax_arg = '-{}max' % color[0] + kmin_arg = '-{}min' % color[0] + for p in profiles: """ This is not the profile to configure """ if profile is not None and profile != p: continue - kmin_old = int(profiles[p]['green_min_threshold']) - kmax_old = int(profiles[p]['green_max_threshold']) + kmin_old = int(profiles[p]['{}_min_threshold' % color]) + kmax_old = int(profiles[p]['{}_max_threshold' % color]) if kmin_old > kmax_old: return False """ Ensure that Kmin is no larger than Kmax during the update """ - gmax_cmd = 'sudo ecnconfig -p {} -gmax {}' - gmin_cmd = 'sudo ecnconfig -p {} -gmin {}' + kmax_cmd = ' '.join(['sudo ecnconfig -p {}', kmax_arg, '{}']) + kmin_cmd = ' '.join(['sudo ecnconfig -p {}', kmin_arg, '{}']) if asic_value is not None: - gmax_cmd = 'sudo ip netns exec %s ecnconfig -p {} -gmax {}' % asic_value - gmin_cmd = 'sudo ip netns exec %s ecnconfig -p {} -gmin {}' % asic_value + kmax_cmd = ' '.join(['sudo ip netns exec', asic_value, 'ecnconfig -p {}', kmax_arg, '{}']) + kmin_cmd = ' '.join(['sudo ip netns exec', asic_value, 'ecnconfig -p {}', kmin_arg, '{}']) if asic_type == 'broadcom': disable_packet_aging(host_ans, asic_value) if kmin > kmin_old: - host_ans.shell(gmax_cmd.format(p, kmax)) - host_ans.shell(gmin_cmd.format(p, kmin)) + host_ans.shell(kmax_cmd.format(p, kmax)) + host_ans.shell(kmin_cmd.format(p, kmin)) else: - host_ans.shell(gmin_cmd.format(p, kmin)) - host_ans.shell(gmax_cmd.format(p, kmax)) + host_ans.shell(kmin_cmd.format(p, kmin)) + host_ans.shell(kmax_cmd.format(p, kmax)) return True @@ -490,7 +530,7 @@ def enable_ecn(host_ans, prio, asic_value=None): return True else: host_ans.shell('sudo ip netns exec {} ecnconfig -q {} on'.format(asic_value, prio)) - results = host_ans.shell('sudo ip netns exec {} ecnconfig {}'.format(asic_value, prio)) + results = host_ans.shell('sudo ip netns exec {} ecnconfig -q {}'.format(asic_value, prio)) if re.search("queue {}: on".format(prio), results['stdout']): return True return False @@ -644,7 +684,7 @@ def get_pfcwd_poll_interval(host_ans, asic_value=None): val = get_pfcwd_config_attr(host_ans=host_ans, config_scope='GLOBAL', attr='POLL_INTERVAL', - namespace=asic_value) + asic_value=asic_value) if val is not None: return int(val) @@ -671,7 +711,7 @@ def get_pfcwd_detect_time(host_ans, intf, asic_value=None): val = get_pfcwd_config_attr(host_ans=host_ans, config_scope=intf, attr='detection_time', - namespace=asic_value) + asic_value=asic_value) if val is not None: return int(val) @@ -698,7 +738,7 @@ def get_pfcwd_restore_time(host_ans, intf, asic_value=None): val = get_pfcwd_config_attr(host_ans=host_ans, config_scope=intf, attr='restoration_time', - namespace=asic_value) + asic_value=asic_value) if val is not None: return int(val) @@ -706,6 +746,43 @@ def get_pfcwd_restore_time(host_ans, intf, asic_value=None): return None +def get_pfcwd_stats(duthost, port, prio, asic_value=None): + """ + Get PFC watchdog statistics for given interface:prio + Args: + duthost : Ansible host instance of the device + port : Port for which stats needs to be gathered. + prio : Lossless priority for which stats needs to be captured. + asic_value : asic value of the host + + Returns: + Dictionary with PFCWD statistics as key-value pair. + If the entry is not present, then values are returned as zero. + """ + + pfcwd_stats = {} + if asic_value is None: + raw_out = duthost.shell("show pfcwd stats | grep -E 'QUEUE|{}:{}'".format(port, prio))['stdout'] + else: + comm = "sudo ip netns exec {} show pfcwd stats | grep -E 'QUEUE|{}:{}'".format(asic_value, port, prio) + raw_out = duthost.shell(comm)['stdout'] + + val_list = [] + key_list = [] + for line in raw_out.split('\n'): + if ('QUEUE' in line): + key_list = (re.sub(r"(\w) (\w)", r"\1_\2", line)).split() + else: + val_list = line.split() + if val_list: + for key, val in zip(key_list, val_list): + pfcwd_stats[key] = val + else: + pfcwd_stats = {key: '0/0' if '/' in key else 0 for key in key_list} + + return pfcwd_stats + + def start_pfcwd(duthost, asic_value=None): """ Start PFC watchdog with default setting @@ -753,10 +830,19 @@ def disable_packet_aging(duthost, asic_value=None): duthost.command("docker exec syncd python /packets_aging.py disable") duthost.command("docker exec syncd rm -rf /packets_aging.py") elif "platform_asic" in duthost.facts and duthost.facts["platform_asic"] == "broadcom-dnx": - try: - duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog disable"'.format(asic_value)) - except Exception: - duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog disable"'.format(asic_value[-1])) + # if asic_value is present, disable packet aging for specific asic_value. + if (asic_value): + try: + duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog disable"'.format(asic_value)) + except Exception: + duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog disable"'.format(asic_value[-1])) + else: + # Disabling packet aging for all asics on given DUT. + for asic_value in duthost.facts['asics_present']: + try: + duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog disable"'.format(asic_value)) + except Exception: + duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog disable"'.format(asic_value[-1])) def enable_packet_aging(duthost, asic_value=None): @@ -774,10 +860,19 @@ def enable_packet_aging(duthost, asic_value=None): duthost.command("docker exec syncd python /packets_aging.py enable") duthost.command("docker exec syncd rm -rf /packets_aging.py") elif "platform_asic" in duthost.facts and duthost.facts["platform_asic"] == "broadcom-dnx": - try: - duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog enable"'.format(asic_value)) - except Exception: - duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog enable"'.format(asic_value[-1])) + # if asic_value is present, enable packet aging for specific asic_value. + if (asic_value): + try: + duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog enable"'.format(asic_value)) + except Exception: + duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog enable"'.format(asic_value[-1])) + else: + # Enabling packet aging for all asics on given DUT. + for asic_value in duthost.facts['asics_present']: + try: + duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog enable"'.format(asic_value)) + except Exception: + duthost.shell('bcmcmd -n {} "BCMSAI credit-watchdog enable"'.format(asic_value[-1])) def get_ipv6_addrs_in_subnet(subnet, number_of_ip): @@ -890,14 +985,22 @@ def get_egress_queue_count(duthost, port, priority): Returns: tuple (int, int): total count of packets and bytes in the queue """ - raw_out = duthost.shell("show queue counters {} | sed -n '/UC{}/p'".format(port, priority))['stdout'] - total_pkts = raw_out.split()[2] if 2 < len(raw_out.split()) else "0" - if total_pkts == "N/A": - total_pkts = "0" + # If DUT is multi-asic, asic will be used. + if duthost.is_multi_asic: + asic = duthost.get_port_asic_instance(port).get_asic_namespace() + raw_out = duthost.shell("sudo ip netns exec {} show queue counters {} | sed -n '/UC{}/p'". + format(asic, port, priority))['stdout'] + total_pkts = "0" if raw_out.split()[2] == "N/A" else raw_out.split()[2] + total_bytes = "0" if raw_out.split()[3] == "N/A" else raw_out.split()[3] + else: + raw_out = duthost.shell("show queue counters {} | sed -n '/UC{}/p'".format(port, priority))['stdout'] + total_pkts = raw_out.split()[2] if 2 < len(raw_out.split()) else "0" + if total_pkts == "N/A": + total_pkts = "0" - total_bytes = raw_out.split()[3] if 3 < len(raw_out.split()) else "0" - if total_bytes == "N/A": - total_bytes = "0" + total_bytes = raw_out.split()[3] if 3 < len(raw_out.split()) else "0" + if total_bytes == "N/A": + total_bytes = "0" return int(total_pkts.replace(',', '')), int(total_bytes.replace(',', '')) @@ -970,3 +1073,142 @@ def calc_pfc_pause_flow_rate(port_speed, oversubscription_ratio=2): pps = int(oversubscription_ratio / pause_dur) return pps + + +def start_pfcwd_fwd(duthost, asic_value=None): + """ + Start PFC watchdog in Forward mode. + Stops the PFCWD cleanly before restarting it in FORWARD mode. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + asic_value: asic value of the host + + Returns: + N/A + """ + + # Starting PFCWD in forward mode with detection and restoration duration of 200msec. + if asic_value is None: + stop_pfcwd(duthost) + duthost.shell('sudo pfcwd start --action forward 200 --restoration-time 200') + else: + stop_pfcwd(duthost, asic_value) + duthost.shell('sudo ip netns exec {} pfcwd start --action forward 200 --restoration-time 200'. + format(asic_value)) + + +def clear_counters(duthost, port): + """ + Clear PFC, Queuecounters, Drop and generic counters from SONiC CLI. + Args: + duthost (Ansible host instance): Device under test + port (str): port name + Returns: + None + """ + + duthost.shell("sudo sonic-clear counters \n") + duthost.shell("sudo sonic-clear pfccounters \n") + duthost.shell("sudo sonic-clear priority-group drop counters \n") + duthost.shell("sonic-clear counters \n") + duthost.shell("sonic-clear pfccounters \n") + + if (duthost.is_multi_asic): + asic = duthost.get_port_asic_instance(port).get_asic_namespace() + duthost.shell("sudo ip netns exec {} sonic-clear queuecounters \n".format(asic)) + duthost.shell("sudo ip netns exec {} sonic-clear dropcounters \n".format(asic)) + else: + duthost.shell("sonic-clear queuecounters \n") + duthost.shell("sonic-clear dropcounters \n") + + +def get_interface_stats(duthost, port): + """ + Get the Rx and Tx port failures, throughput and pkts from SONiC CLI. + This is the equivalent of the "show interface counters" command. + Args: + duthost (Ansible host instance): device under test + port (str): port name + Returns: + i_stats (dict): Returns various parameters for given DUT and port. + """ + # Initializing nested dictionary i_stats + i_stats = defaultdict(dict) + i_stats[duthost.hostname][port] = {} + + n_out = parse_portstat(duthost.command('portstat -i {}'.format(port))['stdout_lines'])[port] + # rx_err, rx_ovr and rx_drp are counted in single counter rx_fail + # tx_err, tx_ovr and tx_drp are counted in single counter tx_fail + rx_err = ['rx_err', 'rx_ovr', 'rx_drp'] + tx_err = ['tx_err', 'tx_ovr', 'tx_drp'] + rx_fail = 0 + tx_fail = 0 + for m in rx_err: + rx_fail = rx_fail + int(n_out[m].replace(',', '')) + for m in tx_err: + tx_fail = tx_fail + int(n_out[m].replace(',', '')) + + # Any throughput below 1MBps is measured as 0 for simplicity. + thrput = n_out['rx_bps'] + if thrput.split(' ')[1] == 'MB/s' and (thrput.split(' ')[0]) != '0.00': + i_stats[duthost.hostname][port]['rx_thrput_Mbps'] = float(thrput.split(' ')[0]) * 8 + else: + i_stats[duthost.hostname][port]['rx_thrput_Mbps'] = 0 + thrput = n_out['tx_bps'] + if thrput.split(' ')[1] == 'MB/s' and (thrput.split(' ')[0]) != '0.00': + i_stats[duthost.hostname][port]['tx_thrput_Mbps'] = float(thrput.split(' ')[0]) * 8 + else: + i_stats[duthost.hostname][port]['rx_thrput_Mbps'] = 0 + + i_stats[duthost.hostname][port]['rx_pkts'] = int(n_out['rx_ok'].replace(',', '')) + i_stats[duthost.hostname][port]['tx_pkts'] = int(n_out['tx_ok'].replace(',', '')) + i_stats[duthost.hostname][port]['rx_fail'] = rx_fail + i_stats[duthost.hostname][port]['tx_fail'] = tx_fail + + return i_stats + + +def get_queue_count_all_prio(duthost, port): + """ + Get the egress queue count in packets and bytes for a given port and priority from SONiC CLI. + This is the equivalent of the "show queue counters" command. + Args: + duthost (Ansible host instance): device under test + port (str): port name + Returns: + queue_dict (dict): key-value with key=dut+port+prio and value=queue count + """ + # Initializing nested dictionary queue_dict + queue_dict = defaultdict(dict) + queue_dict[duthost.hostname][port] = {} + + # Preparing the dictionary for all 7 priority queues. + for priority in range(7): + total_pkts, _ = get_egress_queue_count(duthost, port, priority) + queue_dict[duthost.hostname][port]['prio_' + str(priority)] = total_pkts + + return queue_dict + + +def get_pfc_count(duthost, port): + """ + Get the PFC frame count for a given port from SONiC CLI + Args: + duthost (Ansible host instance): device under test + port (str): port name + Returns: + pfc_dict (dict) : Returns Rx and Tx PFC for the given DUT and interface. + """ + pfc_dict = defaultdict(dict) + pfc_dict[duthost.hostname][port] = {} + raw_out = duthost.shell("show pfc counters | sed -n '/Port Tx/,/^$/p' | grep '{} '".format(port))['stdout'] + pause_frame_count = raw_out.split() + for m in range(1, len(pause_frame_count)): + pfc_dict[duthost.hostname][port]['tx_pfc_'+str(m-1)] = int(pause_frame_count[m].replace(',', '')) + + raw_out = duthost.shell("show pfc counters | sed -n '/Port Rx/,/^$/p' | grep '{} '".format(port))['stdout'] + pause_frame_count = raw_out.split() + for m in range(1, len(pause_frame_count)): + pfc_dict[duthost.hostname][port]['rx_pfc_'+str(m-1)] = int(pause_frame_count[m].replace(',', '')) + + return pfc_dict diff --git a/tests/common/snappi_tests/port.py b/tests/common/snappi_tests/port.py index 4c535667c5..7114e1cf94 100644 --- a/tests/common/snappi_tests/port.py +++ b/tests/common/snappi_tests/port.py @@ -24,6 +24,15 @@ def __init__(self, id, ip, mac, gw, gw_mac, prefix_len, port_type, peer_port): self.type = port_type self.peer_port = peer_port + def __str__(self): + return "".format( + self.id, self.ip, self.mac, self.gateway, self.gateway_mac, + self.prefix_len, self.type, self.peer_port) + + def __repr__(self): + return self.__str__() + def select_ports(port_config_list, pattern, rx_port_id): """ diff --git a/tests/common/snappi_tests/read_pcap.py b/tests/common/snappi_tests/read_pcap.py index af93efa6c7..1cd49f6213 100644 --- a/tests/common/snappi_tests/read_pcap.py +++ b/tests/common/snappi_tests/read_pcap.py @@ -62,12 +62,14 @@ def validate_pfc_frame(pfc_pcap_file, SAMPLE_SIZE=15000, UTIL_THRESHOLD=0.8): return True, None -def get_ip_pkts(pcap_file_name): +def get_ipv4_pkts(pcap_file_name, protocol_num=61): """ - Get IP packets from the pcap/pcapng file + Get IPv4 packets from the pcap/pcapng file Args: pcap_file_name (str): name of the pcap/pcapng file to store captured packets + protocol_num (int): protocol number to filter packets. See + https://en.wikipedia.org/wiki/List_of_IP_protocol_numbers Returns: Captured IP packets (list) @@ -82,7 +84,8 @@ def get_ip_pkts(pcap_file_name): for _, pkt in pcap: eth = dpkt.ethernet.Ethernet(pkt) if isinstance(eth.data, dpkt.ip.IP): - ip_pkts.append(eth.data) + if eth.data.p == protocol_num: + ip_pkts.append(eth.data) return ip_pkts diff --git a/tests/common/snappi_tests/snappi_fixtures.py b/tests/common/snappi_tests/snappi_fixtures.py old mode 100644 new mode 100755 index 1d3e43a27c..4bd8f0e80d --- a/tests/common/snappi_tests/snappi_fixtures.py +++ b/tests/common/snappi_tests/snappi_fixtures.py @@ -2,11 +2,13 @@ This module contains the snappi fixture in the snappi_tests directory. """ import pytest +import time import logging import snappi import sys import random import snappi_convergence +from tests.common.helpers.assertions import pytest_require from ipaddress import ip_address, IPv4Address, IPv6Address from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 from tests.common.snappi_tests.common_helpers import get_addrs_in_subnet, get_peer_snappi_chassis, \ @@ -14,7 +16,9 @@ from tests.common.snappi_tests.snappi_helpers import SnappiFanoutManager, get_snappi_port_location from tests.common.snappi_tests.port import SnappiPortConfig, SnappiPortType from tests.common.helpers.assertions import pytest_assert -from tests.snappi_tests.variables import dut_ip_start, snappi_ip_start, prefix_length +from tests.snappi_tests.variables import dut_ip_start, snappi_ip_start, prefix_length, \ + dut_ipv6_start, snappi_ipv6_start, v6_prefix_length, pfcQueueGroupSize, \ + pfcQueueValueDict # noqa: F401 logger = logging.getLogger(__name__) @@ -61,7 +65,9 @@ def snappi_api(snappi_api_serv_ip, # Going forward, we should be able to specify extension # from command line while running pytest. api = snappi.api(location=location, ext="ixnetwork") - + # TODO - Uncomment to use. Prefer to use environment vars to retrieve this information + # api._username = "" + # api._password = "" yield api if getattr(api, 'assistant', None) is not None: @@ -361,6 +367,11 @@ def snappi_testbed_config(conn_graph_facts, fanout_graph_facts, # noqa F811 - config (obj): Snappi API config of the testbed - port_config_list (list): list of port configuration information """ + # As of now both single dut and multidut fixtures are being called from the same test, + # When this function is called for T2 testbed, just return empty. + if is_snappi_multidut(duthosts): + return None, [] + duthost = duthosts[rand_one_dut_hostname] """ Generate L1 config """ @@ -403,14 +414,26 @@ def snappi_testbed_config(conn_graph_facts, fanout_graph_facts, # noqa F811 pfc = l1_config.flow_control.ieee_802_1qbb pfc.pfc_delay = 0 - pfc.pfc_class_0 = 0 - pfc.pfc_class_1 = 1 - pfc.pfc_class_2 = 2 - pfc.pfc_class_3 = 3 - pfc.pfc_class_4 = 4 - pfc.pfc_class_5 = 5 - pfc.pfc_class_6 = 6 - pfc.pfc_class_7 = 7 + if pfcQueueGroupSize == 8: + pfc.pfc_class_0 = 0 + pfc.pfc_class_1 = 1 + pfc.pfc_class_2 = 2 + pfc.pfc_class_3 = 3 + pfc.pfc_class_4 = 4 + pfc.pfc_class_5 = 5 + pfc.pfc_class_6 = 6 + pfc.pfc_class_7 = 7 + elif pfcQueueGroupSize == 4: + pfc.pfc_class_0 = pfcQueueValueDict[0] + pfc.pfc_class_1 = pfcQueueValueDict[1] + pfc.pfc_class_2 = pfcQueueValueDict[2] + pfc.pfc_class_3 = pfcQueueValueDict[3] + pfc.pfc_class_4 = pfcQueueValueDict[4] + pfc.pfc_class_5 = pfcQueueValueDict[5] + pfc.pfc_class_6 = pfcQueueValueDict[6] + pfc.pfc_class_7 = pfcQueueValueDict[7] + else: + pytest_assert(False, 'pfcQueueGroupSize value is not 4 or 8') port_config_list = [] @@ -512,7 +535,8 @@ def tgen_ports(duthost, conn_graph_facts, fanout_graph_facts): # noqa F811 port['peer_ipv6'], port['ipv6_prefix'] = ipv6_subnet.split("/") port['ipv6'] = get_ipv6_addrs_in_subnet(ipv6_subnet, 1)[0] except Exception: - raise Exception('Configure IPv4 and IPv6 on DUT interfaces') + snappi_ports = pre_configure_dut_interface(duthost, snappi_ports) + logger.info(snappi_ports) return snappi_ports @@ -549,7 +573,7 @@ def snappi_dut_base_config(duthost_list, for i, sp in enumerate(snappi_ports) if sp['location'] in tgen_ports] pytest_assert(len(set([sp['speed'] for sp in new_snappi_ports])) == 1, 'Ports have different link speeds') [config.ports.port(name='Port {}'.format(sp['port_id']), location=sp['location']) for sp in new_snappi_ports] - speed_gbps = int(new_snappi_ports[0]['speed'])/1000 + speed_gbps = int(int(new_snappi_ports[0]['speed'])/1000) config.options.port_options.location_preemption = True l1_config = config.layer1.layer1()[-1] @@ -558,23 +582,67 @@ def snappi_dut_base_config(duthost_list, l1_config.speed = 'speed_{}_gbps'.format(speed_gbps) l1_config.ieee_media_defaults = False l1_config.auto_negotiate = False - l1_config.auto_negotiation.link_training = False + if is_snappi_multidut(duthost_list): + l1_config.auto_negotiation.link_training = False + else: + l1_config.auto_negotiation.link_training = True l1_config.auto_negotiation.rs_fec = True pfc = l1_config.flow_control.ieee_802_1qbb pfc.pfc_delay = 0 - [setattr(pfc, 'pfc_class_{}'.format(i), i) for i in range(8)] + if pfcQueueGroupSize == 8: + pfc.pfc_class_0 = 0 + pfc.pfc_class_1 = 1 + pfc.pfc_class_2 = 2 + pfc.pfc_class_3 = 3 + pfc.pfc_class_4 = 4 + pfc.pfc_class_5 = 5 + pfc.pfc_class_6 = 6 + pfc.pfc_class_7 = 7 + elif pfcQueueGroupSize == 4: + pfc.pfc_class_0 = pfcQueueValueDict[0] + pfc.pfc_class_1 = pfcQueueValueDict[1] + pfc.pfc_class_2 = pfcQueueValueDict[2] + pfc.pfc_class_3 = pfcQueueValueDict[3] + pfc.pfc_class_4 = pfcQueueValueDict[4] + pfc.pfc_class_5 = pfcQueueValueDict[5] + pfc.pfc_class_6 = pfcQueueValueDict[6] + pfc.pfc_class_7 = pfcQueueValueDict[7] + else: + pytest_assert(False, 'pfcQueueGroupSize value is not 4 or 8') port_config_list = [] for index, duthost in enumerate(duthost_list): - config_result = __intf_config_multidut( - config=config, - port_config_list=port_config_list, - duthost=duthost, - snappi_ports=new_snappi_ports) + config_result = __vlan_intf_config(config=config, + port_config_list=port_config_list, + duthost=duthost, + snappi_ports=new_snappi_ports) pytest_assert(config_result is True, 'Fail to configure Vlan interfaces') + for index, duthost in enumerate(duthost_list): + config_result = __portchannel_intf_config(config=config, + port_config_list=port_config_list, + duthost=duthost, + snappi_ports=new_snappi_ports) + pytest_assert(config_result is True, 'Fail to configure portchannel interfaces') + + if is_snappi_multidut(duthost_list): + for index, duthost in enumerate(duthost_list): + config_result = __intf_config_multidut( + config=config, + port_config_list=port_config_list, + duthost=duthost, + snappi_ports=new_snappi_ports) + pytest_assert(config_result is True, 'Fail to configure multidut L3 interfaces') + else: + for index, duthost in enumerate(duthost_list): + config_result = __l3_intf_config(config=config, + port_config_list=port_config_list, + duthost=duthost, + snappi_ports=new_snappi_ports) + pytest_assert(config_result is True, 'Fail to configure L3 interfaces') + return config, port_config_list, new_snappi_ports @@ -600,7 +668,7 @@ def _get_multidut_snappi_ports(line_card_choice, line_card_info): ports = [] for index, host in enumerate(duthosts): snappi_fanout_list = SnappiFanoutManager(fanout_graph_facts) - for i in range(0, 3): + for i in range(len(snappi_fanout_list.fanout_list)): try: snappi_fanout_list.get_fanout_device_details(i) except Exception: @@ -613,6 +681,7 @@ def _get_multidut_snappi_ports(line_card_choice, line_card_info): if port["peer_port"] in asic_port_map[asic] and hostname in port['peer_device']: port['asic_value'] = asic port['asic_type'] = host.facts["asic_type"] + port['duthost'] = host ports.append(port) return ports return _get_multidut_snappi_ports @@ -739,6 +808,7 @@ def __intf_config_multidut(config, port_config_list, duthost, snappi_ports): port['peer_port'], dutIp, prefix_length)) + port['intf_config_changed'] = True device = config.devices.device(name='Device Port {}'.format(port_id))[-1] ethernet = device.ethernets.add() ethernet.name = 'Ethernet Port {}'.format(port_id) @@ -754,7 +824,7 @@ def __intf_config_multidut(config, port_config_list, duthost, snappi_ports): ip=tgenIp, mac=mac, gw=dutIp, - gw_mac=str(duthost.facts['router_mac']), + gw_mac=duthost.get_dut_iface_mac(port['peer_port']), prefix_len=prefix_length, port_type=SnappiPortType.IPInterface, peer_port=port['peer_port'] @@ -777,6 +847,7 @@ def get_multidut_tgen_peer_port_set(line_card_choice, ports, config_set, number_ """ linecards = {} try: + from itertools import product from itertools import izip_longest as zip_longest except ImportError: from itertools import zip_longest @@ -832,8 +903,9 @@ def get_multidut_tgen_peer_port_set(line_card_choice, ports, config_set, number_ elif line_card_choice in ['chassis_multi_line_card_multi_asic']: # Different line card and minimum one port from different asic number if len(linecards.keys()) >= 2: - host_asic = list(zip(config_set[line_card_choice]['hostname'], config_set[line_card_choice]['asic'])) - peer_ports = list(zip_longest(*[linecards[host][asic] for host, asic in host_asic])) + host_asic = list(product(config_set[line_card_choice]['hostname'], config_set[line_card_choice]['asic'])) + peer_ports = list(zip_longest(*[linecards[host][asic] + for host, asic in host_asic if asic in linecards[host]])) peer_ports = [item for sublist in peer_ports for item in sublist] peer_ports = list(filter(None, peer_ports)) return peer_ports[:number_of_tgen_peer_ports] @@ -873,7 +945,7 @@ def cleanup_config(duthost_list, snappi_ports): port_count = len(snappi_ports) dutIps = create_ip_list(dut_ip_start, port_count, mask=prefix_length) for port in snappi_ports: - if port['peer_device'] == duthost.hostname: + if port['peer_device'] == duthost.hostname and port['intf_config_changed']: port_id = port['port_id'] dutIp = dutIps[port_id] logger.info('Removing Configuration on Dut: {} with port {} with ip :{}/{}'.format( @@ -892,3 +964,346 @@ def cleanup_config(duthost_list, snappi_ports): port['peer_port'], dutIp, prefix_length)) + port['intf_config_changed'] = False + + +def pre_configure_dut_interface(duthost, snappi_ports): + """ + Populate tgen ports info of T0 testbed and returns as a list + Args: + duthost (pytest fixture): duthost fixture + snappi_ports: list of snappi ports + """ + + dutIps = create_ip_list(dut_ip_start, len(snappi_ports), mask=prefix_length) + tgenIps = create_ip_list(snappi_ip_start, len(snappi_ports), mask=prefix_length) + dutv6Ips = create_ip_list(dut_ipv6_start, len(snappi_ports), mask=v6_prefix_length) + tgenv6Ips = create_ip_list(snappi_ipv6_start, len(snappi_ports), mask=v6_prefix_length) + snappi_ports_dut = [] + for port in snappi_ports: + if port['peer_device'] == duthost.hostname: + snappi_ports_dut.append(port) + + for port in snappi_ports_dut: + port_id = int(port['port_id'])-1 + port['peer_ip'] = dutIps[port_id] + port['prefix'] = prefix_length + port['ip'] = tgenIps[port_id] + port['peer_ipv6'] = dutv6Ips[port_id] + port['ipv6_prefix'] = v6_prefix_length + port['ipv6'] = tgenv6Ips[port_id] + try: + logger.info('Pre-Configuring Dut: {} with port {} with IP {}/{}'.format( + duthost.hostname, + port['peer_port'], + dutIps[port_id], + prefix_length)) + duthost.command('sudo config interface ip add {} {}/{} \n' .format( + port['peer_port'], + dutIps[port_id], + prefix_length)) + logger.info('Pre-Configuring Dut: {} with port {} with IPv6 {}/{}'.format( + duthost.hostname, + port['peer_port'], + dutv6Ips[port_id], + v6_prefix_length)) + duthost.command('sudo config interface ip add {} {}/{} \n' .format( + port['peer_port'], + dutv6Ips[port_id], + v6_prefix_length)) + except Exception: + pytest_assert(False, "Unable to configure ip on the interface {}".format(port['peer_port'])) + return snappi_ports_dut + + +@pytest.fixture(scope="module") +def multidut_snappi_ports_for_bgp(duthosts, # noqa: F811 + tbinfo, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut): # noqa: F811 + """ + Populate snappi ports and connected DUT ports info of T1 and T2 testbed and returns as a list + Args: + duthost (pytest fixture): duthost fixture + tbinfo (pytest fixture): fixture provides information about testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + Return: + return tuple of duts and snappi ports + """ + speed_type = {'50000': 'speed_50_gbps', + '100000': 'speed_100_gbps', + '200000': 'speed_200_gbps', + '400000': 'speed_400_gbps'} + multidut_snappi_ports = [] + + for duthost in duthosts: + snappi_fanout = get_peer_snappi_chassis(conn_data=conn_graph_facts, + dut_hostname=duthost.hostname) + if snappi_fanout is None: + continue + snappi_fanout_id = list(fanout_graph_facts_multidut.keys()).index(snappi_fanout) + snappi_fanout_list = SnappiFanoutManager(fanout_graph_facts_multidut) + snappi_fanout_list.get_fanout_device_details(device_number=snappi_fanout_id) + snappi_ports = snappi_fanout_list.get_ports(peer_device=duthost.hostname) + port_speed = None + for i in range(len(snappi_ports)): + if port_speed is None: + port_speed = int(snappi_ports[i]['speed']) + + elif port_speed != int(snappi_ports[i]['speed']): + """ All the ports should have the same bandwidth """ + return None + + for port in snappi_ports: + port['location'] = get_snappi_port_location(port) + port['speed'] = speed_type[port['speed']] + port['api_server_ip'] = tbinfo['ptf_ip'] + multidut_snappi_ports = multidut_snappi_ports + snappi_ports + return multidut_snappi_ports + + +@pytest.fixture(scope="module") +def get_snappi_ports_single_dut(duthosts, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts, # noqa: F811 + tbinfo, + snappi_api_serv_ip, + rand_one_dut_hostname, + rand_one_dut_portname_oper_up + ): # noqa: F811 + speed_type = { + '10000': 'speed_10_gbps', + '25000': 'speed_25_gbps', + '40000': 'speed_40_gbps', + '50000': 'speed_50_gbps', + '100000': 'speed_100_gbps', + '200000': 'speed_200_gbps', + '400000': 'speed_400_gbps', + '800000': 'speed_800_gbps'} + + if is_snappi_multidut(duthosts): + return [] + + duthost = duthosts[rand_one_dut_hostname] + + dut_hostname, dut_port = rand_one_dut_portname_oper_up.split('|') + pytest_require(rand_one_dut_hostname == dut_hostname, + "Port is not mapped to the expected DUT") + + """ Generate L1 config """ + snappi_fanout = get_peer_snappi_chassis(conn_data=conn_graph_facts, + dut_hostname=duthost.hostname) + + pytest_assert(snappi_fanout is not None, 'Fail to get snappi_fanout') + + snappi_fanout_id = list(fanout_graph_facts.keys()).index(snappi_fanout) + snappi_fanout_list = SnappiFanoutManager(fanout_graph_facts) + snappi_fanout_list.get_fanout_device_details(device_number=snappi_fanout_id) + + snappi_ports = snappi_fanout_list.get_ports(peer_device=duthost.hostname) + + rx_ports = [] + tx_ports = [] + for port in snappi_ports: + port['intf_config_changed'] = False + port['location'] = get_snappi_port_location(port) + port['speed'] = port['speed'] + port['api_server_ip'] = tbinfo['ptf_ip'] + port['asic_type'] = duthost.facts["asic_type"] + port['duthost'] = duthost + port['snappi_speed_type'] = speed_type[port['speed']] + if duthost.facts["num_asic"] > 1: + port['asic_value'] = duthost.get_port_asic_instance(port['peer_port']).namespace + else: + port['asic_value'] = None + # convert to RX ports first, tx ports later to be consistent with multi-dut + if port['peer_port'] == dut_port: + rx_ports.append(port) + else: + tx_ports.append(port) + return rx_ports + tx_ports + + +@pytest.fixture(scope="module") +def get_snappi_ports_multi_dut(duthosts, # noqa: F811 + tbinfo, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, + ): # noqa: F811 + """ + Populate snappi ports and connected DUT ports info of T1 and T2 testbed and returns as a list + Args: + duthost (pytest fixture): duthost fixture + tbinfo (pytest fixture): fixture provides information about testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + Return: (list) + [{ 'api_server_ip': '10.36.78.59', + 'asic_type': 'broadcom', + 'asic_value': None, + 'card_id': '4', + 'duthost': , + 'ip': '10.36.78.53', + 'location': '10.36.78.53;4;7', + 'peer_device': 'sonic-s6100-dut1', + 'peer_port': 'Ethernet72', + 'port_id': '7', + 'snappi_speed_type': 'speed_100_gbps', + 'speed': '100000' + }, + { 'api_server_ip': '10.36.78.59', + 'asic_type': 'broadcom', + 'asic_value': 'asic0', + 'card_id': '4', + 'duthost': , + 'ip': '10.36.78.53', + 'location': '10.36.78.53;4;8', + 'peer_device': 'sonic-s6100-dut2', + 'peer_port': 'Ethernet76', + 'port_id': '8', + 'snappi_speed_type': 'speed_100_gbps', + 'speed': '100000' + }] + """ + speed_type = { + '10000': 'speed_10_gbps', + '25000': 'speed_25_gbps', + '40000': 'speed_40_gbps', + '50000': 'speed_50_gbps', + '100000': 'speed_100_gbps', + '200000': 'speed_200_gbps', + '400000': 'speed_400_gbps', + '800000': 'speed_800_gbps'} + multidut_snappi_ports = [] + + if not is_snappi_multidut(duthosts): + return [] + + for duthost in duthosts: + snappi_fanout = get_peer_snappi_chassis(conn_data=conn_graph_facts, + dut_hostname=duthost.hostname) + if snappi_fanout is None: + continue + snappi_fanout_id = list(fanout_graph_facts_multidut.keys()).index(snappi_fanout) + snappi_fanout_list = SnappiFanoutManager(fanout_graph_facts_multidut) + snappi_fanout_list.get_fanout_device_details(device_number=snappi_fanout_id) + snappi_ports = snappi_fanout_list.get_ports(peer_device=duthost.hostname) + port_speed = None + for i in range(len(snappi_ports)): + if port_speed is None: + port_speed = int(snappi_ports[i]['speed']) + + elif port_speed != int(snappi_ports[i]['speed']): + """ All the ports should have the same bandwidth """ + return None + + for port in snappi_ports: + port['intf_config_changed'] = False + port['location'] = get_snappi_port_location(port) + port['speed'] = port['speed'] + port['api_server_ip'] = tbinfo['ptf_ip'] + port['asic_type'] = duthost.facts["asic_type"] + port['duthost'] = duthost + port['snappi_speed_type'] = speed_type[port['speed']] + if duthost.facts["num_asic"] > 1: + port['asic_value'] = duthost.get_port_asic_instance(port['peer_port']).namespace + else: + port['asic_value'] = None + multidut_snappi_ports = multidut_snappi_ports + snappi_ports + return multidut_snappi_ports + + +def is_snappi_multidut(duthosts): + if duthosts is None or len(duthosts) == 0: + return False + + return duthosts[0].get_facts().get("modular_chassis") + + +@pytest.fixture(scope="module") +def get_snappi_ports(duthosts, request): + """ + Returns the snappi port info based on the testbed type + Args: + duthosts (pytest fixture): list of DUTs + request (pytest fixture): request fixture + Return: (list) + """ + # call the fixture based on the testbed type for minimize the impact + # use the same fixture for different testbeds in the future if possible? + if is_snappi_multidut(duthosts): + snappi_ports = request.getfixturevalue("get_snappi_ports_multi_dut") + else: + snappi_ports = request.getfixturevalue("get_snappi_ports_single_dut") + return snappi_ports + + +def get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, tx_port_count, rx_port_count, testbed): + """ + Returns the required tx and rx ports for the rdma test + Args: + snappi_port_list (list): List of snappi ports and connected DUT ports info of T1 and T2 testbed + rdma_ports (dict): RDMA port info for testbed subtype defined in variables.py + tx_port_count (int): Number of Tx ports required for the test + rx_port_count (int): Number of Rx ports required for the test + Return: (list) + """ + tx_snappi_ports = [] + rx_snappi_ports = [] + var_tx_ports = random.sample(rdma_ports['tx_ports'], tx_port_count) + var_rx_ports = random.sample(rdma_ports['rx_ports'], rx_port_count) + for port in snappi_port_list: + for var_rx_port in var_rx_ports: + if port['peer_port'] == var_rx_port['port_name'] and port['peer_device'] == var_rx_port['hostname']: + rx_snappi_ports.append(port) + for var_tx_port in var_tx_ports: + if port['peer_port'] == var_tx_port['port_name'] and port['peer_device'] == var_tx_port['hostname']: + tx_snappi_ports.append(port) + + pytest_assert(len(rx_snappi_ports) == rx_port_count, + 'Rx Ports for {} in MULTIDUT_PORT_INFO doesn\'t match with ansible/files/*links.csv'.format(testbed)) + pytest_assert(len(tx_snappi_ports) == tx_port_count, + 'Tx Ports for {} in MULTIDUT_PORT_INFO doesn\'t match with ansible/files/*links.csv'.format(testbed)) + + multidut_snappi_ports = rx_snappi_ports + tx_snappi_ports + return multidut_snappi_ports + + +def clear_fabric_counters(duthost): + """ + Clears the fabric counters for the duthost based on broadcom-DNX platform. + Args: + duthost(obj): dut host object + Returns: + None + """ + if "platform_asic" in duthost.facts and duthost.facts["platform_asic"] == "broadcom-dnx": + logger.info('Clearing fabric counters for DUT:{}'.format(duthost.hostname)) + duthost.shell('sonic-clear fabriccountersport \n') + time.sleep(1) + + +def check_fabric_counters(duthost): + """ + Check for the fabric counters for the duthost based on broadcom-DNX platform. + Test assert if the value of CRC, and FEC_UNCORRECTABLE. + Args: + duthost(obj): dut host object + Returns: + None + """ + if "platform_asic" in duthost.facts and duthost.facts["platform_asic"] == "broadcom-dnx": + raw_out = duthost.shell("show fabric counters port | grep -Ev 'ASIC|---|down'")['stdout'] + logger.info('Verifying fabric counters for DUT:{}'.format(duthost.hostname)) + for line in raw_out.split('\n'): + # Checking if the port is UP. + if 'up' in line: + val_list = line.split() + crc_errors = int(val_list[7].replace(',', '')) + fec_uncor_err = int(val_list[9].replace(',', '')) + # Assert if CRC or FEC uncorrected errors are non-zero. + pytest_assert(crc_errors == 0, 'CRC errors:{} for DUT:{}, ASIC:{}, Port:{}'. + format(crc_errors, duthost.hostname, val_list[0], val_list[1])) + pytest_assert(fec_uncor_err == 0, 'Forward Uncorrectable errors:{} for DUT:{}, ASIC:{}, Port:{}'. + format(fec_uncor_err, duthost.hostname, val_list[0], val_list[1])) diff --git a/tests/common/snappi_tests/snappi_helpers.py b/tests/common/snappi_tests/snappi_helpers.py index 32d3279cdf..d4287967de 100644 --- a/tests/common/snappi_tests/snappi_helpers.py +++ b/tests/common/snappi_tests/snappi_helpers.py @@ -80,7 +80,9 @@ def __init__(self, fanout_data): self.ip_address = '0.0.0.0' for fanout in list(fanout_data.keys()): - self.fanout_list.append(fanout_data[fanout]) + # only snappi fanout, skip other fanout type(eos, sonic, etc.) + if fanout_data[fanout]['device_info']['HwSku'] in ('SNAPPI-tester', 'IXIA-tester'): + self.fanout_list.append(fanout_data[fanout]) def __parse_fanout_connections__(self): device_conn = self.last_device_connection_details @@ -94,7 +96,7 @@ def __parse_fanout_connections__(self): format(self.ip_address, fanout_port, peer_port, peer_device, speed) retval.append(string) - return(retval) + return (retval) def get_fanout_device_details(self, device_number): """With the help of this function you can select the chassis you want @@ -145,7 +147,7 @@ def get_connection_details(self): Returns: Details of the chassis connection as dictionary format. """ - return(self.last_device_connection_details) + return (self.last_device_connection_details) def get_chassis_ip(self): """This function returns IP address of a particular chassis @@ -171,6 +173,9 @@ def get_ports(self, peer_device=None): (first) chassis remains selected. If you do not specify peer_device, this function will return all the ports of the chassis. + For Breakout ports in single appliance , the ports will be in dotted notation, + for example: in links.csv file the numbering will be like Port1.1, Port1.2 + Args: peer_device (str): hostname of the peer device @@ -180,17 +185,29 @@ def get_ports(self, peer_device=None): retval = [] for port in self.current_snappi_port_list: info_list = port.split('/') - dict_element = { - 'ip': info_list[0], - 'card_id': info_list[1].replace('Card', ''), - 'port_id': info_list[2].replace('Port', ''), - 'peer_port': info_list[3], - 'peer_device': info_list[4], - 'speed': info_list[5] - } + if 'Card' in port: + dict_element = { + 'ip': info_list[0], + 'card_id': info_list[1].replace('Card', ''), + 'port_id': info_list[2].replace('Port', ''), + 'peer_port': info_list[3], + 'peer_device': info_list[4], + 'speed': info_list[5] + } - if peer_device is None or info_list[4] == peer_device: - retval.append(dict_element) + if peer_device is None or info_list[4] == peer_device: + retval.append(dict_element) + else: + dict_element = { + 'ip': info_list[0], + 'port_id': info_list[1].replace('Port', ''), + 'peer_port': info_list[2], + 'peer_device': info_list[3], + 'speed': info_list[4] + } + + if peer_device is None or info_list[3] == peer_device: + retval.append(dict_element) return retval @@ -214,10 +231,12 @@ def get_snappi_port_location(intf): Returns: location in string format. Example: '10.36.78.5;1;2' where 1 is card_id and 2 is port_id. """ - keys = set(['ip', 'card_id', 'port_id']) - pytest_assert(keys.issubset(set(intf.keys())), "intf does not have all the keys") - - return "{};{};{}".format(intf['ip'], intf['card_id'], intf['port_id']) + if 'card_id' in intf.keys(): + keys = set(['ip', 'card_id', 'port_id']) + pytest_assert(keys.issubset(set(intf.keys())), "intf does not have all the keys") + return "{};{};{}".format(intf['ip'], intf['card_id'], intf['port_id']) + else: + return "{}/{}".format(intf['ip'], intf['port_id']) def get_dut_port_id(dut_hostname, dut_port, conn_data, fanout_data): diff --git a/tests/common/snappi_tests/traffic_generation.py b/tests/common/snappi_tests/traffic_generation.py index 73c211739c..15951ac1f4 100644 --- a/tests/common/snappi_tests/traffic_generation.py +++ b/tests/common/snappi_tests/traffic_generation.py @@ -1,7 +1,6 @@ """ This module allows various snappi based tests to generate various traffic configurations. """ -import math import time import logging from tests.common.helpers.assertions import pytest_assert @@ -11,6 +10,7 @@ traffic_flow_mode from tests.common.snappi_tests.port import select_ports, select_tx_port from tests.common.snappi_tests.snappi_helpers import wait_for_arp, fetch_snappi_flow_metrics +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) @@ -119,7 +119,10 @@ def generate_test_flows(testbed_config, eth, ipv4 = test_flow.packet.ethernet().ipv4() eth.src.value = base_flow_config["tx_mac"] eth.dst.value = base_flow_config["rx_mac"] - eth.pfc_queue.value = prio + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = prio + else: + eth.pfc_queue.value = pfcQueueValueDict[prio] ipv4.src.value = base_flow_config["tx_port_config"].ip ipv4.dst.value = base_flow_config["rx_port_config"].ip @@ -184,7 +187,10 @@ def generate_background_flows(testbed_config, eth, ipv4 = bg_flow.packet.ethernet().ipv4() eth.src.value = base_flow_config["tx_mac"] eth.dst.value = base_flow_config["rx_mac"] - eth.pfc_queue.value = prio + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = prio + else: + eth.pfc_queue.value = pfcQueueValueDict[prio] ipv4.src.value = base_flow_config["tx_port_config"].ip ipv4.dst.value = base_flow_config["rx_port_config"].ip @@ -520,7 +526,7 @@ def verify_basic_test_flow(flow_metrics, def verify_in_flight_buffer_pkts(duthost, flow_metrics, - snappi_extra_params): + snappi_extra_params, asic_value=None): """ Verify in-flight TX bytes of test flows should be held by switch buffer unless PFC delay is applied for when test traffic is expected to be paused @@ -554,7 +560,7 @@ def verify_in_flight_buffer_pkts(duthost, for peer_port, prios in dut_port_config[0].items(): for prio in prios: - dropped_packets = get_pg_dropped_packets(duthost, peer_port, prio) + dropped_packets = get_pg_dropped_packets(duthost, peer_port, prio, asic_value) pytest_assert(dropped_packets > 0, "Total TX dropped packets {} should be more than 0". format(dropped_packets)) @@ -565,13 +571,14 @@ def verify_in_flight_buffer_pkts(duthost, for peer_port, prios in dut_port_config[0].items(): for prio in prios: - dropped_packets = get_pg_dropped_packets(duthost, peer_port, prio) + dropped_packets = get_pg_dropped_packets(duthost, peer_port, prio, asic_value) pytest_assert(dropped_packets == 0, "Total TX dropped packets {} should be 0". format(dropped_packets)) -def verify_pause_frame_count_dut(duthost, +def verify_pause_frame_count_dut(rx_dut, + tx_dut, test_traffic_pause, global_pause, snappi_extra_params): @@ -580,7 +587,8 @@ def verify_pause_frame_count_dut(duthost, on the DUT Args: - duthost (obj): DUT host object + rx_dut (obj): Ingress DUT host object receiving packets from IXIA transmitter. + tx_dut (obj): Egress DUT host object sending packets to IXIA, hence also receiving PFCs from IXIA. test_traffic_pause (bool): whether test traffic is expected to be paused global_pause (bool): if pause frame is IEEE 802.3X pause i.e. global pause applied snappi_extra_params (SnappiTestParams obj): additional parameters for Snappi traffic @@ -592,7 +600,7 @@ def verify_pause_frame_count_dut(duthost, for peer_port, prios in dut_port_config[1].items(): # PFC pause frames received on DUT's egress port for prio in prios: - pfc_pause_rx_frames = get_pfc_frame_count(duthost, peer_port, prio, is_tx=False) + pfc_pause_rx_frames = get_pfc_frame_count(tx_dut, peer_port, prio, is_tx=False) # For now, all PFC pause test cases send out PFC pause frames from the TGEN RX port to the DUT TX port, # except the case with global pause frames which SONiC does not count currently if global_pause: @@ -609,7 +617,7 @@ def verify_pause_frame_count_dut(duthost, for peer_port, prios in dut_port_config[0].items(): # PFC pause frames sent by DUT's ingress port to TGEN for prio in prios: - pfc_pause_tx_frames = get_pfc_frame_count(duthost, peer_port, prio, is_tx=True) + pfc_pause_tx_frames = get_pfc_frame_count(rx_dut, peer_port, prio, is_tx=True) if test_traffic_pause: pytest_assert(pfc_pause_tx_frames > 0, "PFC pause frames should be transmitted and counted in TX PFC counters for priority {}" @@ -761,80 +769,3 @@ def verify_egress_queue_frame_count(duthost, total_egress_packets, _ = get_egress_queue_count(duthost, peer_port, prios[prio]) pytest_assert(total_egress_packets == test_tx_frames[prio], "Queue counters should increment for invalid PFC pause frames") - - -def verify_m2o_oversubscribtion_results(duthost, - rows, - test_flow_name, - bg_flow_name, - rx_port, - rx_frame_count_deviation, - flag): - """ - Verify if we get expected experiment results - - Args: - duthost (obj): DUT host object - rows (list): per-flow statistics - test_flow_name (str): name of test flows - bg_flow_name (str): name of background flows - rx_port: Rx port of the dut - rx_frame_count_deviation (float): deviation for rx frame count (default to 1%) - flag (dict): Comprises of flow name and its loss criteria ,loss criteria value can be integer values - of string type for definite results or 'continuing' for non definite loss value results - example:{ - 'Test Flow 1 -> 0 Rate:40': { - 'loss': '0' - }, - 'PFC Pause': { - 'loss': '100' - }, - 'Background Flow 1 -> 0 Rate:20': { - 'loss': '5' - }, - 'Background Flow 2 -> 0 Rate:40': { - 'loss': 'continuing' - }, - } - - Returns: - N/A - """ - - sum_rx = 0 - for flow_type, criteria in flag.items(): - for row in rows: - tx_frames = row.frames_tx - rx_frames = row.frames_rx - if flow_type in row.name: - try: - if isinstance(criteria, dict) and isinstance(criteria['loss'], str) and int(criteria['loss']) == 0: - logger.info('{}, TX Frames:{}, RX Frames:{}'.format(row.name, tx_frames, rx_frames)) - pytest_assert(tx_frames == rx_frames, - '{} should not have any dropped packet'.format(row.name)) - pytest_assert(row.loss == 0, - '{} should not have traffic loss'.format(row.name)) - elif (isinstance(criteria, dict) and isinstance(criteria['loss'], str) - and int(criteria['loss']) != 0): - pytest_assert(math.ceil(float(row.loss)) == float(flag[flow_type]['loss']) or - math.floor(float(row.loss)) == float(flag[flow_type]['loss']), - '{} should have traffic loss close to {} percent but got {}'. - format(row.name, float(flag[flow_type]['loss']), float(row.loss))) - else: - pytest_assert(False, 'Wrong criteria given in flag, accepted values are of type \ - string for loss criteria') - except Exception: - if (isinstance(criteria, dict) and isinstance(criteria['loss'], str) - and criteria['loss'] == 'continuing'): - pytest_assert(int(row.loss) > 0, "{} should have continuing traffic loss greater than 0". - format(row.name)) - else: - pytest_assert(False, 'Wrong criteria given in flag, accepted values are of type \ - string for loss criteria') - sum_rx += int(row.frames_rx) - - tx_frames = get_tx_frame_count(duthost, rx_port['peer_port'])[0] - pytest_assert(abs(sum_rx - tx_frames)/sum_rx <= rx_frame_count_deviation, - "FAIL: DUT counters doesn't match with the total frames received on Rx port, \ - Deviation of more than {} observed".format(rx_frame_count_deviation)) - logger.info("PASS: DUT counters match with the total frames received on Rx port") diff --git a/tests/common/str_utils.py b/tests/common/str_utils.py new file mode 100644 index 0000000000..1d54bf6b26 --- /dev/null +++ b/tests/common/str_utils.py @@ -0,0 +1,6 @@ +def str2bool(s): + if isinstance(s, bool): + return s + if s is None: + return False + return s.lower() in ['yes', 'true', 't', 'y', '1'] diff --git a/tests/common/templates/golden_config_db.j2 b/tests/common/templates/golden_config_db.j2 new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/tests/common/templates/golden_config_db.j2 @@ -0,0 +1 @@ +{} diff --git a/tests/common/templates/pfc_storm_eos.j2 b/tests/common/templates/pfc_storm_eos.j2 index f79561c745..5adc52158d 100644 --- a/tests/common/templates/pfc_storm_eos.j2 +++ b/tests/common/templates/pfc_storm_eos.j2 @@ -1,9 +1,9 @@ bash cd /mnt/flash {% if (pfc_asym is defined) and (pfc_asym == True) %} -{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} & +{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} {% if pfc_gen_multiprocess is defined %}-m {% endif %}-p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} & {% else %} -{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}} & +{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} {% if pfc_gen_multiprocess is defined %}-m {% endif %}-p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}} & {% endif %} exit exit diff --git a/tests/common/templates/pfc_storm_mlnx_sonic.j2 b/tests/common/templates/pfc_storm_mlnx_sonic.j2 new file mode 100644 index 0000000000..3a98ef1414 --- /dev/null +++ b/tests/common/templates/pfc_storm_mlnx_sonic.j2 @@ -0,0 +1,5 @@ +{% if (pfc_asym is defined) and (pfc_asym == True) %} +docker exec syncd /bin/bash -c "{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} python /root/{{pfc_gen_file}} -p {{pfc_queue_index}} -n {{pfc_frames_number}} -i {{pfc_fanout_interface}} -l {{pfc_fanout_label_port}}" > /dev/null 2>&1 +{% else %} +docker exec syncd /bin/bash -c "{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} python /root/{{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -n {{pfc_frames_number}} -s {{send_pfc_frame_interval}} -i {{pfc_fanout_interface}} -l {{pfc_fanout_label_port}} -r {{ansible_eth0_ipv4_addr}}" > /dev/null 2>&1 +{% endif %} diff --git a/tests/common/templates/pfc_storm_onyx.j2 b/tests/common/templates/pfc_storm_onyx.j2 index ec5cdd5f42..f3374110c4 100644 --- a/tests/common/templates/pfc_storm_onyx.j2 +++ b/tests/common/templates/pfc_storm_onyx.j2 @@ -6,9 +6,9 @@ configure terminal docker exec {{ container_name }} /bin/bash cd /root/ {% if (pfc_asym is defined) and (pfc_asym == True) %} -{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} nohup python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "sl1p") | replace("/", "sp")}} & +{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} nohup python {{pfc_gen_file}} -p {{pfc_queue_index}} -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "sl1p") | replace("/", "sp")}} & {% else %} -{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} nohup python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -s {{send_pfc_frame_interval}} -i {{pfc_fanout_interface | replace("ernet 1/", "sl1p") | replace("/", "sp")}} -r {{ansible_eth0_ipv4_addr}} & +{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} nohup python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -n {{pfc_frames_number}} -s {{send_pfc_frame_interval}} -i {{pfc_fanout_interface | replace("ernet 1/", "sl1p") | replace("/", "sp")}} -r {{ansible_eth0_ipv4_addr}} & {% endif %} exit diff --git a/tests/common/templates/pfc_storm_sonic.j2 b/tests/common/templates/pfc_storm_sonic.j2 index a626c77fe6..a2f2e70dce 100644 --- a/tests/common/templates/pfc_storm_sonic.j2 +++ b/tests/common/templates/pfc_storm_sonic.j2 @@ -1,6 +1,6 @@ cd {{pfc_gen_dir}} {% if (pfc_asym is defined) and (pfc_asym == True) %} -nohup sh -c "{% if pfc_storm_defer_time is defined %}sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface}}" > /dev/null 2>&1 & +nohup sh -c "{% if pfc_storm_defer_time is defined %}sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} {% if pfc_gen_multiprocess is defined %}-m {% endif %}-p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface}}" > /dev/null 2>&1 & {% else %} -nohup sh -c "{% if pfc_storm_defer_time is defined %}sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface}} -r {{ansible_eth0_ipv4_addr}}" > /dev/null 2>&1 & +nohup sh -c "{% if pfc_storm_defer_time is defined %}sleep {{pfc_storm_defer_time}} &&{% endif %} sudo python {{pfc_gen_file}} {% if pfc_gen_multiprocess is defined %}-m {% endif %}-p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface}} -r {{ansible_eth0_ipv4_addr}}" > /dev/null 2>&1 & {% endif %} diff --git a/tests/common/templates/pfc_storm_stop_eos.j2 b/tests/common/templates/pfc_storm_stop_eos.j2 index 712e927b7f..a084775829 100644 --- a/tests/common/templates/pfc_storm_stop_eos.j2 +++ b/tests/common/templates/pfc_storm_stop_eos.j2 @@ -1,9 +1,9 @@ bash cd /mnt/flash {% if (pfc_asym is defined) and (pfc_asym == True) %} -{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} sudo pkill -f "sudo python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}}" {{'&' if pfc_storm_stop_defer_time is defined else ''}} +{% if pfc_storm_defer_time is defined %} sleep {{pfc_storm_defer_time}} &&{% endif %} sudo pkill -f "python {{pfc_gen_file}} {% if pfc_gen_multiprocess is defined %}-m {% endif %}-p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}}" {{'&' if pfc_storm_stop_defer_time is defined else ''}} {% else %} -{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f "sudo python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}}" {{'&' if pfc_storm_stop_defer_time is defined else ''}} +{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f "python {{pfc_gen_file}} {% if pfc_gen_multiprocess is defined %}-m {% endif %}-p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("Ethernet", "et") | replace("/", "_")}} -r {{ansible_eth0_ipv4_addr}}" {{'&' if pfc_storm_stop_defer_time is defined else ''}} {% endif %} exit exit diff --git a/tests/common/templates/pfc_storm_stop_mlnx_sonic.j2 b/tests/common/templates/pfc_storm_stop_mlnx_sonic.j2 new file mode 100644 index 0000000000..6ba812bfff --- /dev/null +++ b/tests/common/templates/pfc_storm_stop_mlnx_sonic.j2 @@ -0,0 +1,5 @@ +{% if (pfc_asym is defined) and (pfc_asym == True) %} +docker exec syncd /bin/bash -c "{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} python /root/{{pfc_gen_file}} -d -p {{pfc_queue_index}} -n {{pfc_frames_number}} -i {{pfc_fanout_interface}} -l {{pfc_fanout_label_port}}" > /dev/null 2>&1 +{% else %} +docker exec syncd /bin/bash -c "{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} python /root/{{pfc_gen_file}} -d -p {{(1).__lshift__(pfc_queue_index)}} -n {{pfc_frames_number}} -i {{pfc_fanout_interface}} -l {{pfc_fanout_label_port}} -r {{ansible_eth0_ipv4_addr}}" > /dev/null 2>&1 +{% endif %} diff --git a/tests/common/templates/pfc_storm_stop_onyx.j2 b/tests/common/templates/pfc_storm_stop_onyx.j2 index 98ab0b7c91..47fd461966 100644 --- a/tests/common/templates/pfc_storm_stop_onyx.j2 +++ b/tests/common/templates/pfc_storm_stop_onyx.j2 @@ -6,9 +6,9 @@ docker exec {{ container_name }} /bin/bash cd /root/ {% if (pfc_asym is defined) and (pfc_asym == True) %} -{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} pkill -f "python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "sl1p") | replace("/", "sp")}}" {% if pfc_storm_stop_defer_time is defined %}&{% endif %} +{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} nohup python {{pfc_gen_file}} -d -p {{pfc_queue_index}} -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "sl1p") | replace("/", "sp")}} {% if pfc_storm_stop_defer_time is defined %}&{% endif %} {% else %} -{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} pkill -f "python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -s {{send_pfc_frame_interval}} -i {{pfc_fanout_interface | replace("ernet 1/", "sl1p") | replace("/", "sp")}} -r {{ansible_eth0_ipv4_addr}}" {% if pfc_storm_stop_defer_time is defined %}&{% endif %} +{% if pfc_storm_stop_defer_time is defined %} sleep {{pfc_storm_stop_defer_time}} &&{% endif %} nohup python {{pfc_gen_file}} -d -p {{(1).__lshift__(pfc_queue_index)}} -n {{pfc_frames_number}} -i {{pfc_fanout_interface | replace("ernet 1/", "sl1p") | replace("/", "sp")}} -r {{ansible_eth0_ipv4_addr}} {% if pfc_storm_stop_defer_time is defined %}&{% endif %} {% endif %} exit diff --git a/tests/common/templates/pfc_storm_stop_sonic.j2 b/tests/common/templates/pfc_storm_stop_sonic.j2 index 43c4dc5f99..d4b74fada1 100644 --- a/tests/common/templates/pfc_storm_stop_sonic.j2 +++ b/tests/common/templates/pfc_storm_stop_sonic.j2 @@ -1,6 +1,6 @@ cd {{pfc_gen_dir}} {% if (pfc_asym is defined) and (pfc_asym == True) %} -nohup sh -c "{% if pfc_storm_stop_defer_time is defined %}sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f 'python {{pfc_gen_file}} -p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface}}'" > /dev/null 2>&1 & +nohup sh -c "{% if pfc_storm_stop_defer_time is defined %}sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f 'python {{pfc_gen_file}} {% if pfc_gen_multiprocess is defined %}-m {% endif %}-p {{pfc_queue_index}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface}}'" > /dev/null 2>&1 & {% else %} -nohup sh -c "{% if pfc_storm_stop_defer_time is defined %}sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f 'python {{pfc_gen_file}} -p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface}} -r {{ansible_eth0_ipv4_addr}}'" > /dev/null 2>&1 & +nohup sh -c "{% if pfc_storm_stop_defer_time is defined %}sleep {{pfc_storm_stop_defer_time}} &&{% endif %} sudo pkill -f 'python {{pfc_gen_file}} {% if pfc_gen_multiprocess is defined %}-m {% endif %}-p {{(1).__lshift__(pfc_queue_index)}} -t 65535 -n {{pfc_frames_number}} -i {{pfc_fanout_interface}} -r {{ansible_eth0_ipv4_addr}}'" > /dev/null 2>&1 & {% endif %} diff --git a/tests/common/testbed.py b/tests/common/testbed.py index efb8ab0d74..d226603f51 100644 --- a/tests/common/testbed.py +++ b/tests/common/testbed.py @@ -255,7 +255,7 @@ def _generate_sai_ptf_topo(self, tb_dict): return sai_topo def get_testbed_type(self, topo_name): - pattern = re.compile(r'^(wan|t0|t1|ptf|fullmesh|dualtor|t2|tgen|mgmttor|m0|mc0|mx|dpu)') + pattern = re.compile(r'^(wan|t0|t1|ptf|fullmesh|dualtor|t2|tgen|mgmttor|m0|mc0|mx|dpu|ptp)') match = pattern.match(topo_name) if match is None: logger.warning("Unsupported testbed type - {}".format(topo_name)) diff --git a/tests/common/utilities.py b/tests/common/utilities.py index 80e89eadfa..760e885deb 100644 --- a/tests/common/utilities.py +++ b/tests/common/utilities.py @@ -38,6 +38,16 @@ logger = logging.getLogger(__name__) cache = FactsCache() +LA_START_MARKER_SCRIPT = "scripts/find_la_start_marker.sh" +FIND_SYSLOG_MSG_SCRIPT = "scripts/find_log_msg.sh" +# forced mgmt route priority hardcoded to 32764 in following j2 template: +# https://github.com/sonic-net/sonic-buildimage/blob/master/files/image_config/interfaces/interfaces.j2#L82 +FORCED_MGMT_ROUTE_PRIORITY = 32764 +# Wait 300 seconds because sometime 'interfaces-config' service take 45 seconds to response +# interfaces-config service issue track by: https://github.com/sonic-net/sonic-buildimage/issues/19045 +FILE_CHANGE_TIMEOUT = 300 + +NON_USER_CONFIG_TABLES = ["FLEX_COUNTER_TABLE", "ASIC_SENSORS"] def check_skip_release(duthost, release_list): @@ -758,6 +768,29 @@ def get_plt_reboot_ctrl(duthost, tc_name, reboot_type): return reboot_dict +def pdu_reboot(pdu_controller): + """Power-cycle the DUT by turning off and on the PDU outlets. + + Args: + pdu_controller: PDU controller object implementing the BasePduController interface. + User can acquire pdu_controller object from fixture tests.common.plugins.pdu_controller.pdu_controller + + Returns: True if the PDU reboot is successful, False otherwise. + """ + if not pdu_controller: + logging.warning("pdu_controller is None, skip PDU reboot") + return False + hostname = pdu_controller.dut_hostname + if not pdu_controller.turn_off_outlet(): + logging.error("Turn off the PDU outlets of {} failed".format(hostname)) + return False + time.sleep(10) # sleep 10 second to ensure there is gap between power off and on + if not pdu_controller.turn_on_outlet(): + logging.error("Turn on the PDU outlets of {} failed".format(hostname)) + return False + return True + + def get_image_type(duthost): """get the SONiC image type It might be public/microsoft/...or any other type. @@ -1112,7 +1145,7 @@ def capture_and_check_packet_on_dut( wait_time: the time to wait before stopping the packet capture, default is 1 second """ pcap_save_path = "/tmp/func_capture_and_check_packet_on_dut_%s.pcap" % (str(uuid.uuid4())) - cmd_capture_pkts = "sudo nohup tcpdump --immediate-mode -U -i %s -w %s >/dev/null 2>&1 %s & echo $!" \ + cmd_capture_pkts = "nohup tcpdump --immediate-mode -U -i %s -w %s >/dev/null 2>&1 %s & echo $!" \ % (interface, pcap_save_path, pkts_filter) tcpdump_pid = duthost.shell(cmd_capture_pkts)["stdout"] cmd_check_if_process_running = "ps -p %s | grep %s |grep -v grep | wc -l" % (tcpdump_pid, tcpdump_pid) @@ -1178,3 +1211,150 @@ def get_dut_current_passwd(ipv4_address, ipv6_address, username, passwords): except Exception: _, passwd = _paramiko_ssh(ipv6_address, username, passwords) return passwd + + +def check_msg_in_syslog(duthost, log_msg): + """ + Checks for a given log message after the last start-LogAnalyzer message in syslog + + Args: + duthost: Device under test. + log_msg: Log message to be searched + + Yields: + True if log message is present or returns False + """ + la_output = duthost.script(LA_START_MARKER_SCRIPT)["stdout"] + if not la_output: + return False + else: + output = la_output.replace('\r', '').replace('\n', '') + + try: + log_msg = f"\"{log_msg}\"" + log_output = duthost.script("%s %s %s" % (FIND_SYSLOG_MSG_SCRIPT, output, log_msg)) + if log_output: + return True + else: + return False + except Exception: + return False + + +def increment_ipv4_addr(ipv4_addr, incr=1): + octets = str(ipv4_addr).split('.') + last_octet = int(octets[-1]) + last_octet += incr + octets[-1] = str(last_octet) + + return '.'.join(octets) + + +def increment_ipv6_addr(ipv6_addr, incr=1): + octets = str(ipv6_addr).split(':') + last_octet = octets[-1] + if last_octet == '': + last_octet = '0' + incremented_octet = int(last_octet, 16) + incr + new_octet_str = '{:x}'.format(incremented_octet) + + return ':'.join(octets[:-1]) + ':' + new_octet_str + + +def get_file_hash(duthost, file): + hash = duthost.command("sha1sum {}".format(file))["stdout"] + logger.debug("file hash: {}".format(hash)) + + return hash + + +def get_interface_reload_timestamp(duthost): + timestamp = duthost.command("sudo systemctl show --no-pager interfaces-config" + " -p ExecMainExitTimestamp --value")["stdout"] + logger.info("interfaces config timestamp {}".format(timestamp)) + + return timestamp + + +def wait_for_file_changed(duthost, file, action, *args, **kwargs): + original_hash = get_file_hash(duthost, file) + last_timestamp = get_interface_reload_timestamp(duthost) + + action(*args, **kwargs) + + def hash_and_timestamp_changed(duthost, file): + latest_hash = get_file_hash(duthost, file) + latest_timestamp = get_interface_reload_timestamp(duthost) + return latest_hash != original_hash and latest_timestamp != last_timestamp + + exist = wait_until(FILE_CHANGE_TIMEOUT, 1, 0, hash_and_timestamp_changed, duthost, file) + pytest_assert(exist, "File {} does not change after {} seconds.".format(file, FILE_CHANGE_TIMEOUT)) + + +def backup_config(duthost, config, config_backup): + logger.info("Backup {} to {} on {}".format( + config, config_backup, duthost.hostname)) + duthost.shell("cp {} {}".format(config, config_backup)) + + +def restore_config(duthost, config, config_backup): + logger.info("Restore {} with {} on {}".format( + config, config_backup, duthost.hostname)) + duthost.shell("mv {} {}".format(config_backup, config)) + + +def get_running_config(duthost, asic=None): + ns = "-n " + asic if asic else "" + return json.loads(duthost.shell("sonic-cfggen {} -d --print-data".format(ns))['stdout']) + + +def reload_minigraph_with_golden_config(duthost, json_data, safe_reload=True): + """ + for multi-asic/single-asic devices, we only have 1 golden_config_db.json + """ + from tests.common.config_reload import config_reload + golden_config = "/etc/sonic/golden_config_db.json" + duthost.copy(content=json.dumps(json_data, indent=4), dest=golden_config) + config_reload(duthost, config_source="minigraph", safe_reload=safe_reload, override_config=True) + # Cleanup golden config because some other test or device recover may reload config with golden config + duthost.command('mv {} {}_backup'.format(golden_config, golden_config)) + + +def file_exists_on_dut(duthost, filename): + return duthost.stat(path=filename).get('stat', {}).get('exists', False) + + +def compare_dicts_ignore_list_order(dict1, dict2): + def normalize(data): + if isinstance(data, list): + return set(data) + elif isinstance(data, dict): + return {k: normalize(v) for k, v in data.items()} + else: + return data + + dict1_normalized = normalize(dict1) + dict2_normalized = normalize(dict2) + + return dict1_normalized == dict2_normalized + + +def check_output(output, exp_val1, exp_val2): + pytest_assert(not output['failed'], output['stderr']) + for line in output['stdout_lines']: + fds = line.split(':') + if fds[0] == exp_val1: + pytest_assert(fds[4] == exp_val2) + + +def run_show_features(duthosts, enum_dut_hostname): + """Verify show features command output against CONFIG_DB + """ + duthost = duthosts[enum_dut_hostname] + features_dict, succeeded = duthost.get_feature_status() + pytest_assert(succeeded, "failed to obtain feature status") + for cmd_key, cmd_value in list(features_dict.items()): + redis_value = duthost.shell('/usr/bin/redis-cli -n 4 --raw hget "FEATURE|{}" "state"' + .format(cmd_key), module_ignore_errors=False)['stdout'] + pytest_assert(redis_value.lower() == cmd_value.lower(), + "'{}' is '{}' which does not match with config_db".format(cmd_key, cmd_value)) diff --git a/tests/vxlan/vxlan_ecmp_utils.py b/tests/common/vxlan_ecmp_utils.py similarity index 99% rename from tests/vxlan/vxlan_ecmp_utils.py rename to tests/common/vxlan_ecmp_utils.py index 22b21781a2..fd5f0ef8bd 100644 --- a/tests/vxlan/vxlan_ecmp_utils.py +++ b/tests/common/vxlan_ecmp_utils.py @@ -1,9 +1,6 @@ ''' - The functions used by test_vxlan_ecmp.py. Since there are plans to - seperate the test script to multiple files, we need a common location - for these functions. Usage: - from tests.vxlan.ecmp_utils import Ecmp_Utils + from tests.common.vxlan_ecmp_utils import Ecmp_Utils my_own_ecmp_utils = Ecmp_Utils() my_own_ecmp_utils.create_vxlan_tunnel(...) ''' diff --git a/tests/configlet/test_add_rack.py b/tests/configlet/test_add_rack.py index f87b716ebc..a2b66d716a 100644 --- a/tests/configlet/test_add_rack.py +++ b/tests/configlet/test_add_rack.py @@ -5,7 +5,6 @@ from tests.common.utilities import skip_release from .util.base_test import do_test_add_rack, backup_minigraph, restore_orig_minigraph from .util.helpers import log_info -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology("t1") diff --git a/tests/configlet/util/base_test.py b/tests/configlet/util/base_test.py index 916f221839..60bc302430 100644 --- a/tests/configlet/util/base_test.py +++ b/tests/configlet/util/base_test.py @@ -85,7 +85,8 @@ def init(duthost): for i in [data_dir, orig_db_dir, no_t0_db_dir, clet_db_dir, patch_add_t0_dir, patch_rm_t0_dir, files_dir]: - os.mkdir(i) + if not os.path.exists(i): + os.mkdir(i) init_data["files_dir"] = files_dir init_data["data_dir"] = data_dir diff --git a/tests/configlet/util/common.py b/tests/configlet/util/common.py index b743028304..ab1a15ff69 100755 --- a/tests/configlet/util/common.py +++ b/tests/configlet/util/common.py @@ -99,6 +99,12 @@ def do_pause(secs, msg): "NEIGH_TABLE:eth0", "NEIGH_TABLE_DEL_SET", "ROUTE_TABLE:fe80:", + "ROUTE_TABLE:192.168.0.128/25", + "ROUTE_TABLE:20c0:a800::/64", + "ROUTE_TABLE:2064:100::11", + "ROUTE_TABLE:192.168.0.0/25", + "ROUTE_TABLE:100.1.0.17", + "ROUTE_TABLE:20c0:a800:0:80::/64", "ROUTE_TABLE:FE80:", "TUNNEL_DECAP_TABLE", # BUFFER_PG.*3-4 is an auto created entry by buffermgr @@ -107,10 +113,12 @@ def do_pause(secs, msg): # Diff in TUNNEL_DECAP_TERM_TABLE is expected because router port # is set admin down in the test, which leads to tunnel term change "TUNNEL_DECAP_TERM_TABLE:IPINIP_TUNNEL", - "TUNNEL_DECAP_TERM_TABLE:IPINIP_V6_TUNNEL" + "TUNNEL_DECAP_TERM_TABLE:IPINIP_V6_TUNNEL", + "NEIGH_RESOLVE_TABLE*" }, "keys_skip_val_comp": { "last_up_time", + "last_down_time", "flap_count" } }, diff --git a/tests/conftest.py b/tests/conftest.py index 797ae8e780..b1362c402c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import concurrent.futures import os import glob import json @@ -5,11 +6,15 @@ import getpass import random import re +from concurrent.futures import as_completed import pytest import yaml import jinja2 import copy +import time +import subprocess +import threading from datetime import datetime from ipaddress import ip_interface, IPv4Interface @@ -37,6 +42,7 @@ ) from tests.common.helpers.dut_ports import encode_dut_port_name from tests.common.helpers.dut_utils import encode_dut_and_container_name +from tests.common.plugins.sanity_check import recover_chassis from tests.common.system_utils import docker from tests.common.testbed import TestbedInfo from tests.common.utilities import get_inventory_files @@ -51,15 +57,18 @@ from tests.common.config_reload import config_reload from tests.common.connections.console_host import ConsoleHost from tests.common.helpers.assertions import pytest_assert as pt_assert +from tests.common.helpers.sonic_db import AsicDbCli +from tests.common.helpers.inventory_utils import trim_inventory +from tests.common.utilities import InterruptableThread try: from tests.macsec import MacsecPluginT2, MacsecPluginT0 except ImportError as e: logging.error(e) -from tests.platform_tests.args.advanced_reboot_args import add_advanced_reboot_args -from tests.platform_tests.args.cont_warm_reboot_args import add_cont_warm_reboot_args -from tests.platform_tests.args.normal_reboot_args import add_normal_reboot_args +from tests.common.platform.args.advanced_reboot_args import add_advanced_reboot_args +from tests.common.platform.args.cont_warm_reboot_args import add_cont_warm_reboot_args +from tests.common.platform.args.normal_reboot_args import add_normal_reboot_args from ptf import testutils from ptf.mask import Mask @@ -83,7 +92,8 @@ 'tests.platform_tests.api', 'tests.common.plugins.allure_server', 'tests.common.plugins.conditional_mark', - 'tests.common.plugins.random_seed') + 'tests.common.plugins.random_seed', + 'tests.common.plugins.memory_utilization') def pytest_addoption(parser): @@ -199,6 +209,11 @@ def pytest_addoption(parser): parser.addoption("--public_docker_registry", action="store_true", default=False, help="To use public docker registry for syncd swap, by default is disabled (False)") + ############################## + # ansible inventory option # + ############################## + parser.addoption("--trim_inv", action="store_true", default=False, help="Trim inventory files") + def pytest_configure(config): if config.getoption("enable_macsec"): @@ -210,7 +225,7 @@ def pytest_configure(config): @pytest.fixture(scope="session", autouse=True) -def enhance_inventory(request): +def enhance_inventory(request, tbinfo): """ This fixture is to enhance the capability of parsing the value of pytest cli argument '--inventory'. The pytest-ansible plugin always assumes that the value of cli argument '--inventory' is a single @@ -227,7 +242,12 @@ def enhance_inventory(request): if isinstance(inv_opt, list): return inv_files = [inv_file.strip() for inv_file in inv_opt.split(",")] + + if request.config.getoption("trim_inv"): + trim_inventory(inv_files, tbinfo) + try: + logger.info(f"Inventory file: {inv_files}") setattr(request.config.option, "ansible_inventory", inv_files) except AttributeError: logger.error("Failed to set enhanced 'ansible_inventory' to request.config.option") @@ -436,6 +456,17 @@ def rand_one_dut_front_end_hostname(request): return dut_hostnames[0] +@pytest.fixture(scope="module") +def rand_one_tgen_dut_hostname(request, tbinfo, rand_one_dut_front_end_hostname, rand_one_dut_hostname): + """ + Return the randomly selected duthost for TGEN test cases + """ + # For T2, we need to skip supervisor, only use linecards. + if 't2' in tbinfo['topo']['name']: + return rand_one_dut_front_end_hostname + return rand_one_dut_hostname + + @pytest.fixture(scope="module") def rand_selected_front_end_dut(duthosts, rand_one_dut_front_end_hostname): """ @@ -503,6 +534,8 @@ def localhost(ansible_adhoc): @pytest.fixture(scope="session") def ptfhost(enhance_inventory, ansible_adhoc, tbinfo, duthost, request): + if 'ptp' in tbinfo['topo']['name']: + return None if "ptf_image_name" in tbinfo and "docker-keysight-api-server" in tbinfo["ptf_image_name"]: return None if "ptf" in tbinfo: @@ -553,7 +586,7 @@ def nbrhosts(enhance_inventory, ansible_adhoc, tbinfo, creds, request): """ Shortcut fixture for getting VM host """ - + logger.info("Fixture nbrhosts started") devices = {} if (not tbinfo['vm_base'] and 'tgen' in tbinfo['topo']['name']) or 'ptf' in tbinfo['topo']['name']: logger.info("No VMs exist for this topology: {}".format(tbinfo['topo']['name'])) @@ -567,8 +600,8 @@ def nbrhosts(enhance_inventory, ansible_adhoc, tbinfo, creds, request): logger.info("No VMs exist for this topology: {}".format(tbinfo['topo']['properties']['topology'])) return devices - for k, v in list(tbinfo['topo']['properties']['topology']['VMs'].items()): - vm_name = vm_name_fmt % (vm_base + v['vm_offset']) + def initial_neighbor(neighbor_name, vm_name): + logger.info(f"nbrhosts started: {neighbor_name}_{vm_name}") if neighbor_type == "eos": device = NeighborDevice( { @@ -580,7 +613,7 @@ def nbrhosts(enhance_inventory, ansible_adhoc, tbinfo, creds, request): shell_user=creds['eos_root_user'] if 'eos_root_user' in creds else None, shell_passwd=creds['eos_root_password'] if 'eos_root_password' in creds else None ), - 'conf': tbinfo['topo']['properties']['configuration'][k] + 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] } ) elif neighbor_type == "sonic": @@ -592,7 +625,7 @@ def nbrhosts(enhance_inventory, ansible_adhoc, tbinfo, creds, request): ssh_user=creds['sonic_login'] if 'sonic_login' in creds else None, ssh_passwd=creds['sonic_password'] if 'sonic_password' in creds else None ), - 'conf': tbinfo['topo']['properties']['configuration'][k] + 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] } ) elif neighbor_type == "cisco": @@ -604,12 +637,25 @@ def nbrhosts(enhance_inventory, ansible_adhoc, tbinfo, creds, request): creds['cisco_login'], creds['cisco_password'], ), - 'conf': tbinfo['topo']['properties']['configuration'][k] + 'conf': tbinfo['topo']['properties']['configuration'][neighbor_name] } ) else: - raise ValueError("Unknown neighbor type %s" % (neighbor_type, )) - devices[k] = device + raise ValueError("Unknown neighbor type %s" % (neighbor_type,)) + devices[neighbor_name] = device + logger.info(f"nbrhosts finished: {neighbor_name}_{vm_name}") + + executor = concurrent.futures.ThreadPoolExecutor(max_workers=8) + futures = [] + for neighbor_name, neighbor in list(tbinfo['topo']['properties']['topology']['VMs'].items()): + vm_name = vm_name_fmt % (vm_base + neighbor['vm_offset']) + futures.append(executor.submit(initial_neighbor, neighbor_name, vm_name)) + + for future in as_completed(futures): + # if exception caught in the sub-thread, .result() will raise it in the main thread + _ = future.result() + executor.shutdown(wait=True) + logger.info("Fixture nbrhosts finished") return devices @@ -650,9 +696,9 @@ def fanouthosts(enhance_inventory, ansible_adhoc, conn_graph_facts, creds, dutho elif os_type == 'eos': fanout_user = creds.get('fanout_network_user', None) fanout_password = creds.get('fanout_network_password', None) - elif os_type == 'snappi': - fanout_user = creds.get('fanout_network_user', None) - fanout_password = creds.get('fanout_network_password', None) + elif os_type == 'ixia': + # Skip for ixia device which has no fanout + continue else: # when os is mellanox, not supported pytest.fail("os other than sonic and eos not supported") @@ -705,6 +751,8 @@ def fanouthosts(enhance_inventory, ansible_adhoc, conn_graph_facts, creds, dutho @pytest.fixture(scope="session") def vmhost(enhance_inventory, ansible_adhoc, request, tbinfo): + if 'ptp' in tbinfo['topo']['name']: + return None server = tbinfo["server"] inv_files = get_inventory_files(request) vmhost = get_test_server_host(inv_files, server) @@ -787,6 +835,10 @@ def creds_on_dut(duthost): creds["ansible_altpasswords"] = [] + # If ansible_altpasswords is empty, add ansible_altpassword to it + if len(creds["ansible_altpasswords"]) == 0: + creds["ansible_altpasswords"].append(hostvars["ansible_altpassword"]) + passwords = creds["ansible_altpasswords"] + [creds["sonicadmin_password"]] creds['sonicadmin_password'] = get_dut_current_passwd( duthost.mgmt_ip, @@ -863,6 +915,13 @@ def collect_techsupport_all_duts(request, duthosts): [collect_techsupport_on_dut(request, a_dut) for a_dut in duthosts] +@pytest.fixture +def collect_techsupport_all_nbrs(request, nbrhosts): + yield + if request.config.getoption("neighbor_type") == "sonic": + [collect_techsupport_on_dut(request, nbrhosts[nbrhost]['host']) for nbrhost in nbrhosts] + + @pytest.fixture(scope="session", autouse=True) def tag_test_report(request, pytestconfig, tbinfo, duthost, record_testsuite_property): if not request.config.getoption("--junit-xml"): @@ -1163,7 +1222,7 @@ def get_completeness_level_metadata(request): # if completeness_level is not set or an unknown completeness_level is set # return "thorough" to run all test set if not completeness_level or completeness_level not in ["debug", "basic", "confident", "thorough"]: - return "thorough" + return "debug" return completeness_level @@ -2042,7 +2101,7 @@ def __dut_reload(duts_data, node=None, results=None): node.copy(src=asic_cfg_file, dest='/etc/sonic/config_db{}.json'.format(asic_index), verbose=False) os.remove(asic_cfg_file) - config_reload(node, wait_before_force_reload=300) + config_reload(node, wait_before_force_reload=300, safe_reload=True) def compare_running_config(pre_running_config, cur_running_config): @@ -2057,7 +2116,7 @@ def compare_running_config(pre_running_config, cur_running_config): for key in pre_running_config.keys(): if not compare_running_config(pre_running_config[key], cur_running_config[key]): return False - return True + return True # We only have string in list in running config now, so we can ignore the order of the list. elif type(pre_running_config) is list: if set(pre_running_config) != set(cur_running_config): @@ -2069,7 +2128,11 @@ def compare_running_config(pre_running_config, cur_running_config): @pytest.fixture(scope="module", autouse=True) -def core_dump_and_config_check(duthosts, tbinfo, request): +def core_dump_and_config_check(duthosts, tbinfo, + request, + # make sure the tear down of sanity_check happened after core_dump_and_config_check + sanity_check + ): ''' Check if there are new core dump files and if the running config is modified after the test case running. If so, we will reload the running config after test case running. @@ -2177,6 +2240,7 @@ def core_dump_and_config_check(duthosts, tbinfo, request): # Current skipped keys: # 1. "MUX_LINKMGR|LINK_PROBER" # 2. "MUX_LINKMGR|TIMED_OSCILLATION" + # 3. "LOGGER|linkmgrd" # NOTE: this key is edited by the `run_icmp_responder_session` or `run_icmp_responder` # to account for the lower performance of the ICMP responder/mux simulator compared to # real servers and mux cables. @@ -2186,7 +2250,8 @@ def core_dump_and_config_check(duthosts, tbinfo, request): if "dualtor" in tbinfo["topo"]["name"]: EXCLUDE_CONFIG_KEY_NAMES = [ 'MUX_LINKMGR|LINK_PROBER', - 'MUX_LINKMGR|TIMED_OSCILLATION' + 'MUX_LINKMGR|TIMED_OSCILLATION', + 'LOGGER|linkmgrd' ] else: EXCLUDE_CONFIG_KEY_NAMES = [] @@ -2287,19 +2352,20 @@ def _remove_entry(table_name, key_name, config): } logger.warning("Core dump or config check failed for {}, results: {}" .format(module_name, json.dumps(check_result))) - results = parallel_run(__dut_reload, (), {"duts_data": duts_data}, duthosts, timeout=360) + + is_modular_chassis = duthosts[0].get_facts().get("modular_chassis") + if is_modular_chassis: + results = recover_chassis(duthosts) + else: + results = parallel_run(__dut_reload, (), {"duts_data": duts_data}, duthosts, timeout=360) + logger.debug('Results of dut reload: {}'.format(json.dumps(dict(results)))) else: logger.info("Core dump and config check passed for {}".format(module_name)) if check_result: - items = request.session.items - for item in items: - if item.module.__name__ + ".py" == module_name.split("/")[-1]: - item.user_properties.append(('CustomMsg', json.dumps({'DutChekResult': { - 'core_dump_check_pass': core_dump_check_pass, - 'config_db_check_pass': config_db_check_pass - }}))) + request.config.cache.set("core_dump_check_pass", core_dump_check_pass) + request.config.cache.set("config_db_check_pass", config_db_check_pass) @pytest.fixture(scope="function") @@ -2308,6 +2374,7 @@ def on_exit(): Utility to register callbacks for cleanup. Runs callbacks despite assertion failures. Callbacks are executed in reverse order of registration. ''' + class OnExit(): def __init__(self): self.cbs = [] @@ -2334,6 +2401,27 @@ def add_mgmt_test_mark(duthosts): duthosts.shell("touch %s" % mark_file, module_ignore_errors=True) +@pytest.fixture(scope="module") +def asic_db_dut(request, duthosts, enum_frontend_dut_hostname): + duthost = duthosts[enum_frontend_dut_hostname] + asic_db = AsicDbCli(duthost) + yield asic_db + + +@pytest.fixture(scope="module") +def asic_db_dut_rand(request, duthosts, rand_one_dut_hostname): + duthost = duthosts[rand_one_dut_hostname] + asic_db = AsicDbCli(duthost) + yield asic_db + + +@pytest.fixture(scope="module") +def asic_db_dut_supervisor(request, duthosts, enum_supervisor_dut_hostname): + duthost = duthosts[enum_supervisor_dut_hostname] + asic_db = AsicDbCli(duthost) + yield asic_db + + def verify_packets_any_fixed(test, pkt, ports=[], device_number=0, timeout=None): """ Check that a packet is received on _any_ of the specified ports belonging to @@ -2377,3 +2465,58 @@ def format_failure(port, failure): # HACK: We are using set_do_not_care_scapy but it will be deprecated. if not hasattr(Mask, "set_do_not_care_scapy"): Mask.set_do_not_care_scapy = Mask.set_do_not_care_packet + + +def run_logrotate(duthost, stop_event): + logger.info("Start rotate_syslog on {}".format(duthost)) + while not stop_event.is_set(): + try: + # Run logrotate for rsyslog + duthost.shell("logrotate -f /etc/logrotate.conf", module_ignore_errors=True) + except subprocess.CalledProcessError as e: + logger.error("Error: {}".format(str(e))) + # Wait for 60 seconds before the next rotation + time.sleep(60) + + +@pytest.fixture(scope="function") +def rotate_syslog(duthosts, enum_rand_one_per_hwsku_frontend_hostname): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + stop_event = threading.Event() + thread = InterruptableThread( + target=run_logrotate, + args=(duthost, stop_event,) + ) + thread.daemon = True + thread.start() + + yield + stop_event.set() + try: + if thread.is_alive(): + thread.join(timeout=30) + logger.info("thread {} joined".format(thread)) + except Exception as e: + logger.debug("Exception occurred in thread {}".format(str(e))) + + logger.info("rotate_syslog exit {}".format(thread)) + + +@pytest.fixture(scope="module") +def gnxi_path(ptfhost): + """ + gnxi's location is updated from /gnxi to /root/gnxi + in RP https://github.com/sonic-net/sonic-buildimage/pull/10599. + But old docker-ptf images don't have this update, + test case will fail for these docker-ptf images, + because it should still call /gnxi files. + For avoiding this conflict, check gnxi path before test and set GNXI_PATH to correct value. + Add a new gnxi_path module fixture to make sure to set GNXI_PATH before test. + """ + path_exists = ptfhost.stat(path="/root/gnxi/") + if path_exists["stat"]["exists"] and path_exists["stat"]["isdir"]: + gnxipath = "/root/gnxi/" + else: + gnxipath = "/gnxi/" + return gnxipath diff --git a/tests/console/test_console_availability.py b/tests/console/test_console_availability.py index 17ac468455..941d2401db 100644 --- a/tests/console/test_console_availability.py +++ b/tests/console/test_console_availability.py @@ -3,7 +3,6 @@ import pytest from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology("any"), diff --git a/tests/console/test_console_driver.py b/tests/console/test_console_driver.py index f2e34d3f4d..1733638e39 100644 --- a/tests/console/test_console_driver.py +++ b/tests/console/test_console_driver.py @@ -1,7 +1,6 @@ import pytest from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('any') diff --git a/tests/console/test_console_loopback.py b/tests/console/test_console_loopback.py index d6b107e581..b264f896f9 100644 --- a/tests/console/test_console_loopback.py +++ b/tests/console/test_console_loopback.py @@ -2,7 +2,6 @@ import string import random from tests.common.helpers.console_helper import assert_expect_text, create_ssh_client, ensure_console_session_up -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('any') diff --git a/tests/console/test_console_reversessh.py b/tests/console/test_console_reversessh.py index 674aa2227d..126b089f6c 100644 --- a/tests/console/test_console_reversessh.py +++ b/tests/console/test_console_reversessh.py @@ -3,7 +3,6 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('any') diff --git a/tests/console/test_console_udevrule.py b/tests/console/test_console_udevrule.py index 20d2f6bd3f..a3f1d7012a 100644 --- a/tests/console/test_console_udevrule.py +++ b/tests/console/test_console_udevrule.py @@ -1,7 +1,6 @@ import pytest from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('any') diff --git a/tests/container_checker/test_container_checker.py b/tests/container_checker/test_container_checker.py index 2616f3cb3c..68a6e36498 100644 --- a/tests/container_checker/test_container_checker.py +++ b/tests/container_checker/test_container_checker.py @@ -15,7 +15,6 @@ from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer from tests.common.utilities import wait_until from tests.common.helpers.dut_utils import get_disabled_container_list -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) @@ -239,3 +238,41 @@ def test_container_checker(duthosts, enum_rand_one_per_hwsku_hostname, enum_rand # Wait for 70s to 80s such that Monit has a chance to write alerting message into syslog. logger.info("Sleep '{}'s to wait for the alerting message...".format(sleep_time)) time.sleep(sleep_time) + + +def test_container_checker_telemetry(duthosts, rand_one_dut_hostname): + """Tests the feature of container checker. + + This function will verify container checker for telemetry. + + Args: + duthosts: list of DUTs. + rand_one_dut_hostname: Fixture returning dut hostname. + + Returns: + None. + """ + duthost = duthosts[rand_one_dut_hostname] + container_name = "telemetry" + + # Reload config to restore the container + config_reload(duthost, safe_reload=True) + # Monit needs 300 seconds to start monitoring the container + time.sleep(300) + + # Enable LogAnalyzer + loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="container_checker_{}".format(container_name)) + loganalyzer.expect_regex = get_expected_alerting_message(container_name) + marker = loganalyzer.init() + + # Enable telemetry in FEATURE table + dut_command = "sonic-db-cli CONFIG_DB hset \"FEATURE|{}\" state enabled".format(container_name) + duthost.command(dut_command, module_ignore_errors=True) + + # Monit checks services at 1-minute intervals + # Add a 20-second delay to ensure Monit has time to write alert messages to syslog + sleep_time = 80 + logger.info("Sleep '{}'s to wait for the alerting message...".format(sleep_time)) + time.sleep(sleep_time) + analysis = loganalyzer.analyze(marker, fail=False) + pytest_assert(analysis['total']['expected_match'] == 0, 'Monit error: {}'.format(analysis['expect_messages'])) diff --git a/tests/container_hardening/test_container_hardening.py b/tests/container_hardening/test_container_hardening.py index f7fb593707..fe82feb8ce 100644 --- a/tests/container_hardening/test_container_hardening.py +++ b/tests/container_hardening/test_container_hardening.py @@ -3,7 +3,6 @@ import logging from tests.common.helpers.assertions import pytest_assert, pytest_require from tests.common.helpers.dut_utils import is_container_running, get_disabled_container_list -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('any'), diff --git a/tests/copp/scripts/update_copp_config.py b/tests/copp/scripts/update_copp_config.py index e82f3d3da7..a362a63f8f 100644 --- a/tests/copp/scripts/update_copp_config.py +++ b/tests/copp/scripts/update_copp_config.py @@ -64,6 +64,8 @@ def generate_limited_pps_config(pps_limit, input_config_file, output_config_file config_format (str): The format of the input COPP config file """ + DEFAULT_PPS_LIMIT = "300" + with open(input_config_file) as input_stream: copp_config = json.load(input_stream) @@ -75,19 +77,26 @@ def generate_limited_pps_config(pps_limit, input_config_file, output_config_file raise ValueError("Invalid config format specified") for trap_group in trap_groups: - for _, group_config in list(trap_group.items()): + for tg, group_config in list(trap_group.items()): # Notes: # CIR (committed information rate) - bandwidth limit set by the policer # CBS (committed burst size) - largest burst of packets allowed by the policer # # Setting these two values to pps_limit restricts the policer to allowing exactly # that number of packets per second, which is what we want for our tests. - - if "cir" in group_config: - group_config["cir"] = pps_limit - - if "cbs" in group_config: - group_config["cbs"] = pps_limit + # For default trap, use a different CIR other than 600 to easily identify + # if it is getting hit. For queue4_group3, use the default value in copp + # configuration as this is lower than 600 PPS + if tg == "default": + group_config["cir"] = DEFAULT_PPS_LIMIT + group_config["cbs"] = DEFAULT_PPS_LIMIT + elif tg == "queue4_group3": + continue + else: + if "cir" in group_config: + group_config["cir"] = pps_limit + if "cbs" in group_config: + group_config["cbs"] = pps_limit with open(output_config_file, "w+") as output_stream: json.dump(copp_config, output_stream) diff --git a/tests/copp/test_copp.py b/tests/copp/test_copp.py index d6745885e0..2d575fd209 100644 --- a/tests/copp/test_copp.py +++ b/tests/copp/test_copp.py @@ -37,15 +37,14 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import find_duthost_on_role from tests.common.utilities import get_upstream_neigh_type -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 # Module-level fixtures from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 pytestmark = [ - pytest.mark.topology("t0", "t1", "t2", "m0", "mx"), - pytest.mark.device_type('physical') + pytest.mark.topology("t0", "t1", "t2", "m0", "mx") ] _COPPTestParameters = namedtuple("_COPPTestParameters", @@ -57,7 +56,8 @@ "nn_target_interface", "nn_target_namespace", "send_rate_limit", - "nn_target_vlanid"]) + "nn_target_vlanid", + "topo_type"]) _TOR_ONLY_PROTOCOL = ["DHCP", "DHCP6"] _TEST_RATE_LIMIT_DEFAULT = 600 @@ -84,7 +84,7 @@ class TestCOPP(object): "LLDP", "UDLD"]) def test_policer(self, protocol, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - ptfhost, copp_testbed, dut_type): + ptfhost, copp_testbed, dut_type, skip_traffic_test): # noqa F811 """ Validates that rate-limited COPP groups work as expected. @@ -96,11 +96,13 @@ def test_policer(self, protocol, duthosts, enum_rand_one_per_hwsku_frontend_host ptfhost, protocol, copp_testbed, - dut_type) + dut_type, + skip_traffic_test=skip_traffic_test) @pytest.mark.disable_loganalyzer def test_add_new_trap(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - ptfhost, check_image_version, copp_testbed, dut_type, backup_restore_config_db): + ptfhost, check_image_version, copp_testbed, dut_type, backup_restore_config_db, + skip_traffic_test): # noqa F811 """ Validates that one new trap(bgp) can be installed @@ -123,14 +125,16 @@ def test_add_new_trap(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, self.trap_id.upper(), copp_testbed, dut_type, - has_trap=False) + has_trap=False, + skip_traffic_test=skip_traffic_test) logger.info("Set always_enabled of {} to true".format(self.trap_id)) copp_utils.configure_always_enabled_for_trap(duthost, self.trap_id, "true") logger.info("Verify {} trap status is installed by sending traffic".format(self.trap_id)) pytest_assert( - wait_until(60, 20, 0, _copp_runner, duthost, ptfhost, self.trap_id.upper(), copp_testbed, dut_type), + wait_until(60, 20, 0, _copp_runner, duthost, ptfhost, self.trap_id.upper(), copp_testbed, dut_type, + skip_traffic_test=skip_traffic_test), "Installing {} trap fail".format(self.trap_id)) @pytest.mark.disable_loganalyzer @@ -138,7 +142,7 @@ def test_add_new_trap(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, "disable_feature_status"]) def test_remove_trap(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, check_image_version, copp_testbed, dut_type, - backup_restore_config_db, remove_trap_type): + backup_restore_config_db, remove_trap_type, skip_traffic_test): # noqa F811 """ Validates that The trap(bgp) can be uninstalled after deleting the corresponding entry from the feature table @@ -147,13 +151,16 @@ def test_remove_trap(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, 4. Verify the trap status is uninstalled by sending traffic """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + if (duthost.facts["asic_type"] == "cisco-8000"): + logger.info("Sleep 120 seconds for Cisco platform") + time.sleep(120) if self.trap_id == "bgp": logger.info("Uninstall trap ip2me") copp_utils.uninstall_trap(duthost, "ip2me", "ip2me") logger.info("Pre condition: make trap {} is installed".format(self.feature_name)) - pre_condition_install_trap(ptfhost, duthost, copp_testbed, self.trap_id, self.feature_name) + pre_condition_install_trap(ptfhost, duthost, copp_testbed, self.trap_id, self.feature_name, skip_traffic_test) if remove_trap_type == "delete_feature_entry": logger.info("Remove feature entry: {}".format(self.feature_name)) @@ -165,13 +172,13 @@ def test_remove_trap(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, logger.info("Verify {} trap status is uninstalled by sending traffic".format(self.trap_id)) pytest_assert( wait_until(100, 20, 0, _copp_runner, duthost, ptfhost, self.trap_id.upper(), - copp_testbed, dut_type, has_trap=False), + copp_testbed, dut_type, has_trap=False, skip_traffic_test=skip_traffic_test), "uninstalling {} trap fail".format(self.trap_id)) @pytest.mark.disable_loganalyzer def test_trap_config_save_after_reboot(self, duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, check_image_version, copp_testbed, dut_type, - backup_restore_config_db, request): + backup_restore_config_db, request, skip_traffic_test): # noqa F811 """ Validates that the trap configuration is saved or not after reboot(reboot, fast-reboot, warm-reboot) @@ -200,7 +207,8 @@ def test_trap_config_save_after_reboot(self, duthosts, localhost, enum_rand_one_ copp_utils.verify_always_enable_value(duthost, self.trap_id, "true") logger.info("Verify {} trap status is installed by sending traffic".format(self.trap_id)) pytest_assert( - wait_until(200, 20, 0, _copp_runner, duthost, ptfhost, self.trap_id.upper(), copp_testbed, dut_type), + wait_until(200, 20, 0, _copp_runner, duthost, ptfhost, self.trap_id.upper(), copp_testbed, dut_type, + skip_traffic_test=skip_traffic_test), "Installing {} trap fail".format(self.trap_id)) @@ -271,7 +279,7 @@ def ignore_expected_loganalyzer_exceptions(enum_rand_one_per_hwsku_frontend_host loganalyzer[enum_rand_one_per_hwsku_frontend_hostname].ignore_regex.extend(ignoreRegex) -def _copp_runner(dut, ptf, protocol, test_params, dut_type, has_trap=True): +def _copp_runner(dut, ptf, protocol, test_params, dut_type, has_trap=True, skip_traffic_test=False): # noqa F811 """ Configures and runs the PTF test cases. """ @@ -282,12 +290,17 @@ def _copp_runner(dut, ptf, protocol, test_params, dut_type, has_trap=True): "peerip": test_params.peerip, "send_rate_limit": test_params.send_rate_limit, "has_trap": has_trap, - "hw_sku": dut.facts["hwsku"]} + "hw_sku": dut.facts["hwsku"], + "asic_type": dut.facts["asic_type"], + "topo_type": test_params.topo_type} dut_ip = dut.mgmt_ip device_sockets = ["0-{}@tcp://127.0.0.1:10900".format(test_params.nn_target_port), "1-{}@tcp://{}:10900".format(test_params.nn_target_port, dut_ip)] + if skip_traffic_test is True: + logger.info("Skipping traffic test.") + return True # NOTE: debug_level can actually slow the PTF down enough to fail the test cases # that are not rate limited. Until this is addressed, do not use this flag as part of # nightly test runs. @@ -315,6 +328,7 @@ def _gather_test_params(tbinfo, duthost, request, duts_minigraph_facts): swap_syncd = request.config.getoption("--copp_swap_syncd") send_rate_limit = request.config.getoption("--send_rate_limit") topo = tbinfo["topo"]["name"] + topo_type = tbinfo["topo"]["type"] mg_fact = duts_minigraph_facts[duthost.hostname] port_index_map = {} @@ -364,7 +378,8 @@ def _gather_test_params(tbinfo, duthost, request, duts_minigraph_facts): nn_target_interface=nn_target_interface, nn_target_namespace=nn_target_namespace, send_rate_limit=send_rate_limit, - nn_target_vlanid=nn_target_vlanid) + nn_target_vlanid=nn_target_vlanid, + topo_type=topo_type) def _setup_testbed(dut, creds, ptf, test_params, tbinfo, upStreamDuthost, is_backend_topology): @@ -481,14 +496,15 @@ def backup_restore_config_db(duthosts, enum_rand_one_per_hwsku_frontend_hostname copp_utils.restore_config_db(duthost) -def pre_condition_install_trap(ptfhost, duthost, copp_testbed, trap_id, feature_name): +def pre_condition_install_trap(ptfhost, duthost, copp_testbed, trap_id, feature_name, skip_traffic_test): # noqa F811 copp_utils.install_trap(duthost, feature_name) logger.info("Set always_enabled of {} to false".format(trap_id)) copp_utils.configure_always_enabled_for_trap(duthost, trap_id, "false") logger.info("Verify {} trap status is installed by sending traffic in pre_condition".format(trap_id)) pytest_assert( - wait_until(100, 20, 0, _copp_runner, duthost, ptfhost, trap_id.upper(), copp_testbed, dut_type), + wait_until(100, 20, 0, _copp_runner, duthost, ptfhost, trap_id.upper(), copp_testbed, dut_type, + skip_traffic_test=skip_traffic_test), "Installing {} trap fail".format(trap_id)) diff --git a/tests/crm/conftest.py b/tests/crm/conftest.py index cbae1ef58c..3f3871f890 100755 --- a/tests/crm/conftest.py +++ b/tests/crm/conftest.py @@ -3,12 +3,15 @@ import json import logging import re +import ipaddress -from test_crm import RESTORE_CMDS +from test_crm import RESTORE_CMDS, get_nh_ip from tests.common.helpers.crm import CRM_POLLING_INTERVAL from tests.common.errors import RunAnsibleModuleFail from tests.common.utilities import wait_until, recover_acl_rule from tests.common.platform.interface_utils import parse_intf_status +from tests.common.mellanox_data import is_mellanox_device +from tests.common.helpers.dut_utils import get_sai_sdk_dump_file logger = logging.getLogger(__name__) @@ -91,7 +94,7 @@ def crm_thresholds(duthosts, enum_rand_one_per_hwsku_frontend_hostname): return res -@pytest.fixture(scope="function", autouse=True) +@pytest.fixture(scope="module", autouse=True) def crm_interface(duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo, enum_frontend_asic_index): """ Return tuple of two DUT interfaces """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] @@ -188,17 +191,75 @@ def check_interface_status(duthost, intf_list, expected_oper='up'): return True +def configure_a_route_with_same_prefix_as_vlan_for_mlnx(duthost, asichost, tbinfo, crm_interface): + """ + For mellanox device, the crm available counter is related to LPM tree. + When shutdown all interfaces in vlan (e.g. vlan 1000), + it will cause the route (e.g. 192.168.0.1/21 )for vlan to be removed, we have only one route for the prefix (21), + so after it is removed, the corresponding prefix in LPM tree will be removed too, + which will lead to the LPM tree structure is changed. + LPM tree change might cause available counter change dramatically, + but we cannot estimate how long the change is ready. + Therefore, it will lead the first case of test_crm_route fail occasionally + because the expected available counter is not decreased. + So, we add another route with the same prefix(21) as the vlan's so that the LPM tree is not changed. + """ + # Get NH IP + nh_ip = get_nh_ip(duthost, asichost, crm_interface, '4') + + dump_ip_for_construct_test_route_with_same_prefix_as_vlan_interface = '21.21.21.21' + network_with_same_prefix_as_vlan_interface = str( + ipaddress.IPv4Interface( + f"{dump_ip_for_construct_test_route_with_same_prefix_as_vlan_interface}/" + f"{get_vlan_ipv4_prefix_len(asichost, tbinfo)}").network) + add_route_command = f"sudo ip route add {network_with_same_prefix_as_vlan_interface} via {nh_ip}" + duthost.shell(add_route_command) + assert wait_until(30, 5, 0, check_route_exist, duthost, network_with_same_prefix_as_vlan_interface, nh_ip), \ + f"Failed to add route {network_with_same_prefix_as_vlan_interface} via {nh_ip} " + + # Get sai sdk dump file in case test fail, we can get the LPM tree information + get_sai_sdk_dump_file(duthost, "sai_sdk_dump_before_shutdown_vlan_ports") + + del_dump_route_with_same_prefix_as_vlan_interface_cmd = \ + f" sudo ip route del {network_with_same_prefix_as_vlan_interface} via {nh_ip}" + + return del_dump_route_with_same_prefix_as_vlan_interface_cmd + + +def check_route_exist(duthost, network_with_same_prefix_as_vlan_interface, nh_ip): + route_output = duthost.shell(f"show ip route {network_with_same_prefix_as_vlan_interface}")["stdout"] + return f"Routing entry for {network_with_same_prefix_as_vlan_interface}" in route_output and nh_ip in route_output + + +def get_vlan_ipv4_prefix_len(asichost, tbinfo): + mg_facts = asichost.get_extended_minigraph_facts(tbinfo) + for vlan_port_data in mg_facts["minigraph_vlan_interfaces"]: + if ipaddress.ip_interface(vlan_port_data['addr']).version == 4: + logger.info(f"vlan interface v4 prefix is :{vlan_port_data['prefixlen']}") + return vlan_port_data['prefixlen'] + assert False, "Not find v4 prefix for vlan interface config" + + @pytest.fixture(scope="module", autouse=True) -def shutdown_unnecessary_intf(duthosts, tbinfo, enum_frontend_asic_index, enum_rand_one_per_hwsku_frontend_hostname): +def shutdown_unnecessary_intf( + duthosts, tbinfo, enum_frontend_asic_index, enum_rand_one_per_hwsku_frontend_hostname, crm_interface): """ Shutdown unused interfaces to avoid fdb entry influenced by mac learning """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + asichost = duthost.asic_instance(enum_frontend_asic_index) intfs_connect_with_ptf = get_intf_list(duthost, tbinfo, enum_frontend_asic_index) if intfs_connect_with_ptf: + if is_mellanox_device(duthost): + del_dump_route_with_same_prefix_as_vlan_interface_cmd = configure_a_route_with_same_prefix_as_vlan_for_mlnx( + duthost, asichost, tbinfo, crm_interface) logger.info("Shutdown interfaces: {}".format(intfs_connect_with_ptf)) duthost.shutdown_multiple(intfs_connect_with_ptf) assert wait_until(300, 20, 0, check_interface_status, duthost, intfs_connect_with_ptf, 'down'), \ "All interfaces should be down!" + if is_mellanox_device(duthost): + # Get sai sdk dump file in case test fail, we can get the LPM tree information + get_sai_sdk_dump_file(duthost, "sai_sdk_dump_after_shutdown_vlan_ports") + yield if intfs_connect_with_ptf: @@ -206,6 +267,8 @@ def shutdown_unnecessary_intf(duthosts, tbinfo, enum_frontend_asic_index, enum_r duthost.no_shutdown_multiple(intfs_connect_with_ptf) assert wait_until(300, 20, 0, check_interface_status, duthost, intfs_connect_with_ptf), \ "All interfaces should be up!" + if is_mellanox_device(duthost): + duthost.shell(del_dump_route_with_same_prefix_as_vlan_interface_cmd) @pytest.fixture(scope="module") diff --git a/tests/crm/test_crm.py b/tests/crm/test_crm.py index c3df1d7cf9..f2f91499b7 100755 --- a/tests/crm/test_crm.py +++ b/tests/crm/test_crm.py @@ -17,7 +17,8 @@ from tests.common.fixtures.duthost_utils import disable_route_checker # noqa F401 from tests.common.fixtures.duthost_utils import disable_fdb_aging # noqa F401 from tests.common.utilities import wait_until, get_data_acl -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from tests.common.mellanox_data import is_mellanox_device +from tests.common.helpers.dut_utils import get_sai_sdk_dump_file pytestmark = [ @@ -49,7 +50,7 @@ @pytest.fixture(autouse=True) -def ignore_expected_loganalyzer_exceptions(enum_rand_one_per_hwsku_frontend_hostname, loganalyzer): +def ignore_expected_loganalyzer_exceptions(duthosts, enum_rand_one_per_hwsku_frontend_hostname, loganalyzer): """Ignore expected failures logs during test execution. We don't have control over the order events are received by orchagent, so it is @@ -65,9 +66,15 @@ def ignore_expected_loganalyzer_exceptions(enum_rand_one_per_hwsku_frontend_host ignoreRegex = [ ".*ERR swss#orchagent.*removeVlan: Failed to remove non-empty VLAN.*" ] - + # Ignore in KVM test + KVMIgnoreRegex = [ + ".*flushFdbEntries: failed to find fdb entry in info set.*" + ] if loganalyzer: # Skip if loganalyzer is disabled loganalyzer[enum_rand_one_per_hwsku_frontend_hostname].ignore_regex.extend(ignoreRegex) + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + if duthost.facts["asic_type"] == "vs": + loganalyzer[enum_rand_one_per_hwsku_frontend_hostname].ignore_regex.extend(KVMIgnoreRegex) @pytest.fixture(scope="function") @@ -289,7 +296,11 @@ def get_crm_stats(cmd, duthost): return crm_stats_used, crm_stats_available -def check_crm_stats(cmd, duthost, origin_crm_stats_used, origin_crm_stats_available, oper_used="==", oper_ava="=="): +def check_crm_stats(cmd, duthost, origin_crm_stats_used, origin_crm_stats_available, + oper_used="==", oper_ava="==", skip_stats_check=False): + if skip_stats_check is True: + logger.info("Skip CRM stats check") + return True crm_stats_used, crm_stats_available = get_crm_stats(cmd, duthost) if eval("{} {} {}".format(crm_stats_used, oper_used, origin_crm_stats_used)) and \ eval("{} {} {}".format(crm_stats_available, oper_ava, origin_crm_stats_available)): @@ -450,6 +461,16 @@ def get_crm_resources_fdb_and_ip_route(duthost, asic_ix): return result +def get_nh_ip(duthost, asichost, crm_interface, ip_ver): + # Get NH IP + cmd = "{ip_cmd} -{ip_ver} neigh show dev {crm_intf} nud reachable nud stale \ + | grep -v fe80".format(ip_cmd=asichost.ip_cmd, ip_ver=ip_ver, crm_intf=crm_interface[0]) + out = duthost.shell(cmd) + assert out["stdout"] != "", "Get Next Hop IP failed. Neighbor not found" + nh_ip = [item.split()[0] for item in out["stdout"].split("\n") if "REACHABLE" in item][0] + return nh_ip + + @pytest.mark.usefixtures('disable_route_checker') @pytest.mark.parametrize("ip_ver,route_add_cmd,route_del_cmd", [("4", "{} route add 2.{}.2.0/24 via {}", "{} route del 2.{}.2.0/24 via {}"), @@ -460,6 +481,8 @@ def test_crm_route(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro crm_interface, ip_ver, route_add_cmd, route_del_cmd): duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] asichost = duthost.asic_instance(enum_frontend_asic_index) + asic_type = duthost.facts['asic_type'] + skip_stats_check = True if asic_type == "vs" else False RESTORE_CMDS["crm_threshold_name"] = "ipv{ip_ver}_route".format(ip_ver=ip_ver) # Template used to speedup execution of many similar commands on DUT @@ -490,11 +513,7 @@ def test_crm_route(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro logging.info("crm_stats_route_used {}, crm_stats_route_available {}, crm_stats_fdb_used {}".format( crm_stats_route_used, crm_stats_route_available, crm_stats_fdb_used)) # Get NH IP - cmd = "{ip_cmd} -{ip_ver} neigh show dev {crm_intf} nud reachable nud stale \ - | grep -v fe80".format(ip_cmd=asichost.ip_cmd, ip_ver=ip_ver, crm_intf=crm_interface[0]) - out = duthost.shell(cmd) - pytest_assert(out["stdout"] != "", "Get Next Hop IP failed. Neighbor not found") - nh_ip = [item.split()[0] for item in out["stdout"].split("\n") if "REACHABLE" in item][0] + nh_ip = get_nh_ip(duthost, asichost, crm_interface, ip_ver) # Add IPv[4/6] routes # Cisco platforms need an upward of 64 routes for crm_stats_ipv4_route_available to decrement @@ -511,7 +530,7 @@ def test_crm_route(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro duthost.command(route_add) check_available_counters = True - if duthost.facts['asic_type'] in ['broadcom', 'mellanox']: + if duthost.facts['asic_type'] == 'broadcom': check_available_counters = False # Make sure CRM counters updated @@ -529,16 +548,20 @@ def test_crm_route(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro crm_stats_route_available = get_expected_crm_stats_route_available(crm_stats_route_available, crm_stats_fdb_used, crm_stats_fdb_used_after_add_route) - # Verify "crm_stats_ipv[4/6]_route_used" counter was incremented - if not (new_crm_stats_route_used - crm_stats_route_used == total_routes): - for i in range(total_routes): - RESTORE_CMDS["test_crm_route"].append(route_del_cmd.format(asichost.ip_cmd, i, nh_ip)) - pytest.fail("\"crm_stats_ipv{}_route_used\" counter was not incremented".format(ip_ver)) - # Verify "crm_stats_ipv[4/6]_route_available" counter was decremented - if check_available_counters and not (crm_stats_route_available - new_crm_stats_route_available >= 1): - for i in range(total_routes): - RESTORE_CMDS["test_crm_route"].append(route_del_cmd.format(asichost.ip_cmd, i, nh_ip)) - pytest.fail("\"crm_stats_ipv{}_route_available\" counter was not decremented".format(ip_ver)) + if skip_stats_check is False: + # Verify "crm_stats_ipv[4/6]_route_used" counter was incremented + if not (new_crm_stats_route_used - crm_stats_route_used == total_routes): + for i in range(total_routes): + RESTORE_CMDS["test_crm_route"].append(route_del_cmd.format(asichost.ip_cmd, i, nh_ip)) + pytest.fail("\"crm_stats_ipv{}_route_used\" counter was not incremented".format(ip_ver)) + # Verify "crm_stats_ipv[4/6]_route_available" counter was decremented + if check_available_counters and not (crm_stats_route_available - new_crm_stats_route_available >= 1): + if is_mellanox_device(duthost): + # Get sai sdk dump file in case test fail, we can get the LPM tree information + get_sai_sdk_dump_file(duthost, f"sai_sdk_dump_after_add_v{ip_ver}_router") + for i in range(total_routes): + RESTORE_CMDS["test_crm_route"].append(route_del_cmd.format(asichost.ip_cmd, i, nh_ip)) + pytest.fail("\"crm_stats_ipv{}_route_available\" counter was not decremented".format(ip_ver)) # Remove IPv[4/6] routes for i in range(total_routes): @@ -618,6 +641,8 @@ def test_crm_nexthop(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_frontend_asic_index, crm_interface, ip_ver, nexthop, ptfhost, cleanup_ptf_interface): duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] asichost = duthost.asic_instance(enum_frontend_asic_index) + asic_type = duthost.facts['asic_type'] + skip_stats_check = True if asic_type == "vs" else False RESTORE_CMDS["crm_threshold_name"] = "ipv{ip_ver}_nexthop".format(ip_ver=ip_ver) if duthost.facts["asic_type"] == "marvell": if ip_ver == "4": @@ -660,8 +685,9 @@ def test_crm_nexthop(duthosts, enum_rand_one_per_hwsku_frontend_hostname, logger.info("original crm_stats_nexthop_used is: {}, original crm_stats_nexthop_available is {}".format( crm_stats_nexthop_used, crm_stats_nexthop_available)) - crm_stats_checker = wait_until(60, 5, 0, check_crm_stats, get_nexthop_stats, duthost, crm_stats_nexthop_used + 1, - crm_stats_nexthop_available - 1, ">=", "<=") + crm_stats_checker = wait_until(60, 5, 0, check_crm_stats, get_nexthop_stats, duthost, + crm_stats_nexthop_used + 1, crm_stats_nexthop_available - 1, ">=", "<=", + skip_stats_check=skip_stats_check) if not crm_stats_checker: RESTORE_CMDS["test_crm_nexthop"].append(nexthop_del_cmd) pytest_assert(crm_stats_checker, @@ -673,8 +699,9 @@ def test_crm_nexthop(duthosts, enum_rand_one_per_hwsku_frontend_hostname, asichost.shell(ip_remove_cmd) asichost.sonichost.add_member_to_vlan(1000, 'Ethernet1', is_tagged=False) ptfhost.remove_ip_addresses() - crm_stats_checker = wait_until(60, 5, 0, check_crm_stats, get_nexthop_stats, duthost, crm_stats_nexthop_used, - crm_stats_nexthop_available) + crm_stats_checker = wait_until(60, 5, 0, check_crm_stats, get_nexthop_stats, duthost, + crm_stats_nexthop_used, crm_stats_nexthop_available, + skip_stats_check=skip_stats_check) pytest_assert(crm_stats_checker, "\"crm_stats_ipv{}_nexthop_used\" counter was not decremented or " "\"crm_stats_ipv{}_nexthop_available\" counter was not incremented".format(ip_ver, ip_ver)) @@ -704,6 +731,8 @@ def test_crm_neighbor(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_frontend_asic_index, crm_interface, ip_ver, neighbor, host): duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] asichost = duthost.asic_instance(enum_frontend_asic_index) + asic_type = duthost.facts['asic_type'] + skip_stats_check = True if asic_type == "vs" else False RESTORE_CMDS["crm_threshold_name"] = "ipv{ip_ver}_neighbor".format(ip_ver=ip_ver) neighbor_add_cmd = "{ip_cmd} neigh replace {neighbor} lladdr 11:22:33:44:55:66 dev {iface}"\ .format(ip_cmd=asichost.ip_cmd, neighbor=neighbor, iface=crm_interface[0]) @@ -723,8 +752,9 @@ def test_crm_neighbor(duthosts, enum_rand_one_per_hwsku_frontend_hostname, # Add neighbor asichost.shell(neighbor_add_cmd) - crm_stats_checker = wait_until(60, 5, 0, check_crm_stats, get_neighbor_stats, duthost, crm_stats_neighbor_used, - crm_stats_neighbor_available, ">", "<") + crm_stats_checker = wait_until(60, 5, 0, check_crm_stats, get_neighbor_stats, duthost, + crm_stats_neighbor_used, crm_stats_neighbor_available, ">", "<", + skip_stats_check=skip_stats_check) if not crm_stats_checker: RESTORE_CMDS["test_crm_nexthop"].append(neighbor_del_cmd) pytest_assert(crm_stats_checker, @@ -737,8 +767,9 @@ def test_crm_neighbor(duthosts, enum_rand_one_per_hwsku_frontend_hostname, # Remove neighbor asichost.shell(neighbor_del_cmd) - crm_stats_checker = wait_until(60, 5, 0, check_crm_stats, get_neighbor_stats, duthost, crm_stats_neighbor_used, - crm_stats_neighbor_available, ">=", "==") + crm_stats_checker = wait_until(60, 5, 0, check_crm_stats, get_neighbor_stats, duthost, + crm_stats_neighbor_used, crm_stats_neighbor_available, ">=", "==", + skip_stats_check=skip_stats_check) pytest_assert(crm_stats_checker, "\"crm_stats_ipv4_neighbor_used\" counter was not decremented or " "\"crm_stats_ipv4_neighbor_available\" counter was not incremented") @@ -771,6 +802,8 @@ def test_crm_nexthop_group(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_frontend_asic_index, crm_interface, group_member, network): duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] asichost = duthost.asic_instance(enum_frontend_asic_index) + asic_type = duthost.facts['asic_type'] + skip_stats_check = True if asic_type == "vs" else False nhg_del_template = """ %s @@ -821,7 +854,8 @@ def test_crm_nexthop_group(duthosts, enum_rand_one_per_hwsku_frontend_hostname, template_resource = 1 crm_stats_checker = wait_until(60, 5, 0, check_crm_stats, get_nexthop_group_stats, duthost, nexthop_group_used + template_resource, - nexthop_group_available + template_resource, "==", "<=") + nexthop_group_available + template_resource, "==", "<=", + skip_stats_check=skip_stats_check) if not crm_stats_checker: RESTORE_CMDS["test_crm_nexthop_group"].append(del_template.render( iface=crm_interface[0], iface2=crm_interface[1], prefix=network, namespace=asichost.namespace)) @@ -837,8 +871,8 @@ def test_crm_nexthop_group(duthosts, enum_rand_one_per_hwsku_frontend_hostname, prefix=network, namespace=asichost.namespace)) crm_stats_checker = wait_until(60, 5, 0, check_crm_stats, get_nexthop_group_stats, duthost, - nexthop_group_used, - nexthop_group_available) + nexthop_group_used, nexthop_group_available, + skip_stats_check=skip_stats_check) nexthop_group_name = "member_" if group_member else "" pytest_assert(crm_stats_checker, "\"crm_stats_nexthop_group_{}used\" counter was not decremented or " @@ -917,6 +951,8 @@ def test_acl_entry(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro def verify_acl_crm_stats(duthost, asichost, enum_rand_one_per_hwsku_frontend_hostname, enum_frontend_asic_index, asic_collector, tbinfo): + asic_type = duthost.facts['asic_type'] + skip_stats_check = True if asic_type == "vs" else False apply_acl_config(duthost, asichost, "test_acl_entry", asic_collector, entry_num=2) acl_tbl_key = asic_collector["acl_tbl_key"] get_acl_entry_stats = "{db_cli} COUNTERS_DB HMGET {acl_tbl_key} \ @@ -1008,6 +1044,7 @@ def verify_acl_crm_stats(duthost, asichost, enum_rand_one_per_hwsku_frontend_hos duthost, crm_stats_acl_entry_used, crm_stats_acl_entry_available, + skip_stats_check=skip_stats_check ) # Remove ACL @@ -1019,6 +1056,8 @@ def test_acl_counter(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_f duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] asichost = duthost.asic_instance(enum_frontend_asic_index) asic_collector = collector[asichost.asic_index] + asic_type = duthost.facts['asic_type'] + skip_stats_check = True if asic_type == "vs" else False if "acl_tbl_key" not in asic_collector: pytest.skip("acl_tbl_key is not retrieved") @@ -1077,22 +1116,25 @@ def test_acl_counter(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_f # Remove ACL duthost.command("acl-loader delete") crm_stats_checker = wait_until(30, 5, 0, check_crm_stats, get_acl_counter_stats, duthost, - crm_stats_acl_counter_used, - crm_stats_acl_counter_available, "==", ">=") + crm_stats_acl_counter_used, crm_stats_acl_counter_available, "==", ">=", + skip_stats_check=skip_stats_check) pytest_assert(crm_stats_checker, "\"crm_stats_acl_counter_used\" counter was not decremented or " "\"crm_stats_acl_counter_available\" counter was not incremented") - # Verify "crm_stats_acl_counter_available" counter was equal to original value - _, new_crm_stats_acl_counter_available = get_crm_stats(get_acl_counter_stats, duthost) - pytest_assert(original_crm_stats_acl_counter_available - new_crm_stats_acl_counter_available == 0, - "\"crm_stats_acl_counter_available\" counter is not equal to original value") + if skip_stats_check is False: + # Verify "crm_stats_acl_counter_available" counter was equal to original value + _, new_crm_stats_acl_counter_available = get_crm_stats(get_acl_counter_stats, duthost) + pytest_assert(original_crm_stats_acl_counter_available - new_crm_stats_acl_counter_available == 0, + "\"crm_stats_acl_counter_available\" counter is not equal to original value") @pytest.mark.usefixtures('disable_fdb_aging') def test_crm_fdb_entry(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_frontend_asic_index, tbinfo): duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] asichost = duthost.asic_instance(enum_frontend_asic_index) + asic_type = duthost.facts['asic_type'] + skip_stats_check = True if asic_type == "vs" else False get_fdb_stats = "redis-cli --raw -n 2 HMGET CRM:STATS crm_stats_fdb_entry_used crm_stats_fdb_entry_available" topology = tbinfo["topo"]["properties"]["topology"] @@ -1141,29 +1183,30 @@ def test_crm_fdb_entry(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum # Get new "crm_stats_fdb_entry" used and available counter value new_crm_stats_fdb_entry_used, new_crm_stats_fdb_entry_available = get_crm_stats(get_fdb_stats, duthost) - # Verify "crm_stats_fdb_entry_used" counter was incremented - # For Cisco-8000 devices, hardware FDB counter is statistical-based with +/- 1 entry tolerance. - # Hence, the used counter can increase by more than 1. - # For E1031, refer CS00012270660, SDK for Helix4 chip does not support retrieving max l2 entry, - # HW and SW CRM available counter would be out of sync and increase by more than 1. - if is_cisco_device(duthost) or is_cel_e1031_device(duthost): - pytest_assert(new_crm_stats_fdb_entry_used - crm_stats_fdb_entry_used >= 1, - "Counter 'crm_stats_fdb_entry_used' was not incremented") - else: - pytest_assert(new_crm_stats_fdb_entry_used - crm_stats_fdb_entry_used == 1, - "Counter 'crm_stats_fdb_entry_used' was not incremented") - - # Verify "crm_stats_fdb_entry_available" counter was decremented - # For Cisco-8000 devices, hardware FDB counter is statistical-based with +/- 1 entry tolerance. - # Hence, the available counter can decrease by more than 1. - # For E1031, refer CS00012270660, SDK for Helix4 chip does not support retrieving max l2 entry, - # HW and SW CRM available counter would be out of sync and decrease by more than 1. - if is_cisco_device(duthost) or is_cel_e1031_device(duthost): - pytest_assert(crm_stats_fdb_entry_available - new_crm_stats_fdb_entry_available >= 1, - "Counter 'crm_stats_fdb_entry_available' was not decremented") - else: - pytest_assert(crm_stats_fdb_entry_available - new_crm_stats_fdb_entry_available == 1, - "Counter 'crm_stats_fdb_entry_available' was not decremented") + if skip_stats_check is False: + # Verify "crm_stats_fdb_entry_used" counter was incremented + # For Cisco-8000 devices, hardware FDB counter is statistical-based with +/- 1 entry tolerance. + # Hence, the used counter can increase by more than 1. + # For E1031, refer CS00012270660, SDK for Helix4 chip does not support retrieving max l2 entry, + # HW and SW CRM available counter would be out of sync and increase by more than 1. + if is_cisco_device(duthost) or is_cel_e1031_device(duthost): + pytest_assert(new_crm_stats_fdb_entry_used - crm_stats_fdb_entry_used >= 1, + "Counter 'crm_stats_fdb_entry_used' was not incremented") + else: + pytest_assert(new_crm_stats_fdb_entry_used - crm_stats_fdb_entry_used == 1, + "Counter 'crm_stats_fdb_entry_used' was not incremented") + + # Verify "crm_stats_fdb_entry_available" counter was decremented + # For Cisco-8000 devices, hardware FDB counter is statistical-based with +/- 1 entry tolerance. + # Hence, the available counter can decrease by more than 1. + # For E1031, refer CS00012270660, SDK for Helix4 chip does not support retrieving max l2 entry, + # HW and SW CRM available counter would be out of sync and decrease by more than 1. + if is_cisco_device(duthost) or is_cel_e1031_device(duthost): + pytest_assert(crm_stats_fdb_entry_available - new_crm_stats_fdb_entry_available >= 1, + "Counter 'crm_stats_fdb_entry_available' was not decremented") + else: + pytest_assert(crm_stats_fdb_entry_available - new_crm_stats_fdb_entry_available == 1, + "Counter 'crm_stats_fdb_entry_available' was not decremented") used_percent = get_used_percent(new_crm_stats_fdb_entry_used, new_crm_stats_fdb_entry_available) if used_percent < 1: diff --git a/tests/dash/conftest.py b/tests/dash/conftest.py index d66b9623e4..445fba4ae8 100644 --- a/tests/dash/conftest.py +++ b/tests/dash/conftest.py @@ -1,5 +1,8 @@ import logging import pytest +import random +import json +import time from ipaddress import ip_interface from constants import ENI, VM_VNI, VNET1_VNI, VNET2_VNI, REMOTE_CA_IP, LOCAL_CA_IP, REMOTE_ENI_MAC,\ @@ -8,6 +11,7 @@ ROUTING_ACTION_TYPE, LOOKUP_OVERLAY_IP, ACL_GROUP, ACL_STAGE from dash_utils import render_template_to_host, apply_swssconfig_file from gnmi_utils import generate_gnmi_cert, apply_gnmi_cert, recover_gnmi_cert, apply_gnmi_file +from dash_acl import AclGroup, DEFAULT_ACL_GROUP, WAIT_AFTER_CONFIG, DefaultAclRule logger = logging.getLogger(__name__) @@ -43,6 +47,19 @@ def pytest_addoption(parser): help="Skip dataplane checking" ) + parser.addoption( + "--vxlan_udp_dport", + action="store", + default="random", + help="The vxlan udp dst port used in the test" + ) + + parser.addoption( + "--skip_cert_cleanup", + action="store_true", + help="Skip certificates cleanup after test" + ) + @pytest.fixture(scope="module") def config_only(request): @@ -64,6 +81,11 @@ def skip_dataplane_checking(request): return request.config.getoption("--skip_dataplane_checking") +@pytest.fixture(scope="module") +def skip_cert_cleanup(request): + return request.config.getoption("--skip_cert_cleanup") + + @pytest.fixture(scope="module") def config_facts(duthost): return duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] @@ -244,7 +266,7 @@ def apply_direct_configs(dash_outbound_configs, apply_config): @pytest.fixture(scope="module", autouse=True) -def setup_gnmi_server(duthosts, rand_one_dut_hostname, localhost, ptfhost): +def setup_gnmi_server(duthosts, rand_one_dut_hostname, localhost, ptfhost, skip_cert_cleanup): if not ENABLE_GNMI_API: yield return @@ -253,7 +275,7 @@ def setup_gnmi_server(duthosts, rand_one_dut_hostname, localhost, ptfhost): generate_gnmi_cert(localhost, duthost) apply_gnmi_cert(duthost, ptfhost) yield - recover_gnmi_cert(localhost, duthost) + recover_gnmi_cert(localhost, duthost, skip_cert_cleanup) @pytest.fixture(scope="function") @@ -268,3 +290,67 @@ def _check_asic_db(tables): @pytest.fixture(scope="function", params=['udp', 'tcp', 'echo_request', 'echo_reply']) def inner_packet_type(request): return request.param + + +def config_vxlan_udp_dport(duthost, port): + vxlan_port_config = [ + { + "SWITCH_TABLE:switch": {"vxlan_port": f"{port}"}, + "OP": "SET" + } + ] + config_path = "/tmp/vxlan_port_config.json" + duthost.copy(content=json.dumps(vxlan_port_config, indent=4), dest=config_path, verbose=False) + apply_swssconfig_file(duthost, config_path) + + +@pytest.fixture(scope="function") +def vxlan_udp_dport(request, duthost): + """ + Test the traffic with specified or randomly generated VxLAN UDP dst port. + Configuration is applied by swssconfig. + """ + UDP_PORT_RANGE = range(0, 65536) + WELL_KNOWN_UDP_PORT_RANGE = range(0, 1024) + vxlan_udp_dport = request.config.getoption("--vxlan_udp_dport") + if vxlan_udp_dport == "random": + port_candidate_list = ["default", 4789, 13330, 1024, 65535] + while True: + random_port = random.choice(UDP_PORT_RANGE) + if random_port not in WELL_KNOWN_UDP_PORT_RANGE and random_port not in port_candidate_list: + port_candidate_list.append(random_port) + break + vxlan_udp_dport = random.choice(port_candidate_list) + if vxlan_udp_dport != "default": + logger.info(f"Configure the VXLAN UDP dst port {vxlan_udp_dport} to DPU") + vxlan_udp_dport = int(vxlan_udp_dport) + config_vxlan_udp_dport(duthost, vxlan_udp_dport) + else: + logger.info("Use the default VXLAN UDP dst port 4789") + vxlan_udp_dport = 4789 + + yield vxlan_udp_dport + + logger.info("Restore the VXLAN UDP dst port to 4789") + config_vxlan_udp_dport(duthost, 4789) + + +@pytest.fixture(scope="function") +def acl_default_rule(localhost, duthost, ptfhost, dash_config_info): + hwsku = duthost.facts['hwsku'] + hwsku_list_with_default_acl_action_deny = ['Nvidia-9009d3b600CVAA-C1', 'Nvidia-9009d3b600SVAA-C1'] + if hwsku in hwsku_list_with_default_acl_action_deny: + default_acl_group = AclGroup(localhost, duthost, ptfhost, DEFAULT_ACL_GROUP, dash_config_info[ENI]) + default_acl_rule = DefaultAclRule(localhost, duthost, ptfhost, dash_config_info, "allow") + + default_acl_rule.config() + default_acl_group.bind(1) + time.sleep(WAIT_AFTER_CONFIG) + + yield + + if hwsku in hwsku_list_with_default_acl_action_deny: + default_acl_group.unbind() + default_acl_rule.teardown() + del default_acl_group + time.sleep(WAIT_AFTER_CONFIG) diff --git a/tests/dash/dash_acl.py b/tests/dash/dash_acl.py index 713c6b28b4..1ce0abe5f7 100644 --- a/tests/dash/dash_acl.py +++ b/tests/dash/dash_acl.py @@ -11,7 +11,6 @@ from gnmi_utils import apply_gnmi_file import packets import ptf.testutils as testutils -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/dash/dash_utils.py b/tests/dash/dash_utils.py index 7c664fba4a..d37bc558a8 100644 --- a/tests/dash/dash_utils.py +++ b/tests/dash/dash_utils.py @@ -5,7 +5,6 @@ from jinja2 import Template from constants import TEMPLATE_DIR -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/dash/gnmi_utils.py b/tests/dash/gnmi_utils.py index 86a3d10def..95432701df 100644 --- a/tests/dash/gnmi_utils.py +++ b/tests/dash/gnmi_utils.py @@ -7,7 +7,6 @@ import pytest import proto_utils -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) @@ -198,7 +197,7 @@ def apply_gnmi_cert(duthost, ptfhost): time.sleep(env.gnmi_server_start_wait_time) -def recover_gnmi_cert(localhost, duthost): +def recover_gnmi_cert(localhost, duthost, skip_cert_cleanup): """ Restart gnmi server to use default certificate @@ -209,7 +208,8 @@ def recover_gnmi_cert(localhost, duthost): Returns: """ env = GNMIEnvironment(duthost) - localhost.shell("rm -rf "+env.work_dir, module_ignore_errors=True) + if not skip_cert_cleanup: + localhost.shell("rm -rf "+env.work_dir, module_ignore_errors=True) dut_command = "docker exec %s supervisorctl status %s" % (env.gnmi_container, env.gnmi_program) output = duthost.command(dut_command, module_ignore_errors=True)['stdout'].strip() if 'RUNNING' in output: diff --git a/tests/dash/packets.py b/tests/dash/packets.py index c3159a9f99..dbb5ec606a 100644 --- a/tests/dash/packets.py +++ b/tests/dash/packets.py @@ -9,7 +9,6 @@ import logging import sys import time -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 from tests.common.helpers.assertions import pytest_assert from six import StringIO @@ -35,7 +34,7 @@ def set_icmp_sub_type(packet, packet_type): packet[scapy.ICMP].type = 0 -def inbound_vnet_packets(dash_config_info, inner_extra_conf={}, inner_packet_type='udp'): +def inbound_vnet_packets(dash_config_info, inner_extra_conf={}, inner_packet_type='udp', vxlan_udp_dport=4789): inner_packet = generate_inner_packet(inner_packet_type)( eth_src=dash_config_info[REMOTE_ENI_MAC], eth_dst=dash_config_info[LOCAL_ENI_MAC], @@ -49,6 +48,7 @@ def inbound_vnet_packets(dash_config_info, inner_extra_conf={}, inner_packet_typ eth_dst=dash_config_info[DUT_MAC], ip_src=dash_config_info[REMOTE_PA_IP], ip_dst=dash_config_info[LOOPBACK_IP], + udp_dport=vxlan_udp_dport, vxlan_vni=dash_config_info[VNET2_VNI], ip_ttl=64, inner_frame=inner_packet @@ -58,6 +58,7 @@ def inbound_vnet_packets(dash_config_info, inner_extra_conf={}, inner_packet_typ eth_dst=dash_config_info[LOCAL_PTF_MAC], ip_src=dash_config_info[LOOPBACK_IP], ip_dst=dash_config_info[LOCAL_PA_IP], + udp_dport=vxlan_udp_dport, vxlan_vni=dash_config_info[VM_VNI], ip_ttl=255, ip_id=0, @@ -77,7 +78,7 @@ def inbound_vnet_packets(dash_config_info, inner_extra_conf={}, inner_packet_typ return inner_packet, pa_match_vxlan_packet, pa_mismatch_vxlan_packet, masked_exp_packet -def outbound_vnet_packets(dash_config_info, inner_extra_conf={}, inner_packet_type='udp'): +def outbound_vnet_packets(dash_config_info, inner_extra_conf={}, inner_packet_type='udp', vxlan_udp_dport=4789): proto = None if "proto" in inner_extra_conf: proto = int(inner_extra_conf["proto"]) @@ -100,6 +101,7 @@ def outbound_vnet_packets(dash_config_info, inner_extra_conf={}, inner_packet_ty eth_dst=dash_config_info[DUT_MAC], ip_src=dash_config_info[LOCAL_PA_IP], ip_dst=dash_config_info[LOOPBACK_IP], + udp_dport=vxlan_udp_dport, with_udp_chksum=False, vxlan_vni=dash_config_info[VM_VNI], ip_ttl=64, @@ -110,6 +112,7 @@ def outbound_vnet_packets(dash_config_info, inner_extra_conf={}, inner_packet_ty eth_dst=dash_config_info[REMOTE_PTF_MAC], ip_src=dash_config_info[LOOPBACK_IP], ip_dst=dash_config_info[REMOTE_PA_IP], + udp_dport=vxlan_udp_dport, vxlan_vni=dash_config_info[VNET2_VNI], # TODO: Change TTL to 63 after SAI bug is fixed ip_ttl=0xff, diff --git a/tests/dash/proto_utils.py b/tests/dash/proto_utils.py index 798de17d9b..23a6e0b458 100644 --- a/tests/dash/proto_utils.py +++ b/tests/dash/proto_utils.py @@ -18,7 +18,6 @@ from dash_api.acl_in_pb2 import AclIn from dash_api.acl_rule_pb2 import AclRule, Action from dash_api.prefix_tag_pb2 import PrefixTag -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 ENABLE_PROTO = True diff --git a/tests/dash/templates/dash_set_eni_admin_state.j2 b/tests/dash/templates/dash_set_eni_admin_state.j2 new file mode 100644 index 0000000000..afeb9b2c59 --- /dev/null +++ b/tests/dash/templates/dash_set_eni_admin_state.j2 @@ -0,0 +1,13 @@ +[ + { + "DASH_ENI_TABLE:{{ eni }}" : { + "eni_id":"497f23d7-f0ac-4c99-a98f-59b470e8c7bd", + "mac_address":"{{ local_eni_mac }}", + "underlay_ip":"{{ local_pa_ip }}", + "admin_state":"{{ eni_admin_state }}", + "vnet":"{{ vnet1_name }}", + "qos":"qos100" + }, + "OP": "SET" + } +] diff --git a/tests/dash/test_dash_acl.py b/tests/dash/test_dash_acl.py index 7a9ee0a0de..58bb24ffc8 100644 --- a/tests/dash/test_dash_acl.py +++ b/tests/dash/test_dash_acl.py @@ -6,7 +6,6 @@ from dash_acl import check_dataplane, acl_fields_test, acl_multi_stage_test, check_tcp_rst_dataplane, acl_tcp_rst_test # noqa: F401 from dash_acl import acl_tag_test, acl_multi_tag_test, acl_tag_order_test, acl_multi_tag_order_test # noqa: F401 from dash_acl import acl_tag_update_ip_test, acl_tag_remove_ip_test, acl_tag_scale_test, acl_tag_not_exists_test # noqa: F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/dash/test_dash_disable_enable_eni.py b/tests/dash/test_dash_disable_enable_eni.py new file mode 100644 index 0000000000..7561d0f84d --- /dev/null +++ b/tests/dash/test_dash_disable_enable_eni.py @@ -0,0 +1,72 @@ +import logging +import pytest +import ptf.testutils as testutils +import packets + +from constants import LOCAL_PTF_INTF, REMOTE_PTF_INTF +from tests.common.plugins.allure_wrapper import allure_step_wrapper as allure +from gnmi_utils import apply_gnmi_file +from dash_utils import render_template_to_host +from tests.common.utilities import wait_until +from tests.common.helpers.assertions import pytest_assert + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('dpu') +] + + +@pytest.fixture(autouse=True) +def skip_underlay_route(request): + if 'with-underlay-route' in request.node.name: + pytest.skip('Skip the test with param "with-underlay-route", ' + 'it is unnecessary to cover all underlay route scenarios.') + + +def test_dash_disable_enable_eni(ptfadapter, localhost, duthost, ptfhost, apply_vnet_configs, + dash_config_info, asic_db_checker, acl_default_rule): + """ + The test is to verify that after the ENI is disabled, the corresponding traffic should be dropped by the DPU. + """ + asic_db_checker(["SAI_OBJECT_TYPE_VNET", "SAI_OBJECT_TYPE_ENI"]) + with allure.step("Verify the dash traffic when ENI is enabled"): + _, vxlan_packet, expected_packet = packets.outbound_vnet_packets(dash_config_info) + testutils.send(ptfadapter, dash_config_info[LOCAL_PTF_INTF], vxlan_packet, 1) + testutils.verify_packets_any(ptfadapter, expected_packet, ports=dash_config_info[REMOTE_PTF_INTF]) + + def _set_eni_admin_state(state): + eni_set_state_config = "dash_set_eni_admin_state" + template_name = f"{eni_set_state_config}.j2" + dest_path = f"/tmp/{eni_set_state_config}.json" + render_template_to_host(template_name, duthost, dest_path, dash_config_info, eni_admin_state=state) + apply_gnmi_file(localhost, duthost, ptfhost, dest_path) + + def _check_eni_admin_state(state): + asic_db_eni_state = duthost.shell( + f"redis-cli -n 1 hget {asic_db_eni_key} SAI_ENI_ATTR_ADMIN_STATE")["stdout"] + return asic_db_eni_state == state + + with allure.step("Disabled the ENI"): + _set_eni_admin_state("disabled") + + with allure.step("Check ASIC db to confirm the ENI is disabled"): + asic_db_eni_key = duthost.shell("redis-cli -n 1 keys *ENI:oid*")["stdout"] + pytest_assert(wait_until(10, 2, 0, _check_eni_admin_state, "false"), + "The ENI admin state in ASIC_DB is still true") + + with allure.step("Verify the dash traffic is dropped after ENI is disabled"): + testutils.send(ptfadapter, dash_config_info[LOCAL_PTF_INTF], vxlan_packet, 1) + testutils.verify_no_packet_any(ptfadapter, expected_packet, ports=dash_config_info[REMOTE_PTF_INTF]) + + with allure.step("Enable the ENI"): + _set_eni_admin_state("enabled") + + with allure.step("Check ASIC db to confirm the ENI is enabled"): + asic_db_eni_key = duthost.shell("redis-cli -n 1 keys *ENI:oid*")["stdout"] + pytest_assert(wait_until(10, 2, 0, _check_eni_admin_state, "true"), + "The ENI admin state in ASIC_DB is still false") + + with allure.step("Verify the dash traffic is forwarded after ENI is enabled"): + testutils.send(ptfadapter, dash_config_info[LOCAL_PTF_INTF], vxlan_packet, 1) + testutils.verify_packets_any(ptfadapter, expected_packet, ports=dash_config_info[REMOTE_PTF_INTF]) diff --git a/tests/dash/test_dash_vnet.py b/tests/dash/test_dash_vnet.py index 65dabb4188..db11b7da67 100644 --- a/tests/dash/test_dash_vnet.py +++ b/tests/dash/test_dash_vnet.py @@ -1,13 +1,9 @@ import logging - import pytest import ptf.testutils as testutils - -from constants import LOCAL_PTF_INTF, REMOTE_PTF_INTF, ENI -from dash_acl import AclGroup, DEFAULT_ACL_GROUP, WAIT_AFTER_CONFIG, DefaultAclRule import packets -import time -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 + +from constants import LOCAL_PTF_INTF, REMOTE_PTF_INTF logger = logging.getLogger(__name__) @@ -17,27 +13,6 @@ ] -@pytest.fixture(scope="function") -def acl_default_rule(duthost, ptfhost, dash_config_info): - hwsku = duthost.facts['hwsku'] - hwsku_list_with_default_acl_action_deny = ['Nvidia-9009d3b600CVAA-C1', 'Nvidia-9009d3b600SVAA-C1'] - if hwsku in hwsku_list_with_default_acl_action_deny: - default_acl_group = AclGroup(duthost, ptfhost, DEFAULT_ACL_GROUP, dash_config_info[ENI]) - default_acl_rule = DefaultAclRule(duthost, ptfhost, dash_config_info, "allow") - - default_acl_rule.config() - default_acl_group.bind(1) - time.sleep(WAIT_AFTER_CONFIG) - - yield - - if hwsku in hwsku_list_with_default_acl_action_deny: - default_acl_group.unbind() - default_acl_rule.teardown() - del default_acl_group - time.sleep(WAIT_AFTER_CONFIG) - - def test_outbound_vnet( ptfadapter, apply_vnet_configs, @@ -45,7 +20,8 @@ def test_outbound_vnet( skip_dataplane_checking, asic_db_checker, inner_packet_type, - acl_default_rule): + acl_default_rule, + vxlan_udp_dport): """ Send VXLAN packets from the VM VNI """ @@ -53,6 +29,7 @@ def test_outbound_vnet( if skip_dataplane_checking: return _, vxlan_packet, expected_packet = packets.outbound_vnet_packets(dash_config_info, + vxlan_udp_dport=vxlan_udp_dport, inner_packet_type=inner_packet_type) testutils.send(ptfadapter, dash_config_info[LOCAL_PTF_INTF], vxlan_packet, 1) testutils.verify_packets_any(ptfadapter, expected_packet, ports=dash_config_info[REMOTE_PTF_INTF]) @@ -66,11 +43,13 @@ def test_outbound_vnet_direct( skip_dataplane_checking, asic_db_checker, inner_packet_type, - acl_default_rule): + acl_default_rule, + vxlan_udp_dport): asic_db_checker(["SAI_OBJECT_TYPE_VNET", "SAI_OBJECT_TYPE_ENI"]) if skip_dataplane_checking: return _, vxlan_packet, expected_packet = packets.outbound_vnet_packets(dash_config_info, + vxlan_udp_dport=vxlan_udp_dport, inner_packet_type=inner_packet_type) testutils.send(ptfadapter, dash_config_info[LOCAL_PTF_INTF], vxlan_packet, 1) testutils.verify_packets_any(ptfadapter, expected_packet, ports=dash_config_info[REMOTE_PTF_INTF]) @@ -84,11 +63,13 @@ def test_outbound_direct( skip_dataplane_checking, asic_db_checker, inner_packet_type, - acl_default_rule): + acl_default_rule, + vxlan_udp_dport): asic_db_checker(["SAI_OBJECT_TYPE_VNET", "SAI_OBJECT_TYPE_ENI"]) if skip_dataplane_checking: return expected_inner_packet, vxlan_packet, _ = packets.outbound_vnet_packets(dash_config_info, + vxlan_udp_dport=vxlan_udp_dport, inner_packet_type=inner_packet_type) testutils.send(ptfadapter, dash_config_info[LOCAL_PTF_INTF], vxlan_packet, 1) testutils.verify_packets_any(ptfadapter, expected_inner_packet, ports=dash_config_info[REMOTE_PTF_INTF]) @@ -102,7 +83,8 @@ def test_inbound_vnet_pa_validate( skip_dataplane_checking, asic_db_checker, inner_packet_type, - acl_default_rule): + acl_default_rule, + vxlan_udp_dport): """ Send VXLAN packets from the remote VNI with PA validation enabled @@ -115,7 +97,7 @@ def test_inbound_vnet_pa_validate( if skip_dataplane_checking: return _, pa_match_packet, pa_mismatch_packet, expected_packet = packets.inbound_vnet_packets( - dash_config_info, inner_packet_type=inner_packet_type) + dash_config_info, vxlan_udp_dport=vxlan_udp_dport, inner_packet_type=inner_packet_type) testutils.send(ptfadapter, dash_config_info[REMOTE_PTF_INTF], pa_match_packet, 1) testutils.verify_packets_any(ptfadapter, expected_packet, ports=dash_config_info[LOCAL_PTF_INTF]) testutils.send(ptfadapter, dash_config_info[REMOTE_PTF_INTF], pa_mismatch_packet, 1) diff --git a/tests/dash/test_relaxed_match_negative.py b/tests/dash/test_relaxed_match_negative.py new file mode 100644 index 0000000000..1e7a4af908 --- /dev/null +++ b/tests/dash/test_relaxed_match_negative.py @@ -0,0 +1,52 @@ +import logging +import pytest +import ptf.testutils as testutils + +from constants import LOCAL_PTF_INTF, REMOTE_PTF_INTF +from tests.common.plugins.allure_wrapper import allure_step_wrapper as allure +from tests.dash.conftest import config_vxlan_udp_dport +import packets + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('dpu'), +] + + +@pytest.fixture(autouse=True) +def skip_underlay_route(request): + if 'with-underlay-route' in request.node.name: + pytest.skip('Skip the test with param "with-underlay-route", ' + 'it is unnecessary to cover all underlay route scenarios.') + + +@pytest.fixture() +def restore_vxlan_udp_dport(duthost): + + yield + + config_vxlan_udp_dport(duthost, 4789) + + +def test_relaxed_match_negative(duthost, ptfadapter, apply_vnet_configs, dash_config_info, + acl_default_rule, restore_vxlan_udp_dport): + """ + Negative test of dynamically changing the VxLAN UDP dst port + """ + with allure.step("Check the traffic with default port 4789 is forwarded by the DPU"): + _, vxlan_packet, expected_packet = packets.outbound_vnet_packets(dash_config_info) + testutils.send(ptfadapter, dash_config_info[LOCAL_PTF_INTF], vxlan_packet, 1) + testutils.verify_packets_any(ptfadapter, expected_packet, ports=dash_config_info[REMOTE_PTF_INTF]) + with allure.step("Change the port to 13330, check the packet with port 4789 is dropped"): + config_vxlan_udp_dport(duthost, 13330) + testutils.send(ptfadapter, dash_config_info[LOCAL_PTF_INTF], vxlan_packet, 1) + testutils.verify_no_packet_any(ptfadapter, expected_packet, ports=dash_config_info[REMOTE_PTF_INTF]) + with allure.step("Change the port back to 4789, check the packet with port 4789 is forwarded"): + config_vxlan_udp_dport(duthost, 4789) + testutils.send(ptfadapter, dash_config_info[LOCAL_PTF_INTF], vxlan_packet, 1) + testutils.verify_packets_any(ptfadapter, expected_packet, ports=dash_config_info[REMOTE_PTF_INTF]) + with allure.step("Check the packet with port 13330 is dropped"): + _, vxlan_packet, expected_packet = packets.outbound_vnet_packets(dash_config_info, vxlan_udp_dport=13330) + testutils.send(ptfadapter, dash_config_info[LOCAL_PTF_INTF], vxlan_packet, 1) + testutils.verify_no_packet_any(ptfadapter, expected_packet, ports=dash_config_info[REMOTE_PTF_INTF]) diff --git a/tests/database/test_db_scripts.py b/tests/database/test_db_scripts.py index 76af0386bb..fc7d61a7cb 100644 --- a/tests/database/test_db_scripts.py +++ b/tests/database/test_db_scripts.py @@ -6,7 +6,6 @@ from tests.common.utilities import skip_release from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/decap/test_decap.py b/tests/decap/test_decap.py index 447570e2ec..9464ce15a5 100644 --- a/tests/decap/test_decap.py +++ b/tests/decap/test_decap.py @@ -21,6 +21,7 @@ from tests.common.fixtures.ptfhost_utils import remove_ip_addresses # noqa F401 from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 from tests.common.fixtures.ptfhost_utils import set_ptf_port_mapping_mode # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.fixtures.ptfhost_utils import ptf_test_port_map_active_active from tests.common.fixtures.fib_utils import fib_info_files # noqa F401 from tests.common.fixtures.fib_utils import single_fib_for_duts # noqa F401 @@ -33,7 +34,6 @@ from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_random_side # noqa F401 from tests.common.dualtor.nic_simulator_control import mux_status_from_nic_simulator # noqa F401 from tests.common.dualtor.dual_tor_utils import is_tunnel_qos_remap_enabled -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) @@ -44,6 +44,20 @@ ] +@pytest.fixture(autouse=True) +def ignore_expected_loganalyzer_exceptions(duthosts, rand_one_dut_hostname, loganalyzer): + # Ignore in KVM test + KVMIgnoreRegex = [ + ".*unknown decap tunnel table attribute 'dst_ip'.*", + ".*Tunnel TEST_IPINIP_V4_TUNNEL cannot be removed since it doesn't exist.*", + ".*Tunnel TEST_IPINIP_V6_TUNNEL cannot be removed since it doesn't exist.*", + ] + duthost = duthosts[rand_one_dut_hostname] + if loganalyzer: # Skip if loganalyzer is disabled + if duthost.facts["asic_type"] == "vs": + loganalyzer[duthost.hostname].ignore_regex.extend(KVMIgnoreRegex) + + def remove_default_decap_cfg(duthosts): for duthost in duthosts: logger.info('Remove default decap cfg on {}'.format(duthost.hostname)) @@ -179,7 +193,8 @@ def simulate_vxlan_teardown(duthosts, ptfhost, tbinfo): def test_decap(tbinfo, duthosts, ptfhost, setup_teardown, mux_server_url, # noqa F811 toggle_all_simulator_ports_to_random_side, supported_ttl_dscp_params, ip_ver, loopback_ips, # noqa F811 - duts_running_config_facts, duts_minigraph_facts, mux_status_from_nic_simulator): # noqa F811 + duts_running_config_facts, duts_minigraph_facts, mux_status_from_nic_simulator, # noqa F811 + skip_traffic_test): # noqa F811 setup_info = setup_teardown asic_type = duthosts[0].facts["asic_type"] ecn_mode = "copy_from_outer" @@ -199,6 +214,9 @@ def test_decap(tbinfo, duthosts, ptfhost, setup_teardown, mux_server_url, else: apply_decap_cfg(duthosts, ip_ver, loopback_ips, ttl_mode, dscp_mode, ecn_mode, 'SET') + if skip_traffic_test: + return + if 'dualtor' in tbinfo['topo']['name']: wait(30, 'Wait some time for mux active/standby state to be stable after toggled mux state') diff --git a/tests/decap/test_subnet_decap.py b/tests/decap/test_subnet_decap.py new file mode 100644 index 0000000000..bed66fbe68 --- /dev/null +++ b/tests/decap/test_subnet_decap.py @@ -0,0 +1,225 @@ +import pytest +import logging +import json +import time +import random +from collections import defaultdict + +import ptf.packet as packet +import ptf.testutils as testutils +from ptf.mask import Mask +from tests.common.dualtor.dual_tor_utils import rand_selected_interface # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 +from tests.common.fixtures.ptfhost_utils import copy_arp_responder_py # noqa F401 +from tests.common.config_reload import config_reload + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('t0', 't1', 'dualtor') +] + +DECAP_IPINIP_SUBNET_CONFIG_TEMPLATE = "decap/template/decap_ipinip_subnet_config.j2" +DECAP_IPINIP_SUBNET_CONFIG_JSON = "decap_ipinip_subnet_config.json" +DECAP_IPINIP_SUBNET_DEL_TEMPLATE = "decap/template/decap_ipinip_subnet_delete.j2" +DECAP_IPINIP_SUBNET_DEL_JSON = "decap_ipinip_subnet_delete.json" + +SUBNET_DECAP_SRC_IP_V4 = "20.20.20.0/24" +SUBNET_DECAP_SRC_IP_V6 = "fc01::/120" +OUTER_DST_IP_V4 = "192.168.0.200" +OUTER_DST_IP_V6 = "fc02:1000::200" + + +@pytest.fixture(scope='module') +def prepare_subnet_decap_config(rand_selected_dut): + logger.info("Prepare subnet decap config") + rand_selected_dut.shell('sonic-db-cli CONFIG_DB hset "SUBNET_DECAP|subnet_type" \ + "status" "enable" "src_ip" "{}" "src_ip_v6" "{}"' + .format(SUBNET_DECAP_SRC_IP_V4, SUBNET_DECAP_SRC_IP_V6)) + rand_selected_dut.shell('sudo config save -y') + config_reload(rand_selected_dut) + # Wait for all processes come up + time.sleep(120) + + yield + rand_selected_dut.shell('sonic-db-cli CONFIG_DB del "SUBNET_DECAP|subnet_type"') + rand_selected_dut.shell('sudo config save -y') + config_reload(rand_selected_dut, config_source='minigraph') + + +@pytest.fixture(scope='module') +def prepare_vlan_subnet_test_port(rand_selected_dut, tbinfo): + mg_facts = rand_selected_dut.get_extended_minigraph_facts(tbinfo) + topo = tbinfo["topo"]["type"] + dut_port = list(mg_facts['minigraph_portchannels'].keys())[0] + if not dut_port: + pytest.skip('No portchannels found') + dut_eth_port = mg_facts["minigraph_portchannels"][dut_port]["members"][0] + ptf_src_port = mg_facts["minigraph_ptf_indices"][dut_eth_port] + + downstream_port_ids = [] + upstream_port_ids = [] + for interface, neighbor in list(mg_facts["minigraph_neighbors"].items()): + port_id = mg_facts["minigraph_ptf_indices"][interface] + if topo == "t0" and "Servers" in neighbor["name"]: + downstream_port_ids.append(port_id) + elif topo == "t0" and "T1" in neighbor["name"]: + upstream_port_ids.append(port_id) + + logger.info("ptf_src_port: {}, downstream_port_ids: {}, upstream_port_ids: {}" + .format(ptf_src_port, downstream_port_ids, upstream_port_ids)) + return ptf_src_port, downstream_port_ids, upstream_port_ids + + +@pytest.fixture(scope='module') +def prepare_negative_ip_port_map(prepare_vlan_subnet_test_port): + _, downstream_port_ids, _ = prepare_vlan_subnet_test_port + ptf_target_port = random.choice(downstream_port_ids) + ip_to_port = { + OUTER_DST_IP_V4: ptf_target_port, + OUTER_DST_IP_V6: ptf_target_port + } + return ptf_target_port, ip_to_port + + +@pytest.fixture +def setup_arp_responder(rand_selected_dut, ptfhost, prepare_negative_ip_port_map): + ptfhost.command('supervisorctl stop arp_responder', module_ignore_errors=True) + + _, ip_to_port = prepare_negative_ip_port_map + arp_responder_cfg = defaultdict(list) + ip_list = [] + + for ip, port in list(ip_to_port.items()): + iface = "eth{}".format(port) + arp_responder_cfg[iface].append(ip) + ip_list.append(ip) + + CFG_FILE = '/tmp/arp_responder.json' + with open(CFG_FILE, 'w') as file: + json.dump(arp_responder_cfg, file) + + ptfhost.copy(src=CFG_FILE, dest=CFG_FILE) + extra_vars = { + 'arp_responder_args': '--conf {}'.format(CFG_FILE) + } + + ptfhost.host.options['variable_manager'].extra_vars.update(extra_vars) + ptfhost.template(src='templates/arp_responder.conf.j2', dest='/etc/supervisor/conf.d/arp_responder.conf') + + ptfhost.command('supervisorctl reread') + ptfhost.command('supervisorctl update') + + logger.info("Start arp_responder") + ptfhost.command('supervisorctl start arp_responder') + time.sleep(10) + + yield + + ptfhost.command('supervisorctl stop arp_responder', module_ignore_errors=True) + ptfhost.file(path='/tmp/arp_responder.json', state="absent") + rand_selected_dut.command('sonic-clear arp') + + +def build_encapsulated_vlan_subnet_packet(ptfadapter, rand_selected_dut, ip_version, stage): + eth_dst = rand_selected_dut.facts["router_mac"] + eth_src = ptfadapter.dataplane.get_mac(0, 0) + logger.info("eth_src: {}, eth_dst: {}".format(eth_src, eth_dst)) + + if ip_version == "IPv4": + outer_dst_ipv4 = OUTER_DST_IP_V4 + if stage == "positive": + outer_src_ipv4 = "20.20.20.10" + elif stage == "negative": + outer_src_ipv4 = "30.30.30.10" + + inner_packet = testutils.simple_ip_packet( + ip_src="1.1.1.1", + ip_dst="2.2.2.2" + )[packet.IP] + outer_packet = testutils.simple_ipv4ip_packet( + eth_dst=eth_dst, + eth_src=eth_src, + ip_src=outer_src_ipv4, + ip_dst=outer_dst_ipv4, + inner_frame=inner_packet + ) + + elif ip_version == "IPv6": + outer_dst_ipv6 = OUTER_DST_IP_V6 + if stage == "positive": + outer_src_ipv6 = "fc01::10" + elif stage == "negative": + outer_src_ipv6 = "fc01::10:10" + + inner_packet = testutils.simple_tcpv6_packet( + ipv6_src="1::1", + ipv6_dst="2::2" + )[packet.IPv6] + outer_packet = testutils.simple_ipv6ip_packet( + eth_dst=eth_dst, + eth_src=eth_src, + ipv6_src=outer_src_ipv6, + ipv6_dst=outer_dst_ipv6, + inner_frame=inner_packet + ) + + return outer_packet + + +def build_expected_vlan_subnet_packet(encapsulated_packet, ip_version, stage, decrease_ttl=False): + if stage == "positive": + if ip_version == "IPv4": + pkt = encapsulated_packet[packet.IP].payload[packet.IP].copy() + elif ip_version == "IPv6": + pkt = encapsulated_packet[packet.IPv6].payload[packet.IPv6].copy() + # Use dummy mac address that will be ignored in mask + pkt = packet.Ether(src="aa:bb:cc:dd:ee:ff", dst="aa:bb:cc:dd:ee:ff") / pkt + elif stage == "negative": + pkt = encapsulated_packet.copy() + + if ip_version == "IPv4": + pkt.ttl = pkt.ttl - 1 if decrease_ttl else pkt.ttl + elif ip_version == "IPv6": + pkt.hlim = pkt.hlim - 1 if decrease_ttl else pkt.hlim + + exp_pkt = Mask(pkt) + exp_pkt.set_do_not_care_packet(packet.Ether, "dst") + exp_pkt.set_do_not_care_packet(packet.Ether, "src") + if ip_version == "IPv4": + exp_pkt.set_do_not_care_packet(packet.IP, "chksum") + return exp_pkt + + +def verify_packet_with_expected(ptfadapter, stage, pkt, exp_pkt, send_port, + recv_ports=[], recv_port=None, timeout=10, skip_traffic_test=False): # noqa F811 + if skip_traffic_test is True: + logger.info("Skip traffic test") + return + ptfadapter.dataplane.flush() + testutils.send(ptfadapter, send_port, pkt) + if stage == "positive": + testutils.verify_packet_any_port(ptfadapter, exp_pkt, recv_ports, timeout=10) + elif stage == "negative": + testutils.verify_packet(ptfadapter, exp_pkt, recv_port, timeout=10) + + +@pytest.mark.parametrize("ip_version", ["IPv4", "IPv6"]) +@pytest.mark.parametrize("stage", ["positive", "negative"]) +def test_vlan_subnet_decap(request, rand_selected_dut, tbinfo, ptfhost, ptfadapter, ip_version, stage, + prepare_subnet_decap_config, prepare_vlan_subnet_test_port, + prepare_negative_ip_port_map, setup_arp_responder, skip_traffic_test): # noqa F811 + ptf_src_port, _, upstream_port_ids = prepare_vlan_subnet_test_port + + encapsulated_packet = build_encapsulated_vlan_subnet_packet(ptfadapter, rand_selected_dut, ip_version, stage) + exp_pkt = build_expected_vlan_subnet_packet(encapsulated_packet, ip_version, stage, decrease_ttl=True) + + if stage == "negative": + ptf_target_port, _ = prepare_negative_ip_port_map + request.getfixturevalue('setup_arp_responder') + else: + ptf_target_port = None + + verify_packet_with_expected(ptfadapter, stage, encapsulated_packet, exp_pkt, + ptf_src_port, recv_ports=upstream_port_ids, recv_port=ptf_target_port, + skip_traffic_test=skip_traffic_test) diff --git a/tests/dhcp_relay/conftest.py b/tests/dhcp_relay/conftest.py index 909c26b0e0..a06ec75f34 100644 --- a/tests/dhcp_relay/conftest.py +++ b/tests/dhcp_relay/conftest.py @@ -1,5 +1,39 @@ import pytest +from tests.common.utilities import wait_until +from tests.common.helpers.assertions import pytest_assert as py_assert +from tests.dhcp_relay.dhcp_relay_utils import check_routes_to_dhcp_server + +SINGLE_TOR_MODE = 'single' +DUAL_TOR_MODE = 'dual' + + +def pytest_addoption(parser): + """ + Adds options to pytest that are used by the COPP tests. + """ + parser.addoption( + "--stress_restart_round", + action="store", + type=int, + default=10, + help="Set custom restart rounds", + ) + parser.addoption( + "--stress_restart_duration", + action="store", + type=int, + default=90, + help="Set custom restart rounds", + ) + parser.addoption( + "--stress_restart_pps", + action="store", + type=int, + default=100, + help="Set custom restart rounds", + ) + @pytest.fixture(scope="module", autouse=True) def skip_dhcp_relay_tests(tbinfo): @@ -14,3 +48,126 @@ def skip_dhcp_relay_tests(tbinfo): """ if 'backend' in tbinfo['topo']['name']: pytest.skip("Skipping dhcp relay tests. Unsupported topology {}".format(tbinfo['topo']['name'])) + + +@pytest.fixture(scope="module", autouse=True) +def check_dhcp_server_enabled(duthost): + feature_status_output = duthost.show_and_parse("show feature status") + for feature in feature_status_output: + if feature["feature"] == "dhcp_server" and feature["state"] == "enabled": + pytest.skip("DHCPv4 relay is not supported when dhcp_server is enabled") + + +@pytest.fixture(scope="module") +def dut_dhcp_relay_data(duthosts, rand_one_dut_hostname, ptfhost, tbinfo): + """ Fixture which returns a list of dictionaries where each dictionary contains + data necessary to test one instance of a DHCP relay agent running on the DuT. + This fixture is scoped to the module, as the data it gathers can be used by + all tests in this module. It does not need to be run before each test. + """ + duthost = duthosts[rand_one_dut_hostname] + dhcp_relay_data_list = [] + + mg_facts = duthost.get_extended_minigraph_facts(tbinfo) + + switch_loopback_ip = mg_facts['minigraph_lo_interfaces'][0]['addr'] + + # SONiC spawns one DHCP relay agent per VLAN interface configured on the DUT + vlan_dict = mg_facts['minigraph_vlans'] + for vlan_iface_name, vlan_info_dict in list(vlan_dict.items()): + # Filter(remove) PortChannel interfaces from VLAN members list + vlan_members = [port for port in vlan_info_dict['members'] if 'PortChannel' not in port] + + # Gather information about the downlink VLAN interface this relay agent is listening on + downlink_vlan_iface = {} + downlink_vlan_iface['name'] = vlan_iface_name + + for vlan_interface_info_dict in mg_facts['minigraph_vlan_interfaces']: + if vlan_interface_info_dict['attachto'] == vlan_iface_name: + downlink_vlan_iface['addr'] = vlan_interface_info_dict['addr'] + downlink_vlan_iface['mask'] = vlan_interface_info_dict['mask'] + break + + # Obtain MAC address of the VLAN interface + res = duthost.shell('cat /sys/class/net/{}/address'.format(vlan_iface_name)) + downlink_vlan_iface['mac'] = res['stdout'] + + downlink_vlan_iface['dhcp_server_addrs'] = mg_facts['dhcp_servers'] + + # We choose the physical interface where our DHCP client resides to be index of first interface + # with alias (ignore PortChannel) in the VLAN + client_iface = {} + for port in vlan_members: + if port in mg_facts['minigraph_port_name_to_alias_map']: + break + else: + continue + client_iface['name'] = port + client_iface['alias'] = mg_facts['minigraph_port_name_to_alias_map'][client_iface['name']] + client_iface['port_idx'] = mg_facts['minigraph_ptf_indices'][client_iface['name']] + + # Obtain uplink port indicies for this DHCP relay agent + uplink_interfaces = [] + uplink_port_indices = [] + for iface_name, neighbor_info_dict in list(mg_facts['minigraph_neighbors'].items()): + if neighbor_info_dict['name'] in mg_facts['minigraph_devices']: + neighbor_device_info_dict = mg_facts['minigraph_devices'][neighbor_info_dict['name']] + if 'type' in neighbor_device_info_dict and neighbor_device_info_dict['type'] in \ + ['LeafRouter', 'MgmtLeafRouter']: + # If this uplink's physical interface is a member of a portchannel interface, + # we record the name of the portchannel interface here, as this is the actual + # interface the DHCP relay will listen on. + iface_is_portchannel_member = False + for portchannel_name, portchannel_info_dict in list(mg_facts['minigraph_portchannels'].items()): + if 'members' in portchannel_info_dict and iface_name in portchannel_info_dict['members']: + iface_is_portchannel_member = True + if portchannel_name not in uplink_interfaces: + uplink_interfaces.append(portchannel_name) + break + # If the uplink's physical interface is not a member of a portchannel, + # add it to our uplink interfaces list + if not iface_is_portchannel_member: + uplink_interfaces.append(iface_name) + uplink_port_indices.append(mg_facts['minigraph_ptf_indices'][iface_name]) + + other_client_ports_indices = [] + for iface_name in vlan_members: + if mg_facts['minigraph_ptf_indices'][iface_name] == client_iface['port_idx']: + pass + else: + other_client_ports_indices.append(mg_facts['minigraph_ptf_indices'][iface_name]) + + dhcp_relay_data = {} + dhcp_relay_data['downlink_vlan_iface'] = downlink_vlan_iface + dhcp_relay_data['client_iface'] = client_iface + dhcp_relay_data['other_client_ports'] = other_client_ports_indices + dhcp_relay_data['uplink_interfaces'] = uplink_interfaces + dhcp_relay_data['uplink_port_indices'] = uplink_port_indices + dhcp_relay_data['switch_loopback_ip'] = str(switch_loopback_ip) + + # Obtain MAC address of an uplink interface because vlan mac may be different than that of physical interfaces + res = duthost.shell('cat /sys/class/net/{}/address'.format(uplink_interfaces[0])) + dhcp_relay_data['uplink_mac'] = res['stdout'] + dhcp_relay_data['default_gw_ip'] = mg_facts['minigraph_mgmt_interface']['gwaddr'] + + dhcp_relay_data_list.append(dhcp_relay_data) + + return dhcp_relay_data_list + + +@pytest.fixture(scope="module") +def validate_dut_routes_exist(duthosts, rand_one_dut_hostname, dut_dhcp_relay_data): + """Fixture to valid a route to each DHCP server exist + """ + py_assert(wait_until(120, 5, 0, check_routes_to_dhcp_server, duthosts[rand_one_dut_hostname], + dut_dhcp_relay_data), "Failed to find route for DHCP server") + + +@pytest.fixture(scope="module") +def testing_config(duthosts, rand_one_dut_hostname, tbinfo): + duthost = duthosts[rand_one_dut_hostname] + + if 'dualtor' in tbinfo['topo']['name']: + yield DUAL_TOR_MODE, duthost + else: + yield SINGLE_TOR_MODE, duthost diff --git a/tests/dhcp_relay/dhcp_relay_utils.py b/tests/dhcp_relay/dhcp_relay_utils.py new file mode 100644 index 0000000000..c5fe89368f --- /dev/null +++ b/tests/dhcp_relay/dhcp_relay_utils.py @@ -0,0 +1,45 @@ +import ipaddress +import logging +from tests.common.utilities import wait_until +from tests.common.helpers.assertions import pytest_assert + +logger = logging.getLogger(__name__) + + +def check_routes_to_dhcp_server(duthost, dut_dhcp_relay_data): + """Validate there is route on DUT to each DHCP server + """ + default_gw_ip = dut_dhcp_relay_data[0]['default_gw_ip'] + dhcp_servers = set() + for dhcp_relay in dut_dhcp_relay_data: + dhcp_servers |= set(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs']) + + for dhcp_server in dhcp_servers: + rtInfo = duthost.get_ip_route_info(ipaddress.ip_address(dhcp_server)) + nexthops = rtInfo["nexthops"] + if len(nexthops) == 0: + logger.info("Failed to find route to DHCP server '{0}'".format(dhcp_server)) + return False + if len(nexthops) == 1: + # if only 1 route to dst available - check that it's not default route via MGMT iface + route_index_in_list = 0 + ip_dst_index = 0 + route_dst_ip = nexthops[route_index_in_list][ip_dst_index] + if route_dst_ip == ipaddress.ip_address(default_gw_ip): + logger.info("Found route to DHCP server via default GW(MGMT interface)") + return False + return True + + +def restart_dhcp_service(duthost): + duthost.shell('systemctl reset-failed dhcp_relay') + duthost.shell('systemctl restart dhcp_relay') + duthost.shell('systemctl reset-failed dhcp_relay') + + def _is_dhcp_relay_ready(): + output = duthost.shell('docker exec dhcp_relay supervisorctl status | grep dhcp | awk \'{print $2}\'', + module_ignore_errors=True) + return (not output['rc'] and output['stderr'] == '' and + all(element == 'RUNNING' for element in output['stdout_lines'])) + + pytest_assert(wait_until(60, 1, 0, _is_dhcp_relay_ready), "dhcp_relay is not ready after restarting") diff --git a/tests/dhcp_relay/test_dhcp_pkt_fwd.py b/tests/dhcp_relay/test_dhcp_pkt_fwd.py index dc954a19b4..7f8299075b 100644 --- a/tests/dhcp_relay/test_dhcp_pkt_fwd.py +++ b/tests/dhcp_relay/test_dhcp_pkt_fwd.py @@ -6,7 +6,6 @@ import ptf.testutils as testutils import ptf.packet as scapy -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 from ptf.mask import Mask from socket import INADDR_ANY diff --git a/tests/dhcp_relay/test_dhcp_pkt_recv.py b/tests/dhcp_relay/test_dhcp_pkt_recv.py index f4d1843c81..bcc41c5370 100644 --- a/tests/dhcp_relay/test_dhcp_pkt_recv.py +++ b/tests/dhcp_relay/test_dhcp_pkt_recv.py @@ -2,15 +2,16 @@ import ptf.packet as scapy import pytest import random +import re from ptf import testutils from scapy.layers.dhcp6 import DHCP6_Solicit +from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor # noqa F401 from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import capture_and_check_packet_on_dut -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ - pytest.mark.topology('mx') + pytest.mark.topology("t0", "m0", 'mx') ] ACL_TABLE_NAME_DHCPV6_PKT_RECV_TEST = "DHCPV6_PKT_RECV_TEST" @@ -28,7 +29,8 @@ @pytest.fixture(scope="module", autouse=True) -def check_dhcp_relay_feature_state(duthost): +def check_dhcp_relay_feature_state(rand_selected_dut): + duthost = rand_selected_dut features_state, _ = duthost.get_feature_status() if "enabled" not in features_state.get(DHCP_RELAY_FEATRUE_NAME, ""): pytest.skip('dhcp relay feature is not enabled, skip the test') @@ -37,17 +39,44 @@ def check_dhcp_relay_feature_state(duthost): class Dhcpv6PktRecvBase: @pytest.fixture(scope="class") - def setup_teardown(self, duthost, tbinfo): + def setup_teardown(self, rand_selected_dut, tbinfo): + duthost = rand_selected_dut + dut_index = str(tbinfo['duts_map'][duthost.hostname]) disabled_host_interfaces = tbinfo['topo']['properties']['topology'].get('disabled_host_interfaces', []) - ptf_indices = [interface for interface in tbinfo['topo']['properties']['topology'].get('host_interfaces', []) - if interface not in disabled_host_interfaces] + host_interfaces = [intf for intf in tbinfo['topo']['properties']['topology'].get('host_interfaces', []) + if intf not in disabled_host_interfaces] + ptf_indices = self.parse_ptf_indices(host_interfaces, dut_index) dut_intf_ptf_index = duthost.get_extended_minigraph_facts(tbinfo)['minigraph_ptf_indices'] yield ptf_indices, dut_intf_ptf_index - def test_dhcpv6_multicast_recv(self, duthost, ptfadapter, setup_teardown): + def parse_ptf_indices(self, host_interfaces, dut_index): + indices = list() + for _ports in host_interfaces: + # Example: ['0', '1', '2'] + # Example: ['0.0,1.0', '0.1,1.1', '0.2,1.2', ... ] + # Example: ['0.0@0,1.0@0', '0.1@1,1.1@1', '0.2@2,1.2@2', ... ] + for port in str(_ports).split(','): + m = re.match(r"(\d+)(?:\.(\d+))?(?:@(\d+))?", str(port).strip()) + m1, m2, m3 = m.groups() + if m3: + # Format: .@ + indices.append(int(m3)) if m1 == dut_index else None + elif m2: + # Format: . + indices.append(int(m2)) if m1 == dut_index else None + else: + # Format: + indices.append(int(m1)) + return indices + + def test_dhcpv6_multicast_recv(self, rand_selected_dut, + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + setup_standby_ports_on_rand_unselected_tor, + ptfadapter, setup_teardown): """ Test the DUT can receive DHCPv6 multicast packet """ + duthost = rand_selected_dut ptf_indices, dut_intf_ptf_index = setup_teardown ptf_index = random.choice(ptf_indices) intf, ptf_port_id = [(intf, id) for intf, id in dut_intf_ptf_index.items() if id == ptf_index][0] @@ -79,7 +108,8 @@ class TestDhcpv6WithEmptyAclTable(Dhcpv6PktRecvBase): Test the DUT with empty ACL table """ @pytest.fixture(scope="class", autouse=True) - def setup_teardown_acl(self, duthost, setup_teardown): + def setup_teardown_acl(self, rand_selected_dut, setup_teardown): + duthost = rand_selected_dut ptf_indices, dut_intf_ptf_index = setup_teardown ptf_intfs = [intf for intf, index in dut_intf_ptf_index.items() if index in ptf_indices] acl_table_name = ACL_TABLE_NAME_DHCPV6_PKT_RECV_TEST @@ -101,7 +131,8 @@ class TestDhcpv6WithMulticastAccpectAcl(Dhcpv6PktRecvBase): The drop all rule is added by default for L3V6 table type by acl-loader """ @pytest.fixture(scope="class", autouse=True) - def setup_teardown_acl(self, duthost, setup_teardown): + def setup_teardown_acl(self, rand_selected_dut, setup_teardown): + duthost = rand_selected_dut ptf_indices, dut_intf_ptf_index = setup_teardown ptf_intfs = [intf for intf, index in dut_intf_ptf_index.items() if index in ptf_indices] acl_table_name = ACL_TABLE_NAME_DHCPV6_PKT_RECV_TEST diff --git a/tests/dhcp_relay/test_dhcp_relay.py b/tests/dhcp_relay/test_dhcp_relay.py index bcf89bb550..d333252727 100644 --- a/tests/dhcp_relay/test_dhcp_relay.py +++ b/tests/dhcp_relay/test_dhcp_relay.py @@ -1,4 +1,3 @@ -import ipaddress import pytest import random import time @@ -8,6 +7,9 @@ from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F401 +from tests.common.gcu_utils import generate_tmpfile, create_checkpoint, \ + apply_patch, expect_op_success, delete_tmpfile, \ + rollback_or_reload, delete_checkpoint from tests.ptf_runner import ptf_runner from tests.common.utilities import wait_until from tests.common.helpers.dut_utils import check_link_status @@ -16,7 +18,7 @@ from tests.common import config_reload from tests.common.platform.processes_utils import wait_critical_processes from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer, LogAnalyzerError -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from tests.dhcp_relay.dhcp_relay_utils import check_routes_to_dhcp_server, restart_dhcp_service pytestmark = [ pytest.mark.topology('t0', 'm0'), @@ -31,14 +33,6 @@ logger = logging.getLogger(__name__) -@pytest.fixture(scope="module", autouse=True) -def check_dhcp_server_enabled(duthost): - feature_status_output = duthost.show_and_parse("show feature status") - for feature in feature_status_output: - if feature["feature"] == "dhcp_server" and feature["state"] == "enabled": - pytest.skip("DHCPv4 relay is not supported when dhcp_server is enabled") - - @pytest.fixture(autouse=True) def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer): """Ignore expected failures logs during test execution.""" @@ -53,212 +47,62 @@ def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer): yield -@pytest.fixture(scope="module") -def dut_dhcp_relay_data(duthosts, rand_one_dut_hostname, ptfhost, tbinfo): - """ Fixture which returns a list of dictionaries where each dictionary contains - data necessary to test one instance of a DHCP relay agent running on the DuT. - This fixture is scoped to the module, as the data it gathers can be used by - all tests in this module. It does not need to be run before each test. - """ - duthost = duthosts[rand_one_dut_hostname] - dhcp_relay_data_list = [] - - mg_facts = duthost.get_extended_minigraph_facts(tbinfo) - - switch_loopback_ip = mg_facts['minigraph_lo_interfaces'][0]['addr'] - - # SONiC spawns one DHCP relay agent per VLAN interface configured on the DUT - vlan_dict = mg_facts['minigraph_vlans'] - for vlan_iface_name, vlan_info_dict in list(vlan_dict.items()): - # Filter(remove) PortChannel interfaces from VLAN members list - vlan_members = [port for port in vlan_info_dict['members'] if 'PortChannel' not in port] - - # Gather information about the downlink VLAN interface this relay agent is listening on - downlink_vlan_iface = {} - downlink_vlan_iface['name'] = vlan_iface_name - - for vlan_interface_info_dict in mg_facts['minigraph_vlan_interfaces']: - if vlan_interface_info_dict['attachto'] == vlan_iface_name: - downlink_vlan_iface['addr'] = vlan_interface_info_dict['addr'] - downlink_vlan_iface['mask'] = vlan_interface_info_dict['mask'] - break - - # Obtain MAC address of the VLAN interface - res = duthost.shell('cat /sys/class/net/{}/address'.format(vlan_iface_name)) - downlink_vlan_iface['mac'] = res['stdout'] - - downlink_vlan_iface['dhcp_server_addrs'] = mg_facts['dhcp_servers'] - - # We choose the physical interface where our DHCP client resides to be index of first interface - # with alias (ignore PortChannel) in the VLAN - client_iface = {} - for port in vlan_members: - if port in mg_facts['minigraph_port_name_to_alias_map']: - break - else: - continue - client_iface['name'] = port - client_iface['alias'] = mg_facts['minigraph_port_name_to_alias_map'][client_iface['name']] - client_iface['port_idx'] = mg_facts['minigraph_ptf_indices'][client_iface['name']] - - # Obtain uplink port indicies for this DHCP relay agent - uplink_interfaces = [] - uplink_port_indices = [] - for iface_name, neighbor_info_dict in list(mg_facts['minigraph_neighbors'].items()): - if neighbor_info_dict['name'] in mg_facts['minigraph_devices']: - neighbor_device_info_dict = mg_facts['minigraph_devices'][neighbor_info_dict['name']] - if 'type' in neighbor_device_info_dict and neighbor_device_info_dict['type'] in \ - ['LeafRouter', 'MgmtLeafRouter']: - # If this uplink's physical interface is a member of a portchannel interface, - # we record the name of the portchannel interface here, as this is the actual - # interface the DHCP relay will listen on. - iface_is_portchannel_member = False - for portchannel_name, portchannel_info_dict in list(mg_facts['minigraph_portchannels'].items()): - if 'members' in portchannel_info_dict and iface_name in portchannel_info_dict['members']: - iface_is_portchannel_member = True - if portchannel_name not in uplink_interfaces: - uplink_interfaces.append(portchannel_name) - break - # If the uplink's physical interface is not a member of a portchannel, - # add it to our uplink interfaces list - if not iface_is_portchannel_member: - uplink_interfaces.append(iface_name) - uplink_port_indices.append(mg_facts['minigraph_ptf_indices'][iface_name]) - - other_client_ports_indices = [] - for iface_name in vlan_members: - if mg_facts['minigraph_ptf_indices'][iface_name] == client_iface['port_idx']: - pass - else: - other_client_ports_indices.append(mg_facts['minigraph_ptf_indices'][iface_name]) - - dhcp_relay_data = {} - dhcp_relay_data['downlink_vlan_iface'] = downlink_vlan_iface - dhcp_relay_data['client_iface'] = client_iface - dhcp_relay_data['other_client_ports'] = other_client_ports_indices - dhcp_relay_data['uplink_interfaces'] = uplink_interfaces - dhcp_relay_data['uplink_port_indices'] = uplink_port_indices - dhcp_relay_data['switch_loopback_ip'] = str(switch_loopback_ip) - - # Obtain MAC address of an uplink interface because vlan mac may be different than that of physical interfaces - res = duthost.shell('cat /sys/class/net/{}/address'.format(uplink_interfaces[0])) - dhcp_relay_data['uplink_mac'] = res['stdout'] - dhcp_relay_data['default_gw_ip'] = mg_facts['minigraph_mgmt_interface']['gwaddr'] +def check_interface_status(duthost): + if ":67" in duthost.shell("docker exec -t dhcp_relay ss -nlp | grep dhcrelay", + module_ignore_errors=True)["stdout"]: + return True - dhcp_relay_data_list.append(dhcp_relay_data) + return False - return dhcp_relay_data_list +@pytest.fixture(scope="function") +def enable_source_port_ip_in_relay(duthosts, rand_one_dut_hostname, tbinfo): + duthost = duthosts[rand_one_dut_hostname] -def check_routes_to_dhcp_server(duthost, dut_dhcp_relay_data): - """Validate there is route on DUT to each DHCP server """ - default_gw_ip = dut_dhcp_relay_data[0]['default_gw_ip'] - dhcp_servers = set() - for dhcp_relay in dut_dhcp_relay_data: - dhcp_servers |= set(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs']) - - for dhcp_server in dhcp_servers: - rtInfo = duthost.get_ip_route_info(ipaddress.ip_address(dhcp_server)) - nexthops = rtInfo["nexthops"] - if len(nexthops) == 0: - logger.info("Failed to find route to DHCP server '{0}'".format(dhcp_server)) - return False - if len(nexthops) == 1: - # if only 1 route to dst available - check that it's not default route via MGMT iface - route_index_in_list = 0 - ip_dst_index = 0 - route_dst_ip = nexthops[route_index_in_list][ip_dst_index] - if route_dst_ip == ipaddress.ip_address(default_gw_ip): - logger.info("Found route to DHCP server via default GW(MGMT interface)") - return False - return True - - -@pytest.fixture(scope="module") -def validate_dut_routes_exist(duthosts, rand_one_dut_hostname, dut_dhcp_relay_data): - """Fixture to valid a route to each DHCP server exist + Enable source port ip in relay function + -si parameter(Enable source port ip in relay function) will be added if deployment_id is '8', ref: + https://github.com/sonic-net/sonic-buildimage/blob/e0e0c0c1b3c58635bc25fde6a77ca3b0849dfde1/dockers/docker-dhcp-relay/dhcpv4-relay.agents.j2#L16 """ - pytest_assert(wait_until(120, 5, 0, check_routes_to_dhcp_server, duthosts[rand_one_dut_hostname], - dut_dhcp_relay_data), "Failed to find route for DHCP server") - - -def restart_dhcp_service(duthost): - duthost.shell('systemctl reset-failed dhcp_relay') - duthost.shell('systemctl restart dhcp_relay') - duthost.shell('systemctl reset-failed dhcp_relay') - - for retry in range(5): - time.sleep(30) - dhcp_status = duthost.shell('docker container top dhcp_relay | grep dhcrelay | cat')["stdout"] - if dhcp_status != "": - break - else: - assert False, "Failed to restart dhcp docker" - - time.sleep(30) - - -def get_subtype_from_configdb(duthost): - # HEXISTS returns 1 if the key exists, otherwise 0 - subtype_exist = int(duthost.shell('redis-cli -n 4 HEXISTS "DEVICE_METADATA|localhost" "subtype"')["stdout"]) - subtype_value = "" - if subtype_exist: - subtype_value = duthost.shell('redis-cli -n 4 HGET "DEVICE_METADATA|localhost" "subtype"')["stdout"] - return subtype_exist, subtype_value - - -@pytest.fixture(scope="module", params=[SINGLE_TOR_MODE, DUAL_TOR_MODE]) -def testing_config(request, duthosts, rand_one_dut_hostname, tbinfo): - testing_mode = request.param - duthost = duthosts[rand_one_dut_hostname] - subtype_exist, subtype_value = get_subtype_from_configdb(duthost) - - if 'dualtor' in tbinfo['topo']['name']: - if testing_mode == SINGLE_TOR_MODE: - pytest.skip("skip SINGLE_TOR_MODE tests on Dual ToR testbeds") - - if testing_mode == DUAL_TOR_MODE: - if not subtype_exist or subtype_value != 'DualToR': - assert False, "Wrong DHCP setup on Dual ToR testbeds" - - yield testing_mode, duthost, 'dual_testbed' - elif tbinfo['topo']['name'] in ('t0-54-po2vlan', 't0-56-po2vlan'): - if testing_mode == SINGLE_TOR_MODE: - if subtype_exist and subtype_value == 'DualToR': - assert False, "Wrong DHCP setup on po2vlan testbeds" - - yield testing_mode, duthost, 'single_testbed' - - if testing_mode == DUAL_TOR_MODE: - pytest.skip("skip DUAL_TOR_MODE tests on po2vlan testbeds") - else: - if testing_mode == DUAL_TOR_MODE: - pytest.skip("skip DUAL_TOR_MODE tests on Single ToR testbeds") - - if testing_mode == SINGLE_TOR_MODE: - if subtype_exist: - duthost.shell('redis-cli -n 4 HDEL "DEVICE_METADATA|localhost" "subtype"') - restart_dhcp_service(duthost) - - if testing_mode == DUAL_TOR_MODE: - if not subtype_exist or subtype_value != 'DualToR': - duthost.shell('redis-cli -n 4 HSET "DEVICE_METADATA|localhost" "subtype" "DualToR"') - restart_dhcp_service(duthost) - yield testing_mode, duthost, 'single_testbed' - - if testing_mode == DUAL_TOR_MODE: - duthost.shell('redis-cli -n 4 HDEL "DEVICE_METADATA|localhost" "subtype"') - restart_dhcp_service(duthost) - - -def check_interface_status(duthost): - if ":67" in duthost.shell("docker exec -t dhcp_relay ss -nlp | grep dhcrelay", - module_ignore_errors=True)["stdout"]: - return True + json_patch = [ + { + "op": "replace", + "path": "/DEVICE_METADATA/localhost/deployment_id", + "value": "8" + } + ] + + tmpfile = generate_tmpfile(duthost) + logger.info("tmpfile {}".format(tmpfile)) + check_point = "dhcp_relay" + try: + create_checkpoint(duthost, check_point) + output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) + expect_op_success(duthost, output) + restart_dhcp_service(duthost) - return False + def dhcp_ready(enable_source_port_ip_in_relay): + dhcp_relay_running = duthost.is_service_fully_started("dhcp_relay") + dhcp_relay_process = duthost.shell("ps -ef |grep dhcrelay|grep -v grep", + module_ignore_errors=True)["stdout"] + dhcp_mon_process = duthost.shell("ps -ef |grep dhcpmon|grep -v grep", + module_ignore_errors=True)["stdout"] + dhcp_mon_process_running = "dhcpmon" in dhcp_mon_process + if enable_source_port_ip_in_relay: + dhcp_relay_process_ready = "-si" in dhcp_relay_process and "dhcrelay" in dhcp_relay_process + else: + dhcp_relay_process_ready = "-si" not in dhcp_relay_process and "dhcrelay" in dhcp_relay_process + return dhcp_relay_running and dhcp_relay_process_ready and dhcp_mon_process_running + pytest_assert(wait_until(60, 2, 0, dhcp_ready, True), "Source port ip in relay is not enabled!") + yield + finally: + delete_tmpfile(duthost, tmpfile) + logger.info("Rolled back to original checkpoint") + rollback_or_reload(duthost, check_point) + delete_checkpoint(duthost, check_point) + restart_dhcp_service(duthost) + pytest_assert(wait_until(60, 2, 0, dhcp_ready, False), "Source port ip in relay is not disabled!") def test_interface_binding(duthosts, rand_one_dut_hostname, dut_dhcp_relay_data): @@ -313,7 +157,7 @@ def test_dhcp_relay_default(ptfhost, dut_dhcp_relay_data, validate_dut_routes_ex For each DHCP relay agent running on the DuT, verify DHCP packets are relayed properly """ - testing_mode, duthost, testbed_mode = testing_config + testing_mode, duthost = testing_config if testing_mode == DUAL_TOR_MODE: skip_release(duthost, ["201811", "201911"]) @@ -373,7 +217,6 @@ def test_dhcp_relay_default(ptfhost, dut_dhcp_relay_data, validate_dut_routes_ex "client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT, "switch_loopback_ip": dhcp_relay['switch_loopback_ip'], "uplink_mac": str(dhcp_relay['uplink_mac']), - "testbed_mode": testbed_mode, "testing_mode": testing_mode}, log_file="/tmp/dhcp_relay_test.DHCPTest.log", is_python3=True) if not skip_dhcpmon: @@ -394,18 +237,101 @@ def test_dhcp_relay_default(ptfhost, dut_dhcp_relay_data, validate_dut_routes_ex pytest_assert(wait_until(120, 5, 0, check_interface_status, duthost)) +def test_dhcp_relay_with_source_port_ip_in_relay_enabled(ptfhost, dut_dhcp_relay_data, + validate_dut_routes_exist, testing_config, + setup_standby_ports_on_rand_unselected_tor, # noqa F811 + rand_unselected_dut, toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 + enable_source_port_ip_in_relay): + """Test DHCP relay functionality on T0 topology. + For each DHCP relay agent running on the DuT, verify DHCP packets are relayed properly + """ + testing_mode, duthost = testing_config + + if testing_mode == DUAL_TOR_MODE: + skip_release(duthost, ["201811", "201911"]) + + skip_dhcpmon = any(vers in duthost.os_version for vers in ["201811", "201911", "202111"]) + + try: + for dhcp_relay in dut_dhcp_relay_data: + if not skip_dhcpmon: + dhcp_server_num = len(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs']) + if testing_mode == DUAL_TOR_MODE: + standby_duthost = rand_unselected_dut + start_dhcp_monitor_debug_counter(standby_duthost) + expected_standby_agg_counter_message = ( + r".*dhcp_relay#dhcpmon\[[0-9]+\]: " + r"\[\s*Agg-%s\s*-[\sA-Za-z0-9]+\s*rx/tx\] " + r"Discover: +0/ +0, Offer: +0/ +0, Request: +0/ +0, ACK: +0/ +0+" + ) % (dhcp_relay['downlink_vlan_iface']['name']) + loganalyzer_standby = LogAnalyzer(ansible_host=standby_duthost, marker_prefix="dhcpmon counter") + marker_standby = loganalyzer_standby.init() + loganalyzer_standby.expect_regex = [expected_standby_agg_counter_message] + start_dhcp_monitor_debug_counter(duthost) + if testing_mode == DUAL_TOR_MODE: + expected_agg_counter_message = ( + r".*dhcp_relay#dhcpmon\[[0-9]+\]: " + r"\[\s*Agg-%s\s*-[\sA-Za-z0-9]+\s*rx/tx\] " + r"Discover: +1/ +%d, Offer: +1/ +1, Request: +1/ +%d, ACK: +1/ +1+" + ) % (dhcp_relay['downlink_vlan_iface']['name'], dhcp_server_num, dhcp_server_num) + else: + expected_agg_counter_message = ( + r".*dhcp_relay#dhcpmon\[[0-9]+\]: " + r"\[\s*Agg-%s\s*-[\sA-Za-z0-9]+\s*rx/tx\] " + r"Discover: +1/ +%d, Offer: +1/ +1, Request: +2/ +%d, ACK: +1/ +1+" + ) % (dhcp_relay['downlink_vlan_iface']['name'], dhcp_server_num, dhcp_server_num * 2) + loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="dhcpmon counter") + marker = loganalyzer.init() + loganalyzer.expect_regex = [expected_agg_counter_message] + + # Run the DHCP relay test on the PTF host + ptf_runner(ptfhost, + "ptftests", + "dhcp_relay_test.DHCPTest", + platform_dir="ptftests", + params={"hostname": duthost.hostname, + "client_port_index": dhcp_relay['client_iface']['port_idx'], + # This port is introduced to test DHCP relay packet received + # on other client port + "other_client_port": repr(dhcp_relay['other_client_ports']), + "client_iface_alias": str(dhcp_relay['client_iface']['alias']), + "leaf_port_indices": repr(dhcp_relay['uplink_port_indices']), + "num_dhcp_servers": len(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs']), + "server_ip": dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs'], + "relay_iface_ip": str(dhcp_relay['downlink_vlan_iface']['addr']), + "relay_iface_mac": str(dhcp_relay['downlink_vlan_iface']['mac']), + "relay_iface_netmask": str(dhcp_relay['downlink_vlan_iface']['mask']), + "dest_mac_address": BROADCAST_MAC, + "client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT, + "switch_loopback_ip": dhcp_relay['switch_loopback_ip'], + "uplink_mac": str(dhcp_relay['uplink_mac']), + "testing_mode": testing_mode, + "enable_source_port_ip_in_relay": True}, + log_file="/tmp/dhcp_relay_test.DHCPTest.log", is_python3=True) + if not skip_dhcpmon: + time.sleep(36) # dhcpmon debug counter prints every 18 seconds + loganalyzer.analyze(marker) + if testing_mode == DUAL_TOR_MODE: + loganalyzer_standby.analyze(marker_standby) + except LogAnalyzerError as err: + logger.error("Unable to find expected log in syslog") + raise err + + if not skip_dhcpmon: + # Clean up - Restart DHCP relay service on DUT to recover original dhcpmon setting + restart_dhcp_service(duthost) + if testing_mode == DUAL_TOR_MODE: + restart_dhcp_service(standby_duthost) + pytest_assert(wait_until(120, 5, 0, check_interface_status, standby_duthost)) + pytest_assert(wait_until(120, 5, 0, check_interface_status, duthost)) + + def test_dhcp_relay_after_link_flap(ptfhost, dut_dhcp_relay_data, validate_dut_routes_exist, testing_config): """Test DHCP relay functionality on T0 topology after uplinks flap For each DHCP relay agent running on the DuT, with relay agent running, flap the uplinks, then test whether the DHCP relay agent relays packets properly. """ - testing_mode, duthost, testbed_mode = testing_config - - if testbed_mode == 'dual_testbed': - pytest.skip("skip the link flap testcase on dual tor testbeds") - - if testing_mode == DUAL_TOR_MODE: - skip_release(duthost, ["201811", "201911"]) + testing_mode, duthost = testing_config for dhcp_relay in dut_dhcp_relay_data: # Bring all uplink interfaces down @@ -441,7 +367,6 @@ def test_dhcp_relay_after_link_flap(ptfhost, dut_dhcp_relay_data, validate_dut_r "client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT, "switch_loopback_ip": dhcp_relay['switch_loopback_ip'], "uplink_mac": str(dhcp_relay['uplink_mac']), - "testbed_mode": testbed_mode, "testing_mode": testing_mode}, log_file="/tmp/dhcp_relay_test.DHCPTest.log", is_python3=True) @@ -452,13 +377,7 @@ def test_dhcp_relay_start_with_uplinks_down(ptfhost, dut_dhcp_relay_data, valida relay agent while the uplinks are still down. Then test whether the DHCP relay agent relays packets properly. """ - testing_mode, duthost, testbed_mode = testing_config - - if testbed_mode == 'dual_testbed': - pytest.skip("skip the uplinks down testcase on dual tor testbeds") - - if testing_mode == DUAL_TOR_MODE: - skip_release(duthost, ["201811", "201911"]) + testing_mode, duthost = testing_config for dhcp_relay in dut_dhcp_relay_data: # Bring all uplink interfaces down @@ -504,7 +423,6 @@ def test_dhcp_relay_start_with_uplinks_down(ptfhost, dut_dhcp_relay_data, valida "client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT, "switch_loopback_ip": dhcp_relay['switch_loopback_ip'], "uplink_mac": str(dhcp_relay['uplink_mac']), - "testbed_mode": testbed_mode, "testing_mode": testing_mode}, log_file="/tmp/dhcp_relay_test.DHCPTest.log", is_python3=True) @@ -515,10 +433,7 @@ def test_dhcp_relay_unicast_mac(ptfhost, dut_dhcp_relay_data, validate_dut_route """Test DHCP relay functionality on T0 topology with unicast mac Instead of using broadcast MAC, use unicast MAC of DUT and verify that DHCP relay functionality is entact. """ - testing_mode, duthost, testbed_mode = testing_config - - if testing_mode == DUAL_TOR_MODE: - skip_release(duthost, ["201811", "201911"]) + testing_mode, duthost = testing_config if len(dut_dhcp_relay_data) > 1: pytest.skip("skip the unicast mac testcase in the multi-Vlan setting") @@ -538,12 +453,11 @@ def test_dhcp_relay_unicast_mac(ptfhost, dut_dhcp_relay_data, validate_dut_route "relay_iface_ip": str(dhcp_relay['downlink_vlan_iface']['addr']), "relay_iface_mac": str(dhcp_relay['downlink_vlan_iface']['mac']), "relay_iface_netmask": str(dhcp_relay['downlink_vlan_iface']['mask']), - "dest_mac_address": duthost.facts["router_mac"] if testbed_mode != 'dual_testbed' + "dest_mac_address": duthost.facts["router_mac"] if testing_mode != DUAL_TOR_MODE else str(dhcp_relay['downlink_vlan_iface']['mac']), "client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT, "switch_loopback_ip": dhcp_relay['switch_loopback_ip'], "uplink_mac": str(dhcp_relay['uplink_mac']), - "testbed_mode": testbed_mode, "testing_mode": testing_mode}, log_file="/tmp/dhcp_relay_test.DHCPTest.log", is_python3=True) @@ -555,10 +469,7 @@ def test_dhcp_relay_random_sport(ptfhost, dut_dhcp_relay_data, validate_dut_rout If the client is SNAT'd, the source port could be changed to a non-standard port (i.e., not 68). Verify that DHCP relay works with random high sport. """ - testing_mode, duthost, testbed_mode = testing_config - - if testing_mode == DUAL_TOR_MODE: - skip_release(duthost, ["201811", "201911"]) + testing_mode, duthost = testing_config RANDOM_CLIENT_PORT = random.choice(list(range(1000, 65535))) for dhcp_relay in dut_dhcp_relay_data: @@ -580,7 +491,6 @@ def test_dhcp_relay_random_sport(ptfhost, dut_dhcp_relay_data, validate_dut_rout "client_udp_src_port": RANDOM_CLIENT_PORT, "switch_loopback_ip": dhcp_relay['switch_loopback_ip'], "uplink_mac": str(dhcp_relay['uplink_mac']), - "testbed_mode": testbed_mode, "testing_mode": testing_mode}, log_file="/tmp/dhcp_relay_test.DHCPTest.log", is_python3=True) @@ -620,7 +530,7 @@ def init_counter(duthost, ifname): def test_dhcp_relay_counter(ptfhost, dut_dhcp_relay_data, validate_dut_routes_exist, testing_config, setup_standby_ports_on_rand_unselected_tor, toggle_all_simulator_ports_to_rand_selected_tor_m): # noqa F811 - testing_mode, duthost, testbed_mode = testing_config + testing_mode, duthost = testing_config skip_release(duthost, ["201811", "201911", "202012"]) @@ -649,7 +559,6 @@ def test_dhcp_relay_counter(ptfhost, dut_dhcp_relay_data, validate_dut_routes_ex "client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT, "switch_loopback_ip": dhcp_relay['switch_loopback_ip'], "uplink_mac": str(dhcp_relay['uplink_mac']), - "testbed_mode": testbed_mode, "testing_mode": testing_mode}, log_file="/tmp/dhcp_relay_test_counter.DHCPTest.log", is_python3=True) for type in dhcp_message_types: diff --git a/tests/dhcp_relay/test_dhcp_relay_stress.py b/tests/dhcp_relay/test_dhcp_relay_stress.py new file mode 100644 index 0000000000..5b12051b7f --- /dev/null +++ b/tests/dhcp_relay/test_dhcp_relay_stress.py @@ -0,0 +1,97 @@ +import pytest +import time + +from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F401 +from tests.dhcp_relay.dhcp_relay_utils import restart_dhcp_service +from tests.common.helpers.assertions import pytest_assert, pytest_require +from tests.common.utilities import wait_until +from tests.ptf_runner import ptf_runner + +pytestmark = [ + pytest.mark.topology('t0', 'm0'), + pytest.mark.device_type('vs') +] + +BROADCAST_MAC = 'ff:ff:ff:ff:ff:ff' +DEFAULT_DHCP_CLIENT_PORT = 68 + + +def test_dhcp_relay_restart_with_stress(ptfhost, dut_dhcp_relay_data, validate_dut_routes_exist, testing_config, + request, setup_standby_ports_on_rand_unselected_tor, + toggle_all_simulator_ports_to_rand_selected_tor_m): # noqa F811 + """ + This test case is to make sure DHCPv4 relay would work well when startup with stress packets coming + """ + # Only test First Vlan + pytest_require(len(dut_dhcp_relay_data) >= 1, "Skip because cannot get enough vlan data") + testing_mode, duthost = testing_config + + # Unit: s, indicates duration time for sending stress packets + duration = request.config.getoption("--stress_restart_duration") + # Packets sending count per second + pps = request.config.getoption("--stress_restart_pps") + test_rounds = request.config.getoption("--stress_restart_round") + + for _ in range(test_rounds): + # Keep sending packets and then restart dhcp_relay + ptf_runner(ptfhost, "ptftests", "dhcp_relay_stress_test.DHCPContinuousStressTest", platform_dir="ptftests", + params={"hostname": duthost.hostname, + "client_port_index": dut_dhcp_relay_data[0]['client_iface']['port_idx'], + # This port is introduced to test DHCP relay packet received + # on other client port + "other_client_port": repr(dut_dhcp_relay_data[0]['other_client_ports']), + "leaf_port_indices": repr(dut_dhcp_relay_data[0]['uplink_port_indices']), + "num_dhcp_servers": len(dut_dhcp_relay_data[0]['downlink_vlan_iface']['dhcp_server_addrs']), + "server_ip": dut_dhcp_relay_data[0]['downlink_vlan_iface']['dhcp_server_addrs'], + "relay_iface_ip": str(dut_dhcp_relay_data[0]['downlink_vlan_iface']['addr']), + "relay_iface_mac": str(dut_dhcp_relay_data[0]['downlink_vlan_iface']['mac']), + "relay_iface_netmask": str(dut_dhcp_relay_data[0]['downlink_vlan_iface']['mask']), + "dest_mac_address": BROADCAST_MAC, + "client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT, + "switch_loopback_ip": dut_dhcp_relay_data[0]['switch_loopback_ip'], + "uplink_mac": str(dut_dhcp_relay_data[0]['uplink_mac']), + "testing_mode": testing_mode, + "duration": duration, + "pps": pps}, + log_file="/tmp/dhcp_relay_stress_test.DHCPContinuousStressTest.log", is_python3=True, + async_mode=True) + + restart_dhcp_service(duthost) + + # Wait packets send during and after dhcrelay starting + time.sleep(10) + # Make sure there is not stress packets sent + ptfhost.shell("kill -9 $(ps aux | grep DHCPContinuousStress | grep -v 'grep' | awk '{print $2}')", + module_ignore_errors=True) + + def _check_socket_buffer(): + output = duthost.shell('ss -nlpu | grep Vlan | awk \'{print $2}\'', + module_ignore_errors=True) + return (not output['rc'] and output['stderr'] == '' and len(output['stdout_lines']) != 0 and + all(element == '0' for element in output['stdout_lines'])) + + # Make sure there are not packets left in socket buffer. + pytest_assert(wait_until(30, 1, 0, _check_socket_buffer), "Socket buffer is not zero") + + # Run the DHCP relay test on the PTF host, make sure DHCPv4 relay is functionality good + ptf_runner(ptfhost, "ptftests", "dhcp_relay_test.DHCPTest", platform_dir="ptftests", + params={"hostname": duthost.hostname, + "client_port_index": dut_dhcp_relay_data[0]['client_iface']['port_idx'], + # This port is introduced to test DHCP relay packet received + # on other client port + "other_client_port": repr(dut_dhcp_relay_data[0]['other_client_ports']), + "client_iface_alias": str(dut_dhcp_relay_data[0]['client_iface']['alias']), + "leaf_port_indices": repr(dut_dhcp_relay_data[0]['uplink_port_indices']), + "num_dhcp_servers": + len(dut_dhcp_relay_data[0]['downlink_vlan_iface']['dhcp_server_addrs']), + "server_ip": dut_dhcp_relay_data[0]['downlink_vlan_iface']['dhcp_server_addrs'], + "relay_iface_ip": str(dut_dhcp_relay_data[0]['downlink_vlan_iface']['addr']), + "relay_iface_mac": str(dut_dhcp_relay_data[0]['downlink_vlan_iface']['mac']), + "relay_iface_netmask": str(dut_dhcp_relay_data[0]['downlink_vlan_iface']['mask']), + "dest_mac_address": BROADCAST_MAC, + "client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT, + "switch_loopback_ip": dut_dhcp_relay_data[0]['switch_loopback_ip'], + "uplink_mac": str(dut_dhcp_relay_data[0]['uplink_mac']), + "testing_mode": testing_mode}, + log_file="/tmp/dhcp_relay_test.stress.DHCPTest.log", is_python3=True) diff --git a/tests/dhcp_relay/test_dhcpv6_relay.py b/tests/dhcp_relay/test_dhcpv6_relay.py index dc28b82b9f..364f3421d7 100644 --- a/tests/dhcp_relay/test_dhcpv6_relay.py +++ b/tests/dhcp_relay/test_dhcpv6_relay.py @@ -16,7 +16,6 @@ from tests.common.dualtor.dual_tor_utils import config_active_active_dualtor_active_standby # noqa F401 from tests.common.dualtor.dual_tor_utils import validate_active_active_dualtor_setup # noqa F401 from tests.common.dualtor.dual_tor_common import active_active_ports # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('t0', 'm0', 'mx'), @@ -227,7 +226,33 @@ def check_interface_status(duthost): return False -def test_interface_binding(duthosts, rand_one_dut_hostname, dut_dhcp_relay_data): +def restart_dhcp_relay_and_check_dhcp6relay(duthost): + duthost.shell("sudo systemctl reset-failed dhcp_relay") + duthost.shell("sudo systemctl restart dhcp_relay") + wait_until(60, 3, 0, lambda: ("RUNNING" in duthost.shell("docker exec dhcp_relay supervisorctl status " + + "dhcp-relay:dhcp6relay | awk '{print $2}'")["stdout"])) + + +@pytest.fixture(scope="function") +def setup_and_teardown_no_servers_vlan(duthosts, rand_one_dut_hostname): + duthost = duthosts[rand_one_dut_hostname] + new_vlan_id = 4001 + new_vlan_ipv6 = "fc01:5000::1/64" + duthost.shell("sudo config vlan add {}".format(new_vlan_id)) + duthost.shell("sudo config interface ip add Vlan{} {}".format(new_vlan_id, new_vlan_ipv6)) + restart_dhcp_relay_and_check_dhcp6relay(duthost) + + yield new_vlan_id + + duthost.shell("sudo config interface ip remove Vlan{} {}".format(new_vlan_id, new_vlan_ipv6)) + duthost.shell("sudo config vlan del {}".format(new_vlan_id)) + restart_dhcp_relay_and_check_dhcp6relay(duthost) + + +def test_interface_binding(duthosts, rand_one_dut_hostname, dut_dhcp_relay_data, setup_and_teardown_no_servers_vlan): + # Add vlan without dhcpv6_server, which should not be binded + new_vlan_id = setup_and_teardown_no_servers_vlan + duthost = duthosts[rand_one_dut_hostname] skip_release(duthost, ["201911", "202106"]) if not check_interface_status(duthost): @@ -241,6 +266,10 @@ def test_interface_binding(duthosts, rand_one_dut_hostname, dut_dhcp_relay_data) "{} is not found in {}".format("*:{}".format(dhcp_relay['downlink_vlan_iface']['name']), output)) or \ ("*:*" in output, "dhcp6relay socket is not properly binded") + pytest_assert("Vlan{}".format(new_vlan_id) not in output, + "dhcp6relay bind to Vlan{} without dhcpv6_servers configured, which is unexpected" + .format(new_vlan_id)) + @pytest.fixture def setup_active_active_as_active_standby( diff --git a/tests/dhcp_server/conftest.py b/tests/dhcp_server/conftest.py index 6e77f4f5a1..125bfd1bad 100644 --- a/tests/dhcp_server/conftest.py +++ b/tests/dhcp_server/conftest.py @@ -2,6 +2,7 @@ from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert as py_assert from tests.common.helpers.assertions import pytest_require as py_require +from dhcp_server_test_common import clean_dhcp_server_config DHCP_RELAY_CONTAINER_NAME = "dhcp_relay" DHCP_SERVER_CONTAINER_NAME = "dhcp_server" @@ -43,3 +44,12 @@ def is_supervisor_subprocess_running(duthost, container_name, app_name): duthost.shell("config feature state dhcp_server disabled", module_ignore_errors=True) duthost.shell("sudo systemctl restart dhcp_relay.service") duthost.shell("docker rm dhcp_server", module_ignore_errors=True) + + +@pytest.fixture(scope="function", autouse=True) +def clean_dhcp_server_config_after_test(duthost): + clean_dhcp_server_config(duthost) + + yield + + clean_dhcp_server_config(duthost) diff --git a/tests/dhcp_server/dhcp_server_test_common.py b/tests/dhcp_server/dhcp_server_test_common.py index 629c5ae686..5b7407ccab 100644 --- a/tests/dhcp_server/dhcp_server_test_common.py +++ b/tests/dhcp_server/dhcp_server_test_common.py @@ -6,8 +6,7 @@ import pytest import ptf.packet as scapy import ptf.testutils as testutils -import time -from tests.common.utilities import capture_and_check_packet_on_dut +from tests.common.utilities import capture_and_check_packet_on_dut, wait_until from tests.common.helpers.assertions import pytest_assert, pytest_require @@ -38,6 +37,20 @@ ) +def vlan_i2n(vlan_id): + """ + Convert vlan id to vlan name + """ + return "Vlan%s" % vlan_id + + +def vlan_n2i(vlan_name): + """ + Convert vlan name to vlan id + """ + return vlan_name.replace("Vlan", "") + + def clean_fdb_table(duthost): duthost.shell("sonic-clear fdb all") @@ -48,12 +61,33 @@ def ping_dut_refresh_fdb(ptfhost, interface): def clean_dhcp_server_config(duthost): keys = duthost.shell("sonic-db-cli CONFIG_DB KEYS DHCP_SERVER_IPV4*") - for key in keys["stdout_lines"]: - duthost.shell("sonic-db-cli CONFIG_DB DEL '{}'".format(key)) + clean_order = [ + "DHCP_SERVER_IPV4_CUSTOMIZED_OPTIONS", + "DHCP_SERVER_IPV4_RANGE", + "DHCP_SERVER_IPV4_PORT", + "DHCP_SERVER_IPV4" + ] + for key in clean_order: + for line in keys['stdout_lines']: + if line.startswith(key + '|'): + duthost.shell("sonic-db-cli CONFIG_DB DEL '{}'".format(line)) def verify_lease(duthost, dhcp_interface, client_mac, exp_ip, exp_lease_time): - time.sleep(5) # wait for dhcp server to update lease info + pytest_assert( + wait_until( + 11, # it's by design that there is a latency around 0~11 seconds for updating state db + 1, + 3, + lambda _dh, _di, _cm: len(_dh.shell( + "sonic-db-cli STATE_DB KEYS 'DHCP_SERVER_IPV4_LEASE|{}|{}'".format(_di, _cm) + )['stdout']) > 0, + duthost, + dhcp_interface, + client_mac + ), + 'state db doesnt have lease info for client {}'.format(client_mac) + ) lease_start = duthost.shell("sonic-db-cli STATE_DB HGET 'DHCP_SERVER_IPV4_LEASE|{}|{}' '{}'" .format(dhcp_interface, client_mac, STATE_DB_KEY_LEASE_START))['stdout'] lease_end = duthost.shell("sonic-db-cli STATE_DB HGET 'DHCP_SERVER_IPV4_LEASE|{}|{}' '{}'" @@ -127,11 +161,14 @@ def empty_config_patch(customized_options=None): } ] if customized_options: - ret_empty_patch.append({ - "op": "add", - "path": "/DHCP_SERVER_IPV4_CUSTOMIZED_OPTIONS", - "value": {} - }) + ret_empty_patch.insert( + 0, + { + "op": "add", + "path": "/DHCP_SERVER_IPV4_CUSTOMIZED_OPTIONS", + "value": {} + } + ) return ret_empty_patch @@ -145,12 +182,13 @@ def append_common_config_patch( customized_options=None ): pytest_require(len(dut_ports) == len(ip_ranges), "Invalid input, dut_ports and ip_ranges should have same length") - new_patch = generate_dhcp_interface_config_patch(vlan_name, gateway, net_mask, customized_options) + new_patch = [] + if customized_options: + new_patch += generate_dhcp_custom_option_config_patch(customized_options) + new_patch += generate_dhcp_interface_config_patch(vlan_name, gateway, net_mask, customized_options) range_names = ["range_" + ip_range[0] for ip_range in ip_ranges] new_patch += generate_dhcp_range_config_patch(ip_ranges, range_names) new_patch += generate_dhcp_port_config_patch(vlan_name, dut_ports, range_names) - if customized_options: - new_patch += generate_dhcp_custom_option_config_patch(customized_options) config_patch += new_patch @@ -425,11 +463,22 @@ def verify_discover_and_request_then_release( ) if expected_assigned_ip and release_needed: verify_lease(duthost, dhcp_interface, client_mac, expected_assigned_ip, exp_lease_time) - release_pkt = create_dhcp_client_packet( - src_mac=client_mac, - message_type=DHCP_MESSAGE_TYPE_RELEASE_NUM, - client_options=[("server_id", exp_gateway)], - xid=test_xid, - ciaddr=expected_assigned_ip - ) - testutils.send_packet(ptfadapter, ptf_port_index, release_pkt) + send_release_packet(ptfadapter, ptf_port_index, test_xid, client_mac, expected_assigned_ip, server_id) + + +def send_release_packet( + ptfadapter, + ptf_port_index, + xid, + client_mac, + ip_assigned, + server_id +): + release_pkt = create_dhcp_client_packet( + src_mac=client_mac, + message_type=DHCP_MESSAGE_TYPE_RELEASE_NUM, + client_options=[("server_id", server_id)], + xid=xid, + ciaddr=ip_assigned + ) + testutils.send_packet(ptfadapter, ptf_port_index, release_pkt) diff --git a/tests/dhcp_server/test_dhcp_server.py b/tests/dhcp_server/test_dhcp_server.py index de9f93036e..9ab9046f7f 100644 --- a/tests/dhcp_server/test_dhcp_server.py +++ b/tests/dhcp_server/test_dhcp_server.py @@ -6,10 +6,10 @@ from tests.common.helpers.assertions import pytest_assert from dhcp_server_test_common import DHCP_SERVER_CONFIG_TOOL_GCU, DHCP_SERVER_CONFIG_TOOL_CLI, \ create_common_config_patch, generate_common_config_cli_commands, dhcp_server_config, \ - validate_dhcp_server_pkts_custom_option, verify_lease, \ + validate_dhcp_server_pkts_custom_option, verify_lease, send_release_packet, \ verify_discover_and_request_then_release, send_and_verify, DHCP_MESSAGE_TYPE_DISCOVER_NUM, \ DHCP_SERVER_SUPPORTED_OPTION_ID, DHCP_MESSAGE_TYPE_REQUEST_NUM, DHCP_DEFAULT_LEASE_TIME, \ - apply_dhcp_server_config_gcu, create_dhcp_client_packet + apply_dhcp_server_config_gcu, create_dhcp_client_packet, vlan_n2i pytestmark = [ @@ -498,6 +498,8 @@ def test_dhcp_server_port_based_customize_options( pkts_validator_kwargs=pkts_validator_kwargs, refresh_fdb_ptf_port='eth'+str(ptf_port_index) ) + verify_lease(duthost, vlan_name, client_mac, expected_assigned_ip, DHCP_DEFAULT_LEASE_TIME) + send_release_packet(ptfadapter, ptf_port_index, test_xid, client_mac, expected_assigned_ip, gateway) def test_dhcp_server_config_change_dhcp_interface( @@ -596,8 +598,8 @@ def test_dhcp_server_config_change_common( (changed_expected_assigned_ip, changed_gateway, changed_lease_time)) change_to_apply = [ { - "op": "replace", - "path": "/DHCP_SERVER_IPV4_RANGE/%s/range/0" % ("range_" + expected_assigned_ip), + "op": "add", + "path": "/DHCP_SERVER_IPV4_RANGE/%s/range/1" % ("range_" + expected_assigned_ip), "value": "%s" % changed_expected_assigned_ip }, { @@ -612,6 +614,14 @@ def test_dhcp_server_config_change_common( } ] apply_dhcp_server_config_gcu(duthost, change_to_apply) + change_to_apply = [ + { + "op": "remove", + "path": "/DHCP_SERVER_IPV4_RANGE/%s/range/0" % ("range_" + expected_assigned_ip), + "value": "%s" % expected_assigned_ip + } + ] + apply_dhcp_server_config_gcu(duthost, change_to_apply) verify_discover_and_request_then_release( duthost=duthost, ptfhost=ptfhost, @@ -633,11 +643,13 @@ def test_dhcp_server_config_vlan_member_change( duthost, ptfhost, ptfadapter, - parse_vlan_setting_from_running_config + parse_vlan_setting_from_running_config, + loganalyzer ): """ Test if config change on dhcp interface status can take effect """ + loganalyzer[duthost.hostname].ignore_regex.append(".*Failed to get port by bridge port.*") test_xid = 11 vlan_name, gateway, net_mask, vlan_hosts, vlan_members_with_ptf_idx = parse_vlan_setting_from_running_config expected_assigned_ip = random.choice(vlan_hosts) @@ -647,38 +659,28 @@ def test_dhcp_server_config_vlan_member_change( config_to_apply = create_common_config_patch(vlan_name, gateway, net_mask, [dut_port], [[expected_assigned_ip]]) apply_dhcp_server_config_gcu(duthost, config_to_apply) # delete member - config_to_apply = [ - { - "op": "remove", - "path": "/VLAN_MEMBER/%s|%s" % (vlan_name, dut_port) - } - ] - apply_dhcp_server_config_gcu(duthost, config_to_apply) - verify_discover_and_request_then_release( - duthost=duthost, - ptfhost=ptfhost, - ptfadapter=ptfadapter, - dut_port_to_capture_pkt=dut_port, - ptf_port_index=ptf_port_index, - ptf_mac_port_index=ptf_port_index, - test_xid=test_xid, - dhcp_interface=vlan_name, - expected_assigned_ip=None, - exp_gateway=gateway, - server_id=gateway, - net_mask=net_mask - ) + duthost.del_member_from_vlan(vlan_n2i(vlan_name), dut_port) + try: + verify_discover_and_request_then_release( + duthost=duthost, + ptfhost=ptfhost, + ptfadapter=ptfadapter, + dut_port_to_capture_pkt=dut_port, + ptf_port_index=ptf_port_index, + ptf_mac_port_index=ptf_port_index, + test_xid=test_xid, + dhcp_interface=vlan_name, + expected_assigned_ip=None, + exp_gateway=gateway, + server_id=gateway, + net_mask=net_mask + ) + except Exception as e: + duthost.add_member_to_vlan(vlan_n2i(vlan_name), dut_port) + raise e + # restore deleted member - config_to_apply = [ - { - "op": "add", - "path": "/VLAN_MEMBER/%s|%s" % (vlan_name, dut_port), - "value": { - "tagging_mode": "untagged" - } - } - ] - apply_dhcp_server_config_gcu(duthost, config_to_apply) + duthost.add_member_to_vlan(vlan_n2i(vlan_name), dut_port, False) time.sleep(3) # wait for vlan member change take effect verify_discover_and_request_then_release( duthost=duthost, @@ -740,6 +742,7 @@ def test_dhcp_server_lease_config_change( apply_dhcp_server_config_gcu(duthost, change_to_apply) client_mac = ptfadapter.dataplane.get_mac(0, ptf_port_index).decode('utf-8') verify_lease(duthost, vlan_name, client_mac, expected_assigned_ip, DHCP_DEFAULT_LEASE_TIME) + send_release_packet(ptfadapter, ptf_port_index, test_xid, client_mac, expected_assigned_ip, gateway) def test_dhcp_server_config_vlan_intf_change( @@ -777,31 +780,8 @@ def test_dhcp_server_config_vlan_intf_change( apply_dhcp_server_config_gcu(duthost, config_to_apply) # When the subnet not match to VLAN, client won't get IP - patch_replace_subnet = [ - { - "op": "remove", - "path": "/VLAN_INTERFACE/%s|%s" % (vlan_name_1, vlan_ipv4_1.replace('/', '~1')) - }, - { - "op": "add", - "path": "/VLAN_INTERFACE/%s|%s" % (vlan_name_1, vlan_ipv4_2.replace('/', '~1')), - "value": {} - } - ] - - # When the subnet is changed to match VLAN, client can get IP - patch_restore_subnet = [ - { - "op": "remove", - "path": "/VLAN_INTERFACE/%s|%s" % (vlan_name_1, vlan_ipv4_2.replace('/', '~1')) - }, - { - "op": "add", - "path": "/VLAN_INTERFACE/%s|%s" % (vlan_name_1, vlan_ipv4_1.replace('/', '~1')), - "value": {} - } - ] - apply_dhcp_server_config_gcu(duthost, patch_replace_subnet) + duthost.add_ip_addr_to_vlan(vlan_name_1, vlan_ipv4_2) + duthost.remove_ip_addr_from_vlan(vlan_name_1, vlan_ipv4_1) try: verify_discover_and_request_then_release( duthost=duthost, @@ -818,10 +798,13 @@ def test_dhcp_server_config_vlan_intf_change( net_mask=None ) except Exception as e: - apply_dhcp_server_config_gcu(duthost, patch_restore_subnet) + duthost.add_ip_addr_to_vlan(vlan_name_1, vlan_ipv4_1) + duthost.remove_ip_addr_from_vlan(vlan_name_1, vlan_ipv4_2) raise e - apply_dhcp_server_config_gcu(duthost, patch_restore_subnet) + # When the subnet is changed to match VLAN, client can get IP + duthost.add_ip_addr_to_vlan(vlan_name_1, vlan_ipv4_1) + duthost.remove_ip_addr_from_vlan(vlan_name_1, vlan_ipv4_2) verify_discover_and_request_then_release( duthost=duthost, ptfhost=ptfhost, diff --git a/tests/dhcp_server/test_dhcp_server_multi_vlans.py b/tests/dhcp_server/test_dhcp_server_multi_vlans.py index 72ab3de380..2d6d25e825 100644 --- a/tests/dhcp_server/test_dhcp_server_multi_vlans.py +++ b/tests/dhcp_server/test_dhcp_server_multi_vlans.py @@ -4,7 +4,8 @@ import random from tests.common.helpers.assertions import pytest_assert from dhcp_server_test_common import create_common_config_patch, append_common_config_patch, \ - verify_discover_and_request_then_release, apply_dhcp_server_config_gcu, empty_config_patch + verify_discover_and_request_then_release, apply_dhcp_server_config_gcu, empty_config_patch, \ + vlan_n2i pytestmark = [ @@ -115,20 +116,6 @@ def generate_four_vlans_config_patch(vlan_name, vlan_info, vlan_member_with_ptf_ return four_vlans_info, patch_setup, patch_restore -def vlan_i2n(vlan_id): - """ - Convert vlan id to vlan name - """ - return "Vlan%s" % vlan_id - - -def vlan_n2i(vlan_name): - """ - Convert vlan name to vlan id - """ - return vlan_name.replace("Vlan", "") - - def add_vlan_patch(vlan_name): patch = [ { @@ -271,7 +258,7 @@ def test_range_ip_assignment( vlan_name_1, gateway_1, net_mask_1, vlan_hosts_1, vlan_members_with_ptf_idx_1 = vlan_info_1['vlan_name'], \ vlan_info_1['vlan_gateway'], vlan_info_1['vlan_subnet_mask'], vlan_info_1['vlan_hosts'], \ vlan_info_1['members_with_ptf_idx'] - expected_assigned_ip_1 = random.choice(vlan_hosts_1) + expected_assigned_ip_1 = random.choice(vlan_hosts_1[:-1]) last_ip_in_range_1 = random.choice(vlan_hosts_1[vlan_hosts_1.index(expected_assigned_ip_1) + 1:]) dut_port_1, ptf_port_index_1 = random.choice(vlan_members_with_ptf_idx_1) logging.info("expected_assigned_ip_1 is %s, last_ip_in_range_1 is %s, dut_port_1 is %s, ptf_port_index_1 is %s" % @@ -281,7 +268,7 @@ def test_range_ip_assignment( vlan_name_2, gateway_2, net_mask_2, vlan_hosts_2, vlan_members_with_ptf_idx_2 = vlan_info_2['vlan_name'], \ vlan_info_2['vlan_gateway'], vlan_info_2['vlan_subnet_mask'], vlan_info_2['vlan_hosts'], \ vlan_info_2['members_with_ptf_idx'] - expected_assigned_ip_2 = random.choice(vlan_hosts_2) + expected_assigned_ip_2 = random.choice(vlan_hosts_2[:-1]) last_ip_in_range_2 = random.choice(vlan_hosts_2[vlan_hosts_2.index(expected_assigned_ip_2) + 1:]) dut_port_2, ptf_port_index_2 = random.choice(vlan_members_with_ptf_idx_2) logging.info("expected_assigned_ip_2 is %s, last_ip_in_range_2 is %s, dut_port_2 is %s, ptf_port_index_2 is %s" % diff --git a/tests/dhcp_server/test_dhcp_server_stress.py b/tests/dhcp_server/test_dhcp_server_stress.py new file mode 100644 index 0000000000..9046f2604c --- /dev/null +++ b/tests/dhcp_server/test_dhcp_server_stress.py @@ -0,0 +1,106 @@ +import logging +import ipaddress +import pytest +from tests.common.utilities import wait_until +from tests.common.helpers.assertions import pytest_assert +from dhcp_server_test_common import apply_dhcp_server_config_gcu, empty_config_patch, append_common_config_patch + + +pytestmark = [ + pytest.mark.topology('mx'), +] + + +@pytest.fixture(scope="module", autouse=True) +def dhcp_client_setup_teardown_on_ptf(ptfhost, creds): + http_proxy = creds.get("proxy_env", {}).get("http_proxy", "") + http_param = "-o Acquire::http::proxy='{}'".format(http_proxy) if http_proxy != "" else "" + ptfhost.shell("apt-get {} update".format(http_param), module_ignore_errors=True) + ptfhost.shell("apt-get {} install isc-dhcp-client -y".format(http_param)) + + yield + + ptfhost.shell("apt-get remove isc-dhcp-client -y", module_ignore_errors=True) + + +@pytest.fixture(scope="module") +def parse_vlan_setting_from_running_config(duthost, tbinfo): + vlan_brief = duthost.get_vlan_brief() + first_vlan_name = list(vlan_brief.keys())[0] + first_vlan_info = list(vlan_brief.values())[0] + first_vlan_prefix = first_vlan_info['interface_ipv4'][0] + disabled_host_interfaces = tbinfo['topo']['properties']['topology'].get('disabled_host_interfaces', []) + connected_ptf_ports_idx = [interface for interface in + tbinfo['topo']['properties']['topology'].get('host_interfaces', []) + if interface not in disabled_host_interfaces] + dut_intf_to_ptf_index = duthost.get_extended_minigraph_facts(tbinfo)['minigraph_ptf_indices'] + connected_dut_intf_to_ptf_index = {k: v for k, v in dut_intf_to_ptf_index.items() if v in connected_ptf_ports_idx} + vlan_members = first_vlan_info['members'] + vlan_member_with_ptf_idx = [(member, connected_dut_intf_to_ptf_index[member]) + for member in vlan_members if member in connected_dut_intf_to_ptf_index] + pytest_assert(len(vlan_member_with_ptf_idx) >= 2, 'Vlan members is too little for testing') + vlan_net = ipaddress.ip_network(address=first_vlan_prefix, strict=False) + vlan_gateway = first_vlan_prefix.split('/')[0] + vlan_hosts = [str(host) for host in vlan_net.hosts()] + # to avoid configurate an range contains gateway ip, simply ignore all ip before gateway and gateway itself + vlan_hosts_after_gateway = vlan_hosts[vlan_hosts.index(vlan_gateway) + 1:] + pytest_assert(len(vlan_hosts_after_gateway) >= 2, 'Vlan size is too small for testing') + vlan_setting = { + 'vlan_name': first_vlan_name, + 'vlan_gateway': vlan_gateway, + 'vlan_subnet_mask': str(vlan_net.netmask), + 'vlan_hosts': vlan_hosts_after_gateway, + 'vlan_member_with_ptf_idx': vlan_member_with_ptf_idx, + } + + logging.info("The vlan_setting before test is %s" % vlan_setting) + return vlan_setting['vlan_name'], \ + vlan_setting['vlan_gateway'], \ + vlan_setting['vlan_subnet_mask'], \ + vlan_setting['vlan_hosts'], \ + vlan_setting['vlan_member_with_ptf_idx'] + + +def test_dhcp_server_with_multiple_dhcp_clients( + duthost, + ptfhost, + parse_vlan_setting_from_running_config +): + """ + Make sure all ports can get assigend ip when all ports request ip at same time + """ + vlan_name, gateway, net_mask, vlan_hosts, vlan_members_with_ptf_idx = parse_vlan_setting_from_running_config + start_command = " && ".join(["dhclient -nw eth%s" % ptf_index for _, ptf_index in vlan_members_with_ptf_idx]) + end_command = " ; ".join(["dhclient -r eth%s" % ptf_index for _, ptf_index in vlan_members_with_ptf_idx]) + try: + config_to_apply = empty_config_patch() + dut_ports, _ = zip(*vlan_members_with_ptf_idx) + exp_assigned_ip_ranges = [[ip] for ip in vlan_hosts[:len(vlan_members_with_ptf_idx)]] + append_common_config_patch( + config_to_apply, + vlan_name, + gateway, + net_mask, + dut_ports, + exp_assigned_ip_ranges + ) + apply_dhcp_server_config_gcu(duthost, config_to_apply) + ptfhost.shell(start_command) + + def all_ip_shown_up(ptfhost, expected_assigned_ips): + ip_addr_output = ptfhost.shell("ip addr")['stdout'] + for expected_ip in expected_assigned_ips: + if expected_ip not in ip_addr_output: + return False + return True + expected_assigned_ips = [range[0] for range in exp_assigned_ip_ranges] + pytest_assert( + wait_until(20, 1, 1, + all_ip_shown_up, + ptfhost, + expected_assigned_ips), + 'Not all configurated IP shown up on ptf interfaces' + ) + finally: + ptfhost.shell(end_command, module_ignore_errors=True) + ptfhost.shell("killall dhclient", module_ignore_errors=True) diff --git a/tests/disk/test_disk_exhaustion.py b/tests/disk/test_disk_exhaustion.py index 77c6a55fc8..5a08df81fc 100644 --- a/tests/disk/test_disk_exhaustion.py +++ b/tests/disk/test_disk_exhaustion.py @@ -9,7 +9,6 @@ from ptf import mask, packet from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import paramiko_ssh -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/dns/test_dns_resolv_conf.py b/tests/dns/test_dns_resolv_conf.py index 14f807f8dd..6615abcb4a 100644 --- a/tests/dns/test_dns_resolv_conf.py +++ b/tests/dns/test_dns_resolv_conf.py @@ -3,7 +3,6 @@ from tests.common.constants import RESOLV_CONF_NAMESERVERS from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import get_image_type -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology("any") @@ -38,3 +37,21 @@ def test_dns_resolv_conf(duthost): pytest_assert(not (current_nameservers ^ expected_nameservers), "Mismatch between expected and current nameservers! Expected: [{}]. Current: [{}].".format( " ".join(expected_nameservers), " ".join(current_nameservers))) + + containers = duthost.get_running_containers() + for container in containers: + resolv_conf = duthost.shell("docker exec %s cat /etc/resolv.conf" % container, module_ignore_errors=True) + pytest_assert(resolv_conf["rc"] == 0, "Failed to read /etc/resolf.conf!") + current_nameservers = [] + for resolver_line in resolv_conf["stdout_lines"]: + if not resolver_line.startswith("nameserver"): + continue + current_nameservers.append(resolver_line.split()[1]) + + current_nameservers = set(current_nameservers) + + logger.info("{} container, current nameservers: [{}]".format(container, " ".join(current_nameservers))) + + pytest_assert(not (current_nameservers ^ expected_nameservers), + "Mismatch between expected and current nameservers for {}! Expected: [{}]. Current: [{}].".format( + container, " ".join(expected_nameservers), " ".join(current_nameservers))) diff --git a/tests/docs/asic_db_validation.md b/tests/docs/asic_db_validation.md new file mode 100644 index 0000000000..058e47461b --- /dev/null +++ b/tests/docs/asic_db_validation.md @@ -0,0 +1,37 @@ +## ASIC DB Validation for SONiC Management Tests +## Overview of Design + +## Purpose of this Document + +In order to facilitate validating entries from ASIC DB, helper methods and Pytest fixtures have been added. This document outlines the available functionality to access ASIC DB and other Redis database objects via sonic-db-cli. + +## Methodology + +The enhancements to test utilities and fixtures for ASIC DB validation leverage the existing `sonic-db-cli` wrappers and modules detailed in [../common/helpers/sonic_db.py](../common/helpers/sonic_db.py) within the SONiC Management testing framework. It also introduces PyTest fixtures to facilitate ASIC db access. + +## Application + +The `sonic_db.py` [../common/helpers/sonic_db.py](../common/helpers/sonic_db.py) module offers a set of classes with various methods for interfacing with ASIC DB and other database objects using `sonic-db-cli`. Test scripts can utilize fixtures to interact with the `AsicDbCli` methods, allowing them to retrieve information from different ASIC_DB tables. The following fixtures are available: + +- `asic_db_dut` provides an `AsicDbCli` instance for the Device Under Test (DUT). +- `asic_db_dut_rand` offers a random `AsicDbCli` instance for setups with Multi-ASIC configurations. +- `asic_db_supervisor` grants access to an `AsicDbCli` instance from the supervisor SONiC instance in a disaggregated chassis setup. + +`AsicDbCli` class allows caller to access various ASIC_DB object types. Access to the following object types are supported. Access to additional object types, key-values and complex querying can be incorporated into `AsicDbCli` using calls to `SonicDbCli` methods. + +### Example + +``` +def test_asicdb_duthost(asic_db_dut): + key = asic_db_dut.get_switch_key() + logger.info(f'switch key from asic db for duthost {key}') + host_if_list = asic_db_dut.get_hostif_list() + logger.info(f'duthost host interfaces {host_if_list}') + assert host_if_list is not None + assert key is not None + +def test_asic_db_get_nexthop_entries(asic_db_dut): + nexthop_entries = asic_db_dut.get_next_hop_entries() + logger.info(f'nexthop entries = {nexthop_entries}') + assert nexthop_entries is not None +``` diff --git a/tests/drop_packets/drop_packets.py b/tests/drop_packets/drop_packets.py index 6514ecd70e..9948968af0 100644 --- a/tests/drop_packets/drop_packets.py +++ b/tests/drop_packets/drop_packets.py @@ -15,7 +15,8 @@ from tests.common.helpers.constants import DEFAULT_NAMESPACE from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer, LogAnalyzerError from tests.common import config_reload -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 +from tests.common.helpers.dut_utils import is_mellanox_fanout RX_DRP = "RX_DRP" RX_ERR = "RX_ERR" @@ -24,10 +25,10 @@ pytest.SKIP_COUNTERS_FOR_MLNX = False MELLANOX_MAC_UPDATE_SCRIPT = os.path.join(os.path.dirname(__file__), "fanout/mellanox/mlnx_update_mac.j2") -# Ansible config files -LAB_CONNECTION_GRAPH_PATH = os.path.normpath((os.path.join(os.path.dirname(__file__), "../../ansible/files"))) ACL_COUNTERS_UPDATE_INTERVAL = 10 +ACL_TABLE_CREATE_INTERVAL = 30 +PORT_STATE_UPDATE_INTERNAL = 30 LOG_EXPECT_ACL_TABLE_CREATE_RE = ".*Created ACL table.*" LOG_EXPECT_ACL_RULE_CREATE_RE = ".*Successfully created ACL rule.*" LOG_EXPECT_ACL_RULE_REMOVE_RE = ".*Successfully deleted ACL rule.*" @@ -143,43 +144,7 @@ def configure_copp_drop_for_ttl_error(duthosts, rand_one_dut_hostname): yield duthost.command("rm {} {}".format(copp_trap_group_json, copp_trap_rule_json)) - config_reload(duthost) - - -def is_mellanox_devices(hwsku): - """ - A helper function to check if a given sku is Mellanox device - """ - hwsku = hwsku.lower() - return 'mellanox' in hwsku \ - or 'msn' in hwsku \ - or 'mlnx' in hwsku - - -def is_mellanox_fanout(duthost, localhost): - # Ansible localhost fixture which calls ansible playbook on the local host - - try: - dut_facts = \ - localhost.conn_graph_facts(host=duthost.hostname, filepath=LAB_CONNECTION_GRAPH_PATH)["ansible_facts"] - except RunAnsibleModuleFail as e: - logger.info("Get dut_facts failed, reason:{}".format(e.results['msg'])) - return False - - intf = list(dut_facts["device_conn"][duthost.hostname].keys())[0] - fanout_host = dut_facts["device_conn"][duthost.hostname][intf]["peerdevice"] - - try: - fanout_facts = \ - localhost.conn_graph_facts(host=fanout_host, filepath=LAB_CONNECTION_GRAPH_PATH)["ansible_facts"] - except RunAnsibleModuleFail: - return False - - fanout_sku = fanout_facts['device_info'][fanout_host]['HwSku'] - if not is_mellanox_devices(fanout_sku): - return False - - return True + config_reload(duthost, safe_reload=True) def get_fanout_obj(conn_graph_facts, duthost, fanouthosts): @@ -311,7 +276,6 @@ def rif_port_down(duthosts, enum_rand_one_per_hwsku_frontend_hostname, setup, fa """Shut RIF interface and return neighbor IP address attached to this interface.""" duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="drop_packet_rif_port_down") - wait_after_ports_up = 30 if not setup["rif_members"]: pytest.skip("RIF interface is absent") @@ -332,6 +296,9 @@ def rif_port_down(duthosts, enum_rand_one_per_hwsku_frontend_hostname, setup, fa loganalyzer.expect_regex = [LOG_EXPECT_PORT_OPER_DOWN_RE.format(rif_member_iface)] with loganalyzer as _: fanout_neighbor.shutdown(fanout_intf) + # Add a delay to ensure loganalyzer can find a match in the log. Without this delay, there's a + # chance it might miss the matching log. + time.sleep(PORT_STATE_UPDATE_INTERNAL) time.sleep(1) @@ -340,13 +307,21 @@ def rif_port_down(duthosts, enum_rand_one_per_hwsku_frontend_hostname, setup, fa loganalyzer.expect_regex = [LOG_EXPECT_PORT_OPER_UP_RE.format(rif_member_iface)] with loganalyzer as _: fanout_neighbor.no_shutdown(fanout_intf) - time.sleep(wait_after_ports_up) + # Add a delay to ensure loganalyzer can find a match in the log. Without this delay, there's a + # chance it might miss the matching log. + time.sleep(PORT_STATE_UPDATE_INTERNAL) @pytest.fixture(params=["port_channel_members", "vlan_members", "rif_members"]) -def tx_dut_ports(request, setup): +def tx_dut_ports(request, setup, tbinfo): """ Fixture for getting port members of specific port group """ - return setup[request.param] if setup[request.param] else pytest.skip("No {} available".format(request.param)) + if not setup[request.param]: + reason = "No {} available".format(request.param) + if tbinfo["topo"]["type"] != "t0" and request.param == "vlan_members": + reason = "Test case is only suitable for t0 type topology since it requires vlan interfaces" + pytest.skip(reason) + else: + return setup[request.param] @pytest.fixture @@ -416,6 +391,8 @@ def acl_teardown(duthosts, dut_tmp_dir, dut_clear_conf_file_path): duthost.command("config acl update full {}".format(dut_clear_conf_file_path)) logger.info("Removing {}".format(dut_tmp_dir)) duthost.command("rm -rf {}".format(dut_tmp_dir)) + # Add a delay to ensure loganalyzer can find a match in the log. Without this delay, there's a + # chance it might miss the matching log. time.sleep(ACL_COUNTERS_UPDATE_INTERVAL) @@ -476,6 +453,10 @@ def create_or_remove_acl_egress_table(duthost, op): ','.join(table_port_list) ) ) + + # Add a delay to ensure loganalyzer can find a match in the log. Without this delay, there's a + # chance it might miss the matching log. + time.sleep(ACL_TABLE_CREATE_INTERVAL) elif op == "remove": logger.info("Removing ACL table \"{}\" on device {}".format(acl_table_config["table_name"], duthost)) sonic_host_or_asic_inst.command("config acl remove table {}".format(acl_table_config["table_name"])) @@ -535,7 +516,7 @@ def send_packets(pkt, ptfadapter, ptf_tx_port_id, num_packets=1): def test_equal_smac_dmac_drop(do_test, ptfadapter, setup, fanouthost, - pkt_fields, ports_info, enum_fanout_graph_facts): # noqa F811 + pkt_fields, ports_info, enum_fanout_graph_facts, skip_traffic_test): # noqa F811 """ @summary: Create a packet with equal SMAC and DMAC. """ @@ -573,7 +554,8 @@ def test_equal_smac_dmac_drop(do_test, ptfadapter, setup, fanouthost, ) group = "L2" - do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], comparable_pkt=comparable_pkt) + do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + comparable_pkt=comparable_pkt, skip_traffic_test=skip_traffic_test) def test_multicast_smac_drop(do_test, ptfadapter, setup, fanouthost, @@ -617,11 +599,11 @@ def test_multicast_smac_drop(do_test, ptfadapter, setup, fanouthost, group = "L2" do_test(group, pkt, ptfadapter, ports_info, - setup["neighbor_sniff_ports"], comparable_pkt=comparable_pkt) + setup["neighbor_sniff_ports"], comparable_pkt=comparable_pkt, skip_traffic_test=skip_traffic_test) def test_not_expected_vlan_tag_drop(do_test, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - ptfadapter, setup, pkt_fields, ports_info): + ptfadapter, setup, pkt_fields, ports_info, skip_traffic_test): """ @summary: Create a VLAN tagged packet which VLAN ID does not match ingress port VLAN ID. """ @@ -654,10 +636,11 @@ def test_not_expected_vlan_tag_drop(do_test, duthosts, enum_rand_one_per_hwsku_f ) group = "L2" - do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"]) + do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + skip_traffic_test=skip_traffic_test) -def test_dst_ip_is_loopback_addr(do_test, ptfadapter, setup, pkt_fields, tx_dut_ports, ports_info): +def test_dst_ip_is_loopback_addr(do_test, ptfadapter, setup, pkt_fields, tx_dut_ports, ports_info, skip_traffic_test): """ @summary: Create a packet with loopback destination IP adress. """ @@ -675,10 +658,11 @@ def test_dst_ip_is_loopback_addr(do_test, ptfadapter, setup, pkt_fields, tx_dut_ tcp_dport=pkt_fields["tcp_dport"]) group = "L3" - do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], tx_dut_ports) + do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + tx_dut_ports, skip_traffic_test=skip_traffic_test) -def test_src_ip_is_loopback_addr(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, ports_info): +def test_src_ip_is_loopback_addr(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, ports_info, skip_traffic_test): """ @summary: Create a packet with loopback source IP adress. """ @@ -696,10 +680,11 @@ def test_src_ip_is_loopback_addr(do_test, ptfadapter, setup, tx_dut_ports, pkt_f tcp_dport=pkt_fields["tcp_dport"]) group = "L3" - do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], tx_dut_ports) + do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + tx_dut_ports, skip_traffic_test=skip_traffic_test) -def test_dst_ip_absent(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, ports_info): +def test_dst_ip_absent(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, ports_info, skip_traffic_test): """ @summary: Create a packet with absent destination IP address. """ @@ -727,11 +712,13 @@ def test_dst_ip_absent(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, por group = "L3" print(("msm group {}, setup {}".format(group, setup))) - do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], tx_dut_ports) + do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + tx_dut_ports, skip_traffic_test=skip_traffic_test) @pytest.mark.parametrize("ip_addr", ["ipv4", "ipv6"]) -def test_src_ip_is_multicast_addr(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, ip_addr, ports_info): +def test_src_ip_is_multicast_addr(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, ip_addr, + ports_info, skip_traffic_test): """ @summary: Create a packet with multicast source IP adress. """ @@ -764,11 +751,12 @@ def test_src_ip_is_multicast_addr(do_test, ptfadapter, setup, tx_dut_ports, pkt_ ports_info["src_mac"], pkt_fields["ipv4_dst"], ip_src) group = "L3" - do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], tx_dut_ports, ip_ver=ip_addr) + do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + tx_dut_ports, ip_ver=ip_addr, skip_traffic_test=skip_traffic_test) def test_src_ip_is_class_e(do_test, ptfadapter, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - setup, tx_dut_ports, pkt_fields, ports_info): + setup, tx_dut_ports, pkt_fields, ports_info, skip_traffic_test): """ @summary: Create a packet with source IP address in class E. """ @@ -791,12 +779,14 @@ def test_src_ip_is_class_e(do_test, ptfadapter, duthosts, enum_rand_one_per_hwsk tcp_dport=pkt_fields["tcp_dport"]) group = "L3" - do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], tx_dut_ports) + do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + tx_dut_ports, skip_traffic_test=skip_traffic_test) @pytest.mark.parametrize("addr_type, addr_direction", [("ipv4", "src"), ("ipv6", "src"), ("ipv4", "dst"), ("ipv6", "dst")]) -def test_ip_is_zero_addr(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, addr_type, addr_direction, ports_info): +def test_ip_is_zero_addr(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, addr_type, addr_direction, + ports_info, skip_traffic_test): """ @summary: Create a packet with "0.0.0.0" source or destination IP address. """ @@ -843,11 +833,11 @@ def test_ip_is_zero_addr(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, a pytest.skip("Src IP zero packets are not dropped on Broadcom DNX platform currently") do_test(group, pkt, ptfadapter, ports_info, list(setup["dut_to_ptf_port_map"].values()), tx_dut_ports, - ip_ver=addr_type) + ip_ver=addr_type, skip_traffic_test=skip_traffic_test) def test_dst_ip_link_local(do_test, ptfadapter, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - setup, tx_dut_ports, pkt_fields, ports_info): + setup, tx_dut_ports, pkt_fields, ports_info, skip_traffic_test): """ @summary: Create a packet with link-local address "169.254.0.0/16". """ @@ -870,10 +860,11 @@ def test_dst_ip_link_local(do_test, ptfadapter, duthosts, enum_rand_one_per_hwsk group = "L3" logger.info(pkt_params) - do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], tx_dut_ports) + do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + tx_dut_ports, skip_traffic_test=skip_traffic_test) -def test_loopback_filter(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, ports_info): +def test_loopback_filter(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, ports_info, skip_traffic_test): """ @summary: Create a packet drops by loopback-filter. Loop-back filter means that route to the host with DST IP of received packet exists on received interface @@ -901,17 +892,16 @@ def test_loopback_filter(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, p group = "L3" - do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], tx_dut_ports) + do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + tx_dut_ports, skip_traffic_test=skip_traffic_test) def test_ip_pkt_with_expired_ttl(duthost, do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, - ports_info, sai_acl_drop_adj_enabled, configure_copp_drop_for_ttl_error): + ports_info, sai_acl_drop_adj_enabled, configure_copp_drop_for_ttl_error, + skip_traffic_test): """ @summary: Create an IP packet with TTL=0. """ - if "x86_64-mlnx_msn" in duthost.facts["platform"] or "x86_64-nvidia_sn" in duthost.facts["platform"]: - pytest.skip("Not supported on Mellanox devices") - log_pkt_params(ports_info["dut_iface"], ports_info["dst_mac"], ports_info["src_mac"], pkt_fields["ipv4_dst"], pkt_fields["ipv4_src"]) @@ -926,12 +916,12 @@ def test_ip_pkt_with_expired_ttl(duthost, do_test, ptfadapter, setup, tx_dut_por group = "L3" do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], - tx_dut_ports, skip_counter_check=sai_acl_drop_adj_enabled) + tx_dut_ports, skip_counter_check=sai_acl_drop_adj_enabled, skip_traffic_test=skip_traffic_test) @pytest.mark.parametrize("pkt_field, value", [("version", 1), ("chksum", 10), ("ihl", 1)]) def test_broken_ip_header(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, pkt_field, - value, ports_info, sai_acl_drop_adj_enabled): + value, ports_info, sai_acl_drop_adj_enabled, skip_traffic_test): """ @summary: Create a packet with broken IP header. """ @@ -950,10 +940,11 @@ def test_broken_ip_header(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, group = "L3" do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], - tx_dut_ports, skip_counter_check=sai_acl_drop_adj_enabled) + tx_dut_ports, skip_counter_check=sai_acl_drop_adj_enabled, skip_traffic_test=skip_traffic_test) -def test_absent_ip_header(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, ports_info, sai_acl_drop_adj_enabled): +def test_absent_ip_header(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, ports_info, + sai_acl_drop_adj_enabled, skip_traffic_test): """ @summary: Create packets with absent IP header. """ @@ -976,12 +967,12 @@ def test_absent_ip_header(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, group = "L3" do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], - tx_dut_ports, skip_counter_check=sai_acl_drop_adj_enabled) + tx_dut_ports, skip_counter_check=sai_acl_drop_adj_enabled, skip_traffic_test=skip_traffic_test) @pytest.mark.parametrize("eth_dst", ["01:00:5e:00:01:02", "ff:ff:ff:ff:ff:ff"]) def test_unicast_ip_incorrect_eth_dst(do_test, ptfadapter, setup, tx_dut_ports, - pkt_fields, eth_dst, ports_info): + pkt_fields, eth_dst, ports_info, skip_traffic_test): """ @summary: Create packets with multicast/broadcast ethernet dst. """ @@ -1001,14 +992,15 @@ def test_unicast_ip_incorrect_eth_dst(do_test, ptfadapter, setup, tx_dut_ports, ) group = "L3" - do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], tx_dut_ports) + do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + tx_dut_ports, skip_traffic_test=skip_traffic_test) @pytest.mark.parametrize("igmp_version,msg_type", [("v1", "general_query"), ("v3", "general_query"), ("v1", "membership_report"), ("v2", "membership_report"), ("v3", "membership_report"), ("v2", "leave_group")]) def test_non_routable_igmp_pkts(do_test, ptfadapter, setup, fanouthost, tx_dut_ports, - pkt_fields, igmp_version, msg_type, ports_info): + pkt_fields, igmp_version, msg_type, ports_info, skip_traffic_test): """ @summary: Create an IGMP non-routable packets. """ @@ -1093,11 +1085,12 @@ def test_non_routable_igmp_pkts(do_test, ptfadapter, setup, fanouthost, tx_dut_p pkt.getlayer("IP").dst, pkt_fields["ipv4_src"]) group = "L3" - do_test(group, pkt, ptfadapter, ports_info, list(setup["dut_to_ptf_port_map"].values()), tx_dut_ports) + do_test(group, pkt, ptfadapter, ports_info, list(setup["dut_to_ptf_port_map"].values()), + tx_dut_ports, skip_traffic_test=skip_traffic_test) def test_acl_drop(do_test, ptfadapter, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - setup, tx_dut_ports, pkt_fields, acl_ingress, ports_info): + setup, tx_dut_ports, pkt_fields, acl_ingress, ports_info, skip_traffic_test): """ @summary: Verify that DUT drops packet with SRC IP 20.0.0.0/24 matched by ingress ACL """ @@ -1121,11 +1114,12 @@ def test_acl_drop(do_test, ptfadapter, duthosts, enum_rand_one_per_hwsku_fronten tcp_dport=pkt_fields["tcp_dport"] ) - do_test("ACL", pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], tx_dut_ports) + do_test("ACL", pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + tx_dut_ports, skip_traffic_test=skip_traffic_test) def test_acl_egress_drop(do_test, ptfadapter, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - setup, tx_dut_ports, pkt_fields, acl_egress, ports_info): + setup, tx_dut_ports, pkt_fields, acl_egress, ports_info, skip_traffic_test): """ @summary: Verify that DUT drops packet with DST IP 192.168.144.1/24 matched by egress ACL and ACL drop counter incremented @@ -1151,4 +1145,5 @@ def test_acl_egress_drop(do_test, ptfadapter, duthosts, enum_rand_one_per_hwsku_ ip_ttl=64 ) do_test(discard_group="ACL", pkt=pkt, ptfadapter=ptfadapter, ports_info=ports_info, - sniff_ports=setup["neighbor_sniff_ports"], tx_dut_ports=tx_dut_ports, drop_information="OUTDATAACL") + sniff_ports=setup["neighbor_sniff_ports"], tx_dut_ports=tx_dut_ports, drop_information="OUTDATAACL", + skip_traffic_test=skip_traffic_test) diff --git a/tests/drop_packets/test_configurable_drop_counters.py b/tests/drop_packets/test_configurable_drop_counters.py index b83ddb96ee..afc303f657 100644 --- a/tests/drop_packets/test_configurable_drop_counters.py +++ b/tests/drop_packets/test_configurable_drop_counters.py @@ -29,7 +29,6 @@ from tests.common.utilities import is_ipv4_address from tests.common import constants from tests.common import config_reload -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -200,6 +199,7 @@ def test_neighbor_link_down(testbed_params, setup_counters, duthosts, rand_one_d @pytest.mark.parametrize("drop_reason", ["DIP_LINK_LOCAL"]) def test_dip_link_local(testbed_params, setup_counters, duthosts, rand_one_dut_hostname, + toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 setup_standby_ports_on_rand_unselected_tor, # noqa F811 send_dropped_traffic, drop_reason, add_default_route_to_dut, generate_dropped_packet): """ @@ -226,6 +226,7 @@ def test_dip_link_local(testbed_params, setup_counters, duthosts, rand_one_dut_h @pytest.mark.parametrize("drop_reason", ["SIP_LINK_LOCAL"]) def test_sip_link_local(testbed_params, setup_counters, duthosts, rand_one_dut_hostname, + toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 setup_standby_ports_on_rand_unselected_tor, # noqa F811 send_dropped_traffic, drop_reason, add_default_route_to_dut, generate_dropped_packet): """ diff --git a/tests/drop_packets/test_drop_counters.py b/tests/drop_packets/test_drop_counters.py index d3eaef556b..c1835fb8e8 100755 --- a/tests/drop_packets/test_drop_counters.py +++ b/tests/drop_packets/test_drop_counters.py @@ -22,7 +22,7 @@ test_acl_egress_drop # noqa F401 from tests.common.helpers.constants import DEFAULT_NAMESPACE from tests.common.fixtures.conn_graph_facts import enum_fanout_graph_facts # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 pytestmark = [ pytest.mark.topology("any") @@ -46,6 +46,29 @@ COMBINED_ACL_DROP_COUNTER = False +@pytest.fixture(autouse=True) +def ignore_expected_loganalyzer_exceptions(duthosts, rand_one_dut_hostname, loganalyzer): + # Ignore in KVM test + KVMIgnoreRegex = [ + ".*ERR kernel.*Reset adapter.*", + ] + # Ignore time span WD exceeded error, and contextual log event messages + SAISwitchIgnoreRegex = [ + ".* ERR syncd.*#syncd.*logEventData:.*SAI_SWITCH_ATTR.*", + ".* ERR syncd.*#syncd.*logEventData:.*SAI_OBJECT_TYPE_SWITCH.*" + ] + # Ignore syslog error from xcvrd while using copper cables + CopperCableIgnoreRegex = [ + ".* ERR pmon#xcvrd.*no suitable app for the port appl.*host_lane_count.*host_speed.*" + ] + duthost = duthosts[rand_one_dut_hostname] + if loganalyzer: # Skip if loganalyzer is disabled + if duthost.facts["asic_type"] == "vs": + loganalyzer[duthost.hostname].ignore_regex.extend(KVMIgnoreRegex) + loganalyzer[duthost.hostname].ignore_regex.extend(SAISwitchIgnoreRegex) + loganalyzer[duthost.hostname].ignore_regex.extend(CopperCableIgnoreRegex) + + @pytest.fixture(autouse=True, scope="module") def enable_counters(duthosts): """ Fixture which enables RIF and L2 counters """ @@ -117,8 +140,9 @@ def handle_backend_acl(duthost, tbinfo): duthost.shell('systemctl restart backend-acl') -def base_verification(discard_group, pkt, ptfadapter, duthosts, asic_index, ports_info, # noqa F811 - tx_dut_ports=None, skip_counter_check=False, drop_information=None): # noqa F811 +def base_verification(discard_group, pkt, ptfadapter, duthosts, asic_index, ports_info, # noqa F811 + tx_dut_ports=None, skip_counter_check=False, drop_information=None, # noqa F811 + skip_traffic_test=False): # noqa F811 """ Base test function for verification of L2 or L3 packet drops. Verification type depends on 'discard_group' value. Supported 'discard_group' values: 'L2', 'L3', 'ACL', 'NO_DROPS' @@ -138,6 +162,9 @@ def base_verification(discard_group, pkt, ptfadapter, duthosts, asic_index, port if skip_counter_check: logger.info("Skipping counter check") return None + if skip_traffic_test is True: + logger.info("Skipping traffic test") + return None if discard_group == "L2": verify_drop_counters(duthosts, asic_index, ports_info["dut_iface"], @@ -270,7 +297,8 @@ def check_if_skip(): @pytest.fixture(scope='module') def do_test(duthosts): def do_counters_test(discard_group, pkt, ptfadapter, ports_info, sniff_ports, tx_dut_ports=None, # noqa F811 - comparable_pkt=None, skip_counter_check=False, drop_information=None, ip_ver='ipv4'): + comparable_pkt=None, skip_counter_check=False, drop_information=None, ip_ver='ipv4', + skip_traffic_test=False): # noqa F811 """ Execute test - send packet, check that expected discard counters were incremented and packet was dropped @param discard_group: Supported 'discard_group' values: 'L2', 'L3', 'ACL', 'NO_DROPS' @@ -284,18 +312,22 @@ def do_counters_test(discard_group, pkt, ptfadapter, ports_info, sniff_ports, tx check_if_skip() asic_index = ports_info["asic_index"] base_verification(discard_group, pkt, ptfadapter, duthosts, asic_index, ports_info, tx_dut_ports, - skip_counter_check=skip_counter_check, drop_information=drop_information) + skip_counter_check=skip_counter_check, drop_information=drop_information, + skip_traffic_test=skip_traffic_test) # Verify packets were not egresed the DUT if discard_group != "NO_DROPS": exp_pkt = expected_packet_mask(pkt, ip_ver=ip_ver) + if skip_traffic_test is True: + logger.info("Skipping traffic test") + return testutils.verify_no_packet_any(ptfadapter, exp_pkt, ports=sniff_ports) return do_counters_test def test_reserved_dmac_drop(do_test, ptfadapter, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - setup, fanouthost, pkt_fields, ports_info): # noqa F811 + setup, fanouthost, pkt_fields, ports_info, skip_traffic_test): # noqa F811 """ @summary: Verify that packet with reserved DMAC is dropped and L2 drop counter incremented @used_mac_address: @@ -329,10 +361,12 @@ def test_reserved_dmac_drop(do_test, ptfadapter, duthosts, enum_rand_one_per_hws ) group = "L2" - do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"]) + do_test(group, pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + skip_traffic_test=skip_traffic_test) -def test_no_egress_drop_on_down_link(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, rif_port_down, ports_info): # noqa F811 +def test_no_egress_drop_on_down_link(do_test, ptfadapter, setup, tx_dut_ports, # noqa F811 + pkt_fields, rif_port_down, ports_info, skip_traffic_test): # noqa F811 """ @summary: Verify that packets on ingress port are not dropped when egress RIF link is down and check that drop counters not incremented @@ -350,11 +384,12 @@ def test_no_egress_drop_on_down_link(do_test, ptfadapter, setup, tx_dut_ports, p tcp_dport=pkt_fields["tcp_dport"] ) - do_test("NO_DROPS", pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], tx_dut_ports) + do_test("NO_DROPS", pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + tx_dut_ports, skip_traffic_test=skip_traffic_test) def test_src_ip_link_local(do_test, ptfadapter, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - setup, tx_dut_ports, pkt_fields, ports_info): # noqa F811 + setup, tx_dut_ports, pkt_fields, ports_info, skip_traffic_test): # noqa F811 """ @summary: Verify that packet with link-local address "169.254.0.0/16" is dropped and L3 drop counter incremented """ @@ -377,10 +412,12 @@ def test_src_ip_link_local(do_test, ptfadapter, duthosts, enum_rand_one_per_hwsk pkt = testutils.simple_tcp_packet(**pkt_params) logger.info(pkt_params) - do_test("L3", pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], tx_dut_ports) + do_test("L3", pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + tx_dut_ports, skip_traffic_test=skip_traffic_test) -def test_ip_pkt_with_exceeded_mtu(do_test, ptfadapter, setup, tx_dut_ports, pkt_fields, mtu_config, ports_info): # noqa F811 +def test_ip_pkt_with_exceeded_mtu(do_test, ptfadapter, setup, tx_dut_ports, # noqa F811 + pkt_fields, mtu_config, ports_info, skip_traffic_test): # noqa F811 """ @summary: Verify that IP packet with exceeded MTU is dropped and L3 drop counter incremented """ @@ -410,6 +447,7 @@ def test_ip_pkt_with_exceeded_mtu(do_test, ptfadapter, setup, tx_dut_ports, pkt_ ) L2_COL_KEY = RX_ERR try: - do_test("L2", pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"]) + do_test("L2", pkt, ptfadapter, ports_info, setup["neighbor_sniff_ports"], + skip_traffic_test=skip_traffic_test) finally: L2_COL_KEY = RX_DRP diff --git a/tests/dualtor/conftest.py b/tests/dualtor/conftest.py index 7f13a7517e..eb933e61f2 100644 --- a/tests/dualtor/conftest.py +++ b/tests/dualtor/conftest.py @@ -65,16 +65,17 @@ def pytest_addoption(parser): @pytest.fixture(scope="module", autouse=True) def common_setup_teardown(rand_selected_dut, request, tbinfo, vmhost): # Skip dualtor test cases on unsupported platform - supported_platforms = ['broadcom_td3_hwskus', 'broadcom_th2_hwskus', 'cisco_hwskus', 'mellanox_dualtor_hwskus'] - hostvars = get_host_visible_vars(rand_selected_dut.host.options['inventory'], rand_selected_dut.hostname) - hwsku = rand_selected_dut.facts['hwsku'] - skip = True - for platform in supported_platforms: - supported_skus = hostvars.get(platform, []) - if hwsku in supported_skus: - skip = False - break - py_require(not skip, "Skip on unsupported platform") + if rand_selected_dut.facts['asic_type'] != 'vs': + supported_platforms = ['broadcom_td3_hwskus', 'broadcom_th2_hwskus', 'cisco_hwskus', 'mellanox_dualtor_hwskus'] + hostvars = get_host_visible_vars(rand_selected_dut.host.options['inventory'], rand_selected_dut.hostname) + hwsku = rand_selected_dut.facts['hwsku'] + skip = True + for platform in supported_platforms: + supported_skus = hostvars.get(platform, []) + if hwsku in supported_skus: + skip = False + break + py_require(not skip, "Skip on unsupported platform") if 'dualtor' in tbinfo['topo']['name']: request.getfixturevalue('run_garp_service') diff --git a/tests/dualtor/test_ipinip.py b/tests/dualtor/test_ipinip.py index af3f9d498e..d5e0b15476 100644 --- a/tests/dualtor/test_ipinip.py +++ b/tests/dualtor/test_ipinip.py @@ -28,13 +28,13 @@ from tests.common.fixtures.ptfhost_utils import run_icmp_responder # noqa F401 from tests.common.fixtures.ptfhost_utils import run_garp_service # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.utilities import dump_scapy_packet_show_output from tests.common.dualtor.dual_tor_utils import config_active_active_dualtor_active_standby # noqa F401 from tests.common.dualtor.dual_tor_utils import validate_active_active_dualtor_setup # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ - pytest.mark.topology("t0") + pytest.mark.topology("dualtor") ] logger = logging.getLogger(__name__) @@ -105,7 +105,7 @@ def build_expected_packet_to_server(encapsulated_packet, decrease_ttl=False): def test_decap_active_tor( build_encapsulated_packet, request, ptfhost, rand_selected_interface, ptfadapter, # noqa F401 - tbinfo, rand_selected_dut, tunnel_traffic_monitor): # noqa F401 + tbinfo, rand_selected_dut, tunnel_traffic_monitor, skip_traffic_test): # noqa F811 @contextlib.contextmanager def stop_garp(ptfhost): @@ -129,6 +129,9 @@ def stop_garp(ptfhost): ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send encapsulated packet from ptf t1 interface %s", ptf_t1_intf) + if skip_traffic_test is True: + logging.info("Skip following traffic test") + return with stop_garp(ptfhost): ptfadapter.dataplane.flush() testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), encapsulated_packet) @@ -138,7 +141,7 @@ def stop_garp(ptfhost): def test_decap_standby_tor( build_encapsulated_packet, request, rand_selected_interface, ptfadapter, # noqa F401 - tbinfo, rand_selected_dut, tunnel_traffic_monitor # noqa F401 + tbinfo, rand_selected_dut, tunnel_traffic_monitor, skip_traffic_test # noqa F401 ): def verify_downstream_packet_to_server(ptfadapter, port, exp_pkt): @@ -167,6 +170,9 @@ def verify_downstream_packet_to_server(ptfadapter, port, exp_pkt): ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send encapsulated packet from ptf t1 interface %s", ptf_t1_intf) + if skip_traffic_test is True: + logging.info("Skip following traffic test") + return with tunnel_traffic_monitor(tor, existing=False): testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), encapsulated_packet, count=10) time.sleep(2) @@ -296,7 +302,7 @@ def setup_active_active_ports(active_active_ports, rand_selected_dut, rand_unsel def test_encap_with_mirror_session(rand_selected_dut, rand_selected_interface, # noqa F811 ptfadapter, tbinfo, setup_mirror_session, toggle_all_simulator_ports_to_rand_unselected_tor, # noqa F811 - tunnel_traffic_monitor, # noqa F811 + tunnel_traffic_monitor, skip_traffic_test, # noqa F811 setup_standby_ports_on_rand_selected_tor): # noqa F811 """ A test case to verify the bounced back packet from Standby ToR to T1 doesn't have an unexpected vlan id (4095) @@ -315,5 +321,8 @@ def test_encap_with_mirror_session(rand_selected_dut, rand_selected_interface, logging.info("Sending packet from ptf t1 interface {}".format(src_port_id)) inner_packet = pkt_to_server[scapy.all.IP].copy() inner_packet[IP].ttl -= 1 + if skip_traffic_test is True: + logging.info("Skip following traffic test") + return with tunnel_traffic_monitor(rand_selected_dut, inner_packet=inner_packet, check_items=()): testutils.send(ptfadapter, src_port_id, pkt_to_server) diff --git a/tests/dualtor/test_orch_stress.py b/tests/dualtor/test_orch_stress.py index 38cfbfc293..edfd008cbd 100644 --- a/tests/dualtor/test_orch_stress.py +++ b/tests/dualtor/test_orch_stress.py @@ -27,7 +27,6 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.dualtor.dual_tor_utils import tor_mux_intfs # noqa F401 from tests.common.dualtor.dual_tor_mock import * # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology("t0") diff --git a/tests/dualtor/test_orchagent_active_tor_downstream.py b/tests/dualtor/test_orchagent_active_tor_downstream.py index 5677a9a558..45d3506eaf 100644 --- a/tests/dualtor/test_orchagent_active_tor_downstream.py +++ b/tests/dualtor/test_orchagent_active_tor_downstream.py @@ -21,9 +21,9 @@ from tests.common.fixtures.ptfhost_utils import run_icmp_responder # noqa F401 from tests.common.fixtures.ptfhost_utils import run_garp_service # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -68,7 +68,7 @@ def neighbor_reachable(duthost, neighbor_ip): def test_active_tor_remove_neighbor_downstream_active( conn_graph_facts, ptfadapter, ptfhost, testbed_setup, rand_selected_dut, tbinfo, set_crm_polling_interval, - tunnel_traffic_monitor, vmhost # noqa F811 + tunnel_traffic_monitor, vmhost, skip_traffic_test # noqa F811 ): """ @Verify those two scenarios: @@ -102,18 +102,18 @@ def remove_neighbor(ptfhost, duthost, server_ip, ip_version, neighbor_details): ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send traffic to server %s from ptf t1 interface %s", server_ip, ptf_t1_intf) server_traffic_monitor = ServerTrafficMonitor( - tor, ptfhost, vmhost, tbinfo, test_port, - conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo) # noqa F405 + tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, + existing=True, is_mocked=is_mocked_dualtor(tbinfo), skip_traffic_test=skip_traffic_test # noqa F405 ) - tunnel_monitor = tunnel_traffic_monitor(tor, existing=False) + tunnel_monitor = tunnel_traffic_monitor(tor, existing=False, skip_traffic_test=skip_traffic_test) with crm_neighbor_checker(tor, ip_version, expect_change=ip_version == "ipv6"), \ tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after removing neighbor entry", server_ip) server_traffic_monitor = ServerTrafficMonitor( - tor, ptfhost, vmhost, tbinfo, test_port, - conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) # noqa F405 + tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, + existing=False, is_mocked=is_mocked_dualtor(tbinfo), skip_traffic_test=skip_traffic_test # noqa F405 ) remove_neighbor_ct = remove_neighbor(ptfhost, tor, server_ip, ip_version, removed_neighbor) with crm_neighbor_checker(tor, ip_version, expect_change=ip_version == "ipv6"), \ @@ -125,8 +125,8 @@ def remove_neighbor(ptfhost, duthost, server_ip, ip_version, neighbor_details): logging.info("send traffic to server %s after neighbor entry is restored", server_ip) server_traffic_monitor = ServerTrafficMonitor( - tor, ptfhost, vmhost, tbinfo, test_port, - conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo) # noqa F405 + tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, + existing=True, is_mocked=is_mocked_dualtor(tbinfo), skip_traffic_test=skip_traffic_test # noqa F405 ) with crm_neighbor_checker(tor, ip_version, expect_change=ip_version == "ipv6"), \ tunnel_monitor, server_traffic_monitor: @@ -146,7 +146,7 @@ def remove_neighbor(ptfhost, duthost, server_ip, ip_version, neighbor_details): def test_downstream_ecmp_nexthops( ptfadapter, rand_selected_dut, tbinfo, - toggle_all_simulator_ports, tor_mux_intfs, ip_version # noqa F811 + toggle_all_simulator_ports, tor_mux_intfs, ip_version, skip_traffic_test # noqa F811 ): nexthops_count = 4 set_mux_state(rand_selected_dut, tbinfo, 'active', tor_mux_intfs, toggle_all_simulator_ports) # noqa F405 @@ -172,7 +172,7 @@ def test_downstream_ecmp_nexthops( try: logging.info("Verify traffic to this route destination is sent to single downlink or uplink") check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_addr, - tbinfo, nexthop_interfaces) + tbinfo, nexthop_interfaces, skip_traffic_test) nexthop_interfaces_copy = nexthop_interfaces.copy() @@ -183,7 +183,7 @@ def test_downstream_ecmp_nexthops( nexthop_interfaces_copy.remove(interface) logging.info("Verify traffic to this route destination is sent to single downlink or uplink") check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_addr, - tbinfo, nexthop_interfaces_copy) + tbinfo, nexthop_interfaces_copy, skip_traffic_test) # Revert two mux states to active for index, interface in reversed(list(enumerate(nexthop_interfaces))): @@ -192,7 +192,7 @@ def test_downstream_ecmp_nexthops( nexthop_interfaces_copy.append(interface) logging.info("Verify traffic to this route destination is sent to single downlink or uplink") check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_addr, - tbinfo, nexthop_interfaces_copy) + tbinfo, nexthop_interfaces_copy, skip_traffic_test) finally: # Remove the nexthop route remove_static_routes(rand_selected_dut, dst_server_addr) diff --git a/tests/dualtor/test_orchagent_mac_move.py b/tests/dualtor/test_orchagent_mac_move.py index e6667f90c0..7aa0a25e39 100644 --- a/tests/dualtor/test_orchagent_mac_move.py +++ b/tests/dualtor/test_orchagent_mac_move.py @@ -13,8 +13,8 @@ from tests.common.fixtures.ptfhost_utils import run_icmp_responder # noqa F401 from tests.common.fixtures.ptfhost_utils import run_garp_service # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.utilities import dump_scapy_packet_show_output -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -85,7 +85,7 @@ def test_mac_move( announce_new_neighbor, apply_active_state_to_orchagent, conn_graph_facts, ptfadapter, ptfhost, rand_selected_dut, set_crm_polling_interval, - tbinfo, tunnel_traffic_monitor, vmhost # noqa F811 + tbinfo, tunnel_traffic_monitor, vmhost, skip_traffic_test # noqa F811 ): tor = rand_selected_dut ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) @@ -96,10 +96,10 @@ def test_mac_move( announce_new_neighbor.send(None) logging.info("let new neighbor learnt on active port %s", test_port) pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, NEW_NEIGHBOR_IPV4_ADDR) - tunnel_monitor = tunnel_traffic_monitor(tor, existing=False) + tunnel_monitor = tunnel_traffic_monitor(tor, existing=False, skip_traffic_test=skip_traffic_test) server_traffic_monitor = ServerTrafficMonitor( - tor, ptfhost, vmhost, tbinfo, test_port, - conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo) # noqa F405 + tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, + existing=True, is_mocked=is_mocked_dualtor(tbinfo), skip_traffic_test=skip_traffic_test # noqa F405 ) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) @@ -109,10 +109,10 @@ def test_mac_move( announce_new_neighbor.send(lambda iface: set_dual_tor_state_to_orchagent(tor, "standby", [iface])) # noqa F405 logging.info("mac move to a standby port %s", test_port) pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, NEW_NEIGHBOR_IPV4_ADDR) - tunnel_monitor = tunnel_traffic_monitor(tor, existing=True) + tunnel_monitor = tunnel_traffic_monitor(tor, existing=True, skip_traffic_test=skip_traffic_test) server_traffic_monitor = ServerTrafficMonitor( - tor, ptfhost, vmhost, tbinfo, test_port, - conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) # noqa F405 + tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, + existing=False, is_mocked=is_mocked_dualtor(tbinfo), skip_traffic_test=skip_traffic_test # noqa F405 ) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) @@ -120,8 +120,8 @@ def test_mac_move( # standby forwarding check after fdb ageout/flush tor.shell("fdbclear") server_traffic_monitor = ServerTrafficMonitor( - tor, ptfhost, vmhost, tbinfo, test_port, - conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) # noqa F405 + tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, + existing=False, is_mocked=is_mocked_dualtor(tbinfo), skip_traffic_test=skip_traffic_test # noqa F405 ) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) @@ -131,10 +131,10 @@ def test_mac_move( announce_new_neighbor.send(None) logging.info("mac move to another active port %s", test_port) pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, NEW_NEIGHBOR_IPV4_ADDR) - tunnel_monitor = tunnel_traffic_monitor(tor, existing=False) + tunnel_monitor = tunnel_traffic_monitor(tor, existing=False, skip_traffic_test=skip_traffic_test) server_traffic_monitor = ServerTrafficMonitor( - tor, ptfhost, vmhost, tbinfo, test_port, - conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo) # noqa F405 + tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, + existing=True, is_mocked=is_mocked_dualtor(tbinfo), skip_traffic_test=skip_traffic_test # noqa F405 ) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) @@ -144,8 +144,8 @@ def test_mac_move( if not (tor.facts['asic_type'] == 'mellanox' or tor.facts['asic_type'] == 'cisco-8000'): tor.shell("fdbclear") server_traffic_monitor = ServerTrafficMonitor( - tor, ptfhost, vmhost, tbinfo, test_port, - conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) # noqa F405 + tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, + existing=False, is_mocked=is_mocked_dualtor(tbinfo), skip_traffic_test=skip_traffic_test # noqa F405 ) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) diff --git a/tests/dualtor/test_orchagent_slb.py b/tests/dualtor/test_orchagent_slb.py index fdf65ae1b8..4b0aeb8962 100644 --- a/tests/dualtor/test_orchagent_slb.py +++ b/tests/dualtor/test_orchagent_slb.py @@ -2,6 +2,7 @@ import pytest import random import time +import logging import scapy.all as scapyall from ptf import testutils @@ -19,9 +20,9 @@ from tests.common.fixtures.ptfhost_utils import run_icmp_responder # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.helpers import bgp from tests.common.utilities import is_ipv4_address -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -216,7 +217,7 @@ def test_orchagent_slb( force_active_tor, upper_tor_host, lower_tor_host, # noqa F811 ptfadapter, ptfhost, setup_interfaces, toggle_all_simulator_ports_to_upper_tor, tbinfo, # noqa F811 - tunnel_traffic_monitor, vmhost # noqa F811 + tunnel_traffic_monitor, vmhost, skip_traffic_test # noqa F811 ): def verify_bgp_session(duthost, bgp_neighbor): @@ -234,7 +235,11 @@ def verify_route(duthost, route, existing=True): else: assert len(existing_route["nexthops"]) == 0 - def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_existed=True): + def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_existed=True, + skip_traffic_test=skip_traffic_test): + if skip_traffic_test is True: + logging.info("Skip traffic test.") + return prefix = ipaddress.ip_network(route["prefix"]) dst_host = str(next(prefix.hosts())) pkt, exp_pkt = build_packet_to_server(duthost, ptfadapter, dst_host) @@ -261,6 +266,7 @@ def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_ ) with tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) + time.sleep(5) connections = setup_interfaces @@ -289,11 +295,11 @@ def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_ # STEP 3: verify the route by sending some downstream traffic verify_traffic( upper_tor_host, connections["upper_tor"], constants.route, - is_duthost_active=True, is_route_existed=True + is_duthost_active=True, is_route_existed=True, skip_traffic_test=skip_traffic_test ) verify_traffic( lower_tor_host, connections["lower_tor"], constants.route, - is_duthost_active=False, is_route_existed=True + is_duthost_active=False, is_route_existed=True, skip_traffic_test=skip_traffic_test ) # STEP 4: withdraw the announced route to both ToRs @@ -308,11 +314,11 @@ def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_ # STEP 5: verify the route is removed by verifying that downstream traffic is dropped verify_traffic( upper_tor_host, connections["upper_tor"], constants.route, - is_duthost_active=True, is_route_existed=False + is_duthost_active=True, is_route_existed=False, skip_traffic_test=skip_traffic_test ) verify_traffic( lower_tor_host, connections["lower_tor"], constants.route, - is_duthost_active=False, is_route_existed=False + is_duthost_active=False, is_route_existed=False, skip_traffic_test=skip_traffic_test ) # STEP 6: toggle mux state change @@ -335,11 +341,11 @@ def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_ # STEP 8: verify the route by sending some downstream traffic verify_traffic( upper_tor_host, connections["upper_tor"], constants.route, - is_duthost_active=False, is_route_existed=True + is_duthost_active=False, is_route_existed=True, skip_traffic_test=skip_traffic_test ) verify_traffic( lower_tor_host, connections["lower_tor"], constants.route, - is_duthost_active=True, is_route_existed=True + is_duthost_active=True, is_route_existed=True, skip_traffic_test=skip_traffic_test ) # STEP 9: verify teardown diff --git a/tests/dualtor/test_orchagent_standby_tor_downstream.py b/tests/dualtor/test_orchagent_standby_tor_downstream.py index 7064e1e3b4..1d26d74187 100644 --- a/tests/dualtor/test_orchagent_standby_tor_downstream.py +++ b/tests/dualtor/test_orchagent_standby_tor_downstream.py @@ -15,17 +15,17 @@ from tests.common.dualtor.dual_tor_utils import build_packet_to_server from tests.common.dualtor.dual_tor_utils import crm_neighbor_checker from tests.common.dualtor.dual_tor_utils import add_nexthop_routes, remove_static_routes -from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa: F401 -from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa: F401 -from tests.common.fixtures.ptfhost_utils import run_garp_service # noqa: F401 -from tests.common.fixtures.ptfhost_utils import run_icmp_responder # noqa: F401 +from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import run_garp_service # noqa F401 +from tests.common.fixtures.ptfhost_utils import run_icmp_responder # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.helpers.assertions import pytest_assert as pt_assert -from tests.common.dualtor.tunnel_traffic_utils import tunnel_traffic_monitor # noqa: F401 +from tests.common.dualtor.tunnel_traffic_utils import tunnel_traffic_monitor # noqa F401 from tests.common.dualtor.server_traffic_utils import ServerTrafficMonitor -from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports # noqa: F401 -from tests.common.dualtor.tor_failure_utils import shutdown_bgp_sessions # noqa: F401 +from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports # noqa F401 +from tests.common.dualtor.tor_failure_utils import shutdown_bgp_sessions # noqa F401 from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -62,12 +62,13 @@ def get_function_completeness_level(pytestconfig): @pytest.fixture def get_testbed_params(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, - ip_version, setup_testbed_ipv6, get_function_completeness_level): + ip_version, setup_testbed_ipv6, get_function_completeness_level, skip_traffic_test): # noqa F811 """Return a function to get testbed params.""" def _get_testbed_params(): params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, get_function_completeness_level) params["check_ipv6"] = (ip_version == "ipv6") + params["skip_traffic_test"] = skip_traffic_test return params return _get_testbed_params @@ -145,7 +146,8 @@ def test_standby_tor_downstream(rand_selected_dut, get_testbed_params): def test_standby_tor_downstream_t1_link_recovered( - rand_selected_dut, verify_crm_nexthop_counter_not_increased, tbinfo, get_testbed_params + rand_selected_dut, verify_crm_nexthop_counter_not_increased, + tbinfo, get_testbed_params ): """ Verify traffic is distributed evenly after t1 link is recovered; @@ -206,7 +208,7 @@ def route_matches_expected_state(duthost, route_ip, expect_route): @pytest.fixture def remove_peer_loopback_route(rand_selected_dut, rand_unselected_dut, - shutdown_bgp_sessions, get_testbed_params): # noqa: F811 + shutdown_bgp_sessions, get_testbed_params): # noqa F811 """ Remove routes to peer ToR loopback IP by shutting down BGP sessions on the peer """ @@ -272,9 +274,9 @@ def test_standby_tor_downstream_loopback_route_readded( def test_standby_tor_remove_neighbor_downstream_standby( conn_graph_facts, ptfadapter, ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, - set_crm_polling_interval, tunnel_traffic_monitor, # noqa: F811 + set_crm_polling_interval, tunnel_traffic_monitor, # noqa: F811 vmhost, get_testbed_params, - ip_version + ip_version, skip_traffic_test # noqa: F811 ): """ @summary: Verify that after removing neighbor entry for a server over standby @@ -305,15 +307,15 @@ def stop_neighbor_advertiser(ptfhost, ip_version): pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, target_server) ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send traffic to server %s from ptf t1 interface %s", target_server, ptf_t1_intf) - tunnel_monitor = tunnel_traffic_monitor(tor, existing=True) + tunnel_monitor = tunnel_traffic_monitor(tor, existing=True, skip_traffic_test=skip_traffic_test) with tunnel_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after removing neighbor entry", target_server) tunnel_monitor.existing = False server_traffic_monitor = ServerTrafficMonitor( - tor, ptfhost, vmhost, tbinfo, test_params["selected_port"], - conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) + tor, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, + existing=False, is_mocked=is_mocked_dualtor(tbinfo), skip_traffic_test=skip_traffic_test ) # for real dualtor testbed, leave the neighbor restoration to garp service flush_neighbor_ct = flush_neighbor(tor, target_server, restore=is_t0_mocked_dualtor) @@ -330,9 +332,9 @@ def stop_neighbor_advertiser(ptfhost, ip_version): def test_downstream_standby_mux_toggle_active( conn_graph_facts, ptfadapter, ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, - tunnel_traffic_monitor, vmhost, # noqa: F811 - toggle_all_simulator_ports, tor_mux_intfs, # noqa: F811 - ip_version, get_testbed_params + tunnel_traffic_monitor, vmhost, # noqa: F811 + toggle_all_simulator_ports, tor_mux_intfs, # noqa: F811 + ip_version, get_testbed_params, skip_traffic_test # noqa: F811 ): # set rand_selected_dut as standby and rand_unselected_dut to active tor test_params = get_testbed_params() @@ -346,7 +348,10 @@ def test_downstream_standby_mux_toggle_active( pkt, exp_pkt = build_packet_to_server(rand_selected_dut, ptfadapter, random_dst_ip) ptf_t1_intf = random.choice(get_t1_ptf_ports(rand_selected_dut, tbinfo)) - def monitor_tunnel_and_server_traffic(torhost, expect_tunnel_traffic=True, expect_server_traffic=True): + def monitor_tunnel_and_server_traffic(torhost, expect_tunnel_traffic=True, + expect_server_traffic=True, skip_traffic_test=False): + if skip_traffic_test is True: + return tunnel_monitor = tunnel_traffic_monitor(rand_selected_dut, existing=True) server_traffic_monitor = ServerTrafficMonitor( torhost, ptfhost, vmhost, tbinfo, test_params["selected_port"], @@ -364,14 +369,16 @@ def monitor_tunnel_and_server_traffic(torhost, expect_tunnel_traffic=True, expec time.sleep(30) logger.info("Step 1.2: Verify traffic to this route dst is forwarded to Active ToR and equally distributed") check_tunnel_balance(**test_params) - monitor_tunnel_and_server_traffic(rand_selected_dut, expect_server_traffic=False, expect_tunnel_traffic=True) + monitor_tunnel_and_server_traffic(rand_selected_dut, expect_server_traffic=False, + expect_tunnel_traffic=True, skip_traffic_test=skip_traffic_test) logger.info("Stage 2: Verify Active Forwarding") logger.info("Step 2.1: Simulate Mux state change to active") set_mux_state(rand_selected_dut, tbinfo, 'active', tor_mux_intfs, toggle_all_simulator_ports) time.sleep(30) logger.info("Step 2.2: Verify traffic to this route dst is forwarded directly to server") - monitor_tunnel_and_server_traffic(rand_selected_dut, expect_server_traffic=True, expect_tunnel_traffic=False) + monitor_tunnel_and_server_traffic(rand_selected_dut, expect_server_traffic=True, + expect_tunnel_traffic=False, skip_traffic_test=skip_traffic_test) logger.info("Stage 3: Verify Standby Forwarding Again") logger.info("Step 3.1: Simulate Mux state change to standby") @@ -379,7 +386,8 @@ def monitor_tunnel_and_server_traffic(torhost, expect_tunnel_traffic=True, expec time.sleep(30) logger.info("Step 3.2: Verify traffic to this route dst \ is now redirected back to Active ToR and equally distributed") - monitor_tunnel_and_server_traffic(rand_selected_dut, expect_server_traffic=False, expect_tunnel_traffic=True) + monitor_tunnel_and_server_traffic(rand_selected_dut, expect_server_traffic=False, + expect_tunnel_traffic=True, skip_traffic_test=skip_traffic_test) check_tunnel_balance(**test_params) remove_static_routes(rand_selected_dut, random_dst_ip) diff --git a/tests/dualtor/test_standalone_tunnel_route.py b/tests/dualtor/test_standalone_tunnel_route.py new file mode 100644 index 0000000000..e34ffb820b --- /dev/null +++ b/tests/dualtor/test_standalone_tunnel_route.py @@ -0,0 +1,137 @@ +import json +import ipaddress +import pytest +import random +import time +import logging +import scapy.all as scapyall + +from ptf import testutils + +from tests.common.dualtor.dual_tor_common import active_active_ports # noqa F401 +from tests.common.dualtor.dual_tor_utils import build_packet_to_server +from tests.common.dualtor.dual_tor_utils import mux_cable_server_ip # noqa F401 +from tests.common.dualtor.dual_tor_utils import upper_tor_host # noqa F401 +from tests.common.dualtor.dual_tor_utils import lower_tor_host # noqa F401 +from tests.common.dualtor.dual_tor_utils import get_t1_ptf_ports +from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_upper_tor # noqa F401 +from tests.common.dualtor.dual_tor_common import cable_type # noqa F401 +from tests.common.dualtor.dual_tor_common import CableType +from tests.common.dualtor.tunnel_traffic_utils import tunnel_traffic_monitor # noqa F401 +from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import is_ipv4_address +from tests.common.utilities import wait_until + + +pytestmark = [ + pytest.mark.topology("dualtor") +] + + +@pytest.fixture(autouse=True) +def cleanup_neighbors(duthosts): + """Cleanup neighbors.""" + duthosts.shell("sonic-clear arp") + duthosts.shell("sonic-clear ndp") + return + + +@pytest.fixture +def constants(lower_tor_host, tbinfo): # noqa F811 + class _C(object): + """Dummy class to save test constants.""" + + def _find_ipv4_vlan(mg_facts): + for vlan_intf in mg_facts["minigraph_vlan_interfaces"]: + if is_ipv4_address(vlan_intf["addr"]): + return vlan_intf + + def _find_ipv6_vlan(mg_facts): + for vlan_intf in mg_facts["minigraph_vlan_interfaces"]: + if not is_ipv4_address(vlan_intf["addr"]): + return vlan_intf + + lower_tor_mg_facts = lower_tor_host.get_extended_minigraph_facts(tbinfo) + lower_tor_vlan = _find_ipv4_vlan(lower_tor_mg_facts) + lower_tor_vlan_ipv6 = _find_ipv6_vlan(lower_tor_mg_facts) + vlan_subnet = ipaddress.ip_network(lower_tor_vlan["subnet"]) + vlan_subnet_v6 = ipaddress.ip_network(lower_tor_vlan_ipv6["subnet"]) + selected_target_ip = vlan_subnet.network_address + 500 + selected_target_ipv6 = vlan_subnet_v6.network_address + 500 + pytest_assert(selected_target_ip in vlan_subnet) + pytest_assert(selected_target_ipv6 in vlan_subnet_v6) + + _constants = _C() + _constants.target_ip = selected_target_ip + _constants.target_ipv6 = selected_target_ipv6 + return _constants + + +def test_standalone_tunnel_route( + cable_type, constants, upper_tor_host, lower_tor_host, # noqa F811 + ptfadapter, toggle_all_simulator_ports_to_upper_tor, tbinfo, # noqa F811 + tunnel_traffic_monitor # noqa F811 +): + def _verify_traffic(duthost, target_ip): + pkt, _ = build_packet_to_server(duthost, ptfadapter, str(target_ip)) + ptf_t1_intf = random.choice(get_t1_ptf_ports(duthost, tbinfo)) + ptf_t1_intf_index = int(ptf_t1_intf.strip("eth")) + + if target_ip.version == 4: + tunnel_innner_pkt = pkt[scapyall.IP].copy() + tunnel_innner_pkt[scapyall.IP].ttl -= 1 + else: + tunnel_innner_pkt = pkt[scapyall.IPv6].copy() + tunnel_innner_pkt[scapyall.IPv6].hlim -= 1 + tunnel_monitor = tunnel_traffic_monitor( + duthost, + existing=True, + inner_packet=tunnel_innner_pkt, + check_items=["ttl", "queue"] + ) + with tunnel_monitor: + # Those downstream packets are trapped to kernel to learn + # the neighbors, and SONiC needs time to process the zero + # mac and program the tunnel, so there could be packet loss. + # Let's send twice, first round to setup the tunnel. + testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) + time.sleep(5) + testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) + time.sleep(5) + + def _verify_failed_neighbor(duthost, target_ip): + result = duthost.shell("ip neighbor show %s" % target_ip)["stdout"] + pytest_assert("FAILED" in result) + + def _check_mux_status(duthost, target_status): + all_mux_status = json.loads(duthost.shell("show mux status --json")["stdout"])["MUX_CABLE"] + mux_status = {port: status for port, status in list(all_mux_status.items()) if port in active_active_ports} + for port in mux_status: + status = mux_status[port]["STATUS"].lower() + if status != target_status: + return False + return True + + logging.info("check upper tor %s", upper_tor_host) + _verify_traffic(upper_tor_host, constants.target_ip) + _verify_traffic(upper_tor_host, constants.target_ipv6) + _verify_failed_neighbor(upper_tor_host, constants.target_ip) + _verify_failed_neighbor(upper_tor_host, constants.target_ipv6) + + logging.info("check lower tor %s", upper_tor_host) + _verify_traffic(lower_tor_host, constants.target_ip) + _verify_traffic(lower_tor_host, constants.target_ipv6) + _verify_failed_neighbor(lower_tor_host, constants.target_ip) + _verify_failed_neighbor(lower_tor_host, constants.target_ipv6) + + if cable_type == CableType.active_active: + try: + logging.info("toggle lower tor %s to standby", lower_tor_host) + lower_tor_host.shell("config mux mode standby all") + wait_until(30, 5, 0, lambda: _check_mux_status(lower_tor_host, "standby")) + _verify_traffic(lower_tor_host, constants.target_ip) + _verify_failed_neighbor(lower_tor_host, constants.target_ip) + _verify_failed_neighbor(lower_tor_host, constants.target_ip) + _verify_failed_neighbor(lower_tor_host, constants.target_ipv6) + finally: + lower_tor_host.shell("config mux mode auto all") diff --git a/tests/dualtor/test_standby_tor_upstream_mux_toggle.py b/tests/dualtor/test_standby_tor_upstream_mux_toggle.py index 7426ce61f9..e776541229 100644 --- a/tests/dualtor/test_standby_tor_upstream_mux_toggle.py +++ b/tests/dualtor/test_standby_tor_upstream_mux_toggle.py @@ -10,8 +10,7 @@ from tests.common.config_reload import config_reload from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses, run_garp_service, \ - run_icmp_responder # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 + run_icmp_responder, skip_traffic_test # noqa F401 logger = logging.getLogger(__file__) @@ -34,8 +33,8 @@ def test_cleanup(rand_selected_dut): def test_standby_tor_upstream_mux_toggle( - rand_selected_dut, tbinfo, ptfadapter, rand_selected_interface, # noqa F811 - toggle_all_simulator_ports, set_crm_polling_interval): # noqa F811 + rand_selected_dut, tbinfo, ptfadapter, rand_selected_interface, # noqa F811 + toggle_all_simulator_ports, set_crm_polling_interval, skip_traffic_test): # noqa F811 itfs, ip = rand_selected_interface PKT_NUM = 100 # Step 1. Set mux state to standby and verify traffic is dropped by ACL rule and drop counters incremented @@ -50,7 +49,8 @@ def test_standby_tor_upstream_mux_toggle( itfs=itfs, server_ip=ip['server_ipv4'].split('/')[0], pkt_num=PKT_NUM, - drop=True) + drop=True, + skip_traffic_test=skip_traffic_test) time.sleep(5) # Step 2. Toggle mux state to active, and verify traffic is not dropped by ACL and fwd-ed to uplinks; @@ -65,7 +65,8 @@ def test_standby_tor_upstream_mux_toggle( itfs=itfs, server_ip=ip['server_ipv4'].split('/')[0], pkt_num=PKT_NUM, - drop=False) + drop=False, + skip_traffic_test=skip_traffic_test) # Step 3. Toggle mux state to standby, and verify traffic is dropped by ACL; # verify CRM show and no nexthop objects are stale @@ -79,7 +80,8 @@ def test_standby_tor_upstream_mux_toggle( itfs=itfs, server_ip=ip['server_ipv4'].split('/')[0], pkt_num=PKT_NUM, - drop=True) + drop=True, + skip_traffic_test=skip_traffic_test) crm_facts1 = rand_selected_dut.get_crm_facts() unmatched_crm_facts = compare_crm_facts(crm_facts0, crm_facts1) pt_assert(len(unmatched_crm_facts) == 0, 'Unmatched CRM facts: {}' diff --git a/tests/dualtor/test_switchover_failure.py b/tests/dualtor/test_switchover_failure.py index 2404838d42..467790c540 100644 --- a/tests/dualtor/test_switchover_failure.py +++ b/tests/dualtor/test_switchover_failure.py @@ -11,7 +11,6 @@ from tests.common.fixtures.ptfhost_utils import run_icmp_responder, run_garp_service # noqa: F401 from tests.common.utilities import wait_until from tests.common.dualtor.dual_tor_common import cable_type, CableType # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/dualtor/test_tor_ecn.py b/tests/dualtor/test_tor_ecn.py index 9b01b9892b..5e965dde25 100644 --- a/tests/dualtor/test_tor_ecn.py +++ b/tests/dualtor/test_tor_ecn.py @@ -28,12 +28,12 @@ from tests.common.fixtures.ptfhost_utils import run_icmp_responder # noqa F401 from tests.common.fixtures.ptfhost_utils import run_garp_service # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.utilities import dump_scapy_packet_show_output from tests.common.dualtor.tunnel_traffic_utils import derive_queue_id_from_dscp, derive_out_dscp_from_inner_dscp from tests.common.dualtor.dual_tor_utils import config_active_active_dualtor_active_standby # noqa F401 from tests.common.dualtor.dual_tor_utils import validate_active_active_dualtor_setup # noqa F401 from tests.common.dualtor.dual_tor_utils import is_tunnel_qos_remap_enabled -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology("dualtor") @@ -276,7 +276,7 @@ def test_dscp_to_queue_during_decap_on_active( inner_dscp, ptfhost, setup_dualtor_tor_active, request, rand_selected_interface, ptfadapter, # noqa F811 tbinfo, rand_selected_dut, tunnel_traffic_monitor, # noqa F811 - duthosts, rand_one_dut_hostname + duthosts, rand_one_dut_hostname, skip_traffic_test # noqa F811 ): """ Test if DSCP to Q mapping for inner header is matching with outer header during decap on active @@ -296,6 +296,9 @@ def test_dscp_to_queue_during_decap_on_active( duthost.shell('sonic-clear queuecounters') logging.info("Clearing queue counters before starting traffic") + if skip_traffic_test is True: + logging.info("Skip following test due traffic test skipped") + return with stop_garp(ptfhost): ptfadapter.dataplane.flush() ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) @@ -347,7 +350,8 @@ def test_dscp_to_queue_during_encap_on_standby( duthosts, rand_one_dut_hostname, write_standby, - setup_standby_ports_on_rand_selected_tor # noqa F811 + setup_standby_ports_on_rand_selected_tor, # noqa F811 + skip_traffic_test # noqa F811 ): """ Test if DSCP to Q mapping for outer header is matching with inner header during encap on standby @@ -368,6 +372,9 @@ def test_dscp_to_queue_during_encap_on_standby( ptfadapter.dataplane.flush() ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send IP packet from ptf t1 interface %s", ptf_t1_intf) + if skip_traffic_test is True: + logging.info("Skip following test due traffic test skipped") + return with tunnel_traffic_monitor(tor, existing=True, packet_count=PACKET_NUM): testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), non_encapsulated_packet, count=PACKET_NUM) @@ -376,7 +383,8 @@ def test_dscp_to_queue_during_encap_on_standby( def test_ecn_during_decap_on_active( inner_dscp, ptfhost, setup_dualtor_tor_active, request, rand_selected_interface, ptfadapter, # noqa F811 - tbinfo, rand_selected_dut, tunnel_traffic_monitor # noqa F811 + tbinfo, rand_selected_dut, tunnel_traffic_monitor, # noqa F811 + skip_traffic_test # noqa F811 ): """ Test if the ECN stamping on inner header is matching with outer during decap on active @@ -396,6 +404,10 @@ def test_ecn_during_decap_on_active( exp_tos = encapsulated_packet[IP].payload[IP].tos exp_ecn = exp_tos & 3 + + if skip_traffic_test is True: + logging.info("Skip following test due traffic test skipped") + return with stop_garp(ptfhost): tor.shell("portstat -c") tor.shell("show arp") @@ -412,7 +424,8 @@ def test_ecn_during_encap_on_standby( rand_selected_interface, ptfadapter, # noqa F811 tbinfo, rand_selected_dut, tunnel_traffic_monitor, # noqa F811 write_standby, - setup_standby_ports_on_rand_selected_tor # noqa F811 + setup_standby_ports_on_rand_selected_tor, # noqa F811 + skip_traffic_test # noqa F811 ): """ Test if the ECN stamping on outer header is matching with inner during encap on standby @@ -427,5 +440,8 @@ def test_ecn_during_encap_on_standby( ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send IP packet from ptf t1 interface %s", ptf_t1_intf) + if skip_traffic_test is True: + logging.info("Skip following test due traffic test skipped") + return with tunnel_traffic_monitor(tor, existing=True, packet_count=PACKET_NUM): testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), non_encapsulated_packet, count=PACKET_NUM) diff --git a/tests/dualtor/test_tunnel_memory_leak.py b/tests/dualtor/test_tunnel_memory_leak.py index 09fa861292..748db89747 100644 --- a/tests/dualtor/test_tunnel_memory_leak.py +++ b/tests/dualtor/test_tunnel_memory_leak.py @@ -11,9 +11,9 @@ import time import contextlib from ptf import testutils -from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_upper_tor # noqa: F401 -from tests.common.dualtor.dual_tor_common import cable_type # noqa: F401 -from tests.common.dualtor.dual_tor_utils import upper_tor_host, lower_tor_host # noqa: F401 +from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_upper_tor # noqa F401 +from tests.common.dualtor.dual_tor_common import cable_type # noqa F401 +from tests.common.dualtor.dual_tor_utils import upper_tor_host, lower_tor_host # noqa F401 from tests.common.dualtor.server_traffic_utils import ServerTrafficMonitor from tests.common.helpers.assertions import pytest_assert from tests.common.dualtor.dual_tor_utils import get_t1_ptf_ports @@ -21,9 +21,9 @@ from tests.common.dualtor.dual_tor_utils import build_packet_to_server from tests.common.dualtor.dual_tor_utils import delete_neighbor from tests.common.helpers.dut_utils import get_program_info -from tests.common.fixtures.ptfhost_utils import run_garp_service, run_icmp_responder # noqa: F401 +from tests.common.fixtures.ptfhost_utils import run_garp_service, run_icmp_responder # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -115,8 +115,9 @@ def _check_memory(duthost): return not wait_until(timeout, interval, delay, _check_memory, duthost) -def test_tunnel_memory_leak(toggle_all_simulator_ports_to_upper_tor, upper_tor_host, lower_tor_host, # noqa: F811 - ptfhost, ptfadapter, conn_graph_facts, tbinfo, vmhost, run_arp_responder): # noqa: F811 +def test_tunnel_memory_leak(toggle_all_simulator_ports_to_upper_tor, upper_tor_host, lower_tor_host, # noqa F811 + ptfhost, ptfadapter, conn_graph_facts, tbinfo, vmhost, run_arp_responder, # noqa F811 + skip_traffic_test): # noqa F811 """ Test if there is memory leak for service tunnel_packet_handler. Send ip packets from standby TOR T1 to Server, standby TOR will @@ -170,6 +171,9 @@ def prepare_services(ptfhost): pkt, exp_pkt = build_packet_to_server(lower_tor_host, ptfadapter, server_ipv4) + if skip_traffic_test is True: + logging.info("Skip traffic test.") + continue server_traffic_monitor = ServerTrafficMonitor( upper_tor_host, ptfhost, vmhost, tbinfo, iface, conn_graph_facts, exp_pkt, existing=True, is_mocked=False @@ -183,9 +187,11 @@ def prepare_services(ptfhost): mem_usage, mem_limit, mem_percent = get_memory_info(upper_tor_host) logging.info( "SWSS MEM USAGE:{} LIMIT:{} PERCENT:{}".format(mem_usage, mem_limit, mem_percent)) - pytest_assert(validate_neighbor_entry_exist(upper_tor_host, server_ipv4), - "The server ip {} doesn't exist in neighbor table on dut {}. \ - tunnel_packet_handler isn't triggered.".format(server_ipv4, upper_tor_host.hostname)) + if not skip_traffic_test: + pytest_assert(validate_neighbor_entry_exist(upper_tor_host, server_ipv4), + "The server ip {} doesn't exist in neighbor table on dut {}. \ + tunnel_packet_handler isn't triggered." + .format(server_ipv4, upper_tor_host.hostname)) except Exception as e: logging.error("Capture exception {}, continue the process.".format(repr(e))) if len(server_traffic_monitor.matched_packets) == 0: diff --git a/tests/dualtor_io/test_grpc_server_failure.py b/tests/dualtor_io/test_grpc_server_failure.py index a13bb35c98..8723f72067 100644 --- a/tests/dualtor_io/test_grpc_server_failure.py +++ b/tests/dualtor_io/test_grpc_server_failure.py @@ -14,7 +14,6 @@ from tests.common.dualtor.dual_tor_common import CableType from tests.common.dualtor.nic_simulator_control import stop_nic_grpc_server # noqa F401 from tests.common.dualtor.nic_simulator_control import restart_nic_simulator # noqa F401 -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ diff --git a/tests/dualtor_io/test_heartbeat_failure.py b/tests/dualtor_io/test_heartbeat_failure.py index becc19afc0..49afb7994a 100644 --- a/tests/dualtor_io/test_heartbeat_failure.py +++ b/tests/dualtor_io/test_heartbeat_failure.py @@ -1,4 +1,5 @@ import pytest +import logging from tests.common.dualtor.control_plane_utils import verify_tor_states from tests.common.dualtor.data_plane_utils import send_t1_to_server_with_action, \ @@ -9,10 +10,10 @@ from tests.common.dualtor.tor_failure_utils import shutdown_tor_heartbeat # noqa F401 from tests.common.fixtures.ptfhost_utils import run_icmp_responder, run_garp_service, \ copy_ptftests_directory, change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.dualtor.constants import MUX_SIM_ALLOWED_DISRUPTION_SEC from tests.common.dualtor.dual_tor_common import cable_type # noqa F401 from tests.common.dualtor.dual_tor_common import CableType -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -36,15 +37,17 @@ def ignore_expected_loganalyzer_exception(loganalyzer, duthosts): def test_active_tor_heartbeat_failure_upstream( toggle_all_simulator_ports_to_upper_tor, upper_tor_host, lower_tor_host, # noqa F811 - send_server_to_t1_with_action, shutdown_tor_heartbeat, cable_type # noqa F811 + send_server_to_t1_with_action, shutdown_tor_heartbeat, cable_type, skip_traffic_test # noqa F811 ): """ Send upstream traffic and stop the LinkProber module on the active ToR. Confirm switchover and disruption lasts < 1 second. """ + logging.info("skip_traffic_test: {}".format(skip_traffic_test)) send_server_to_t1_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: shutdown_tor_heartbeat(upper_tor_host) + action=lambda: shutdown_tor_heartbeat(upper_tor_host), + skip_traffic_test=skip_traffic_test ) if cable_type == CableType.active_standby: @@ -65,7 +68,7 @@ def test_active_tor_heartbeat_failure_upstream( @pytest.mark.enable_active_active def test_active_tor_heartbeat_failure_downstream_active( toggle_all_simulator_ports_to_upper_tor, upper_tor_host, lower_tor_host, # noqa F811 - send_t1_to_server_with_action, shutdown_tor_heartbeat, cable_type # noqa F811 + send_t1_to_server_with_action, shutdown_tor_heartbeat, cable_type, skip_traffic_test # noqa F811 ): """ Send downstream traffic from T1 to the active ToR and stop the LinkProber module on the active ToR. @@ -73,7 +76,8 @@ def test_active_tor_heartbeat_failure_downstream_active( """ send_t1_to_server_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: shutdown_tor_heartbeat(upper_tor_host) + action=lambda: shutdown_tor_heartbeat(upper_tor_host), + skip_traffic_test=skip_traffic_test ) if cable_type == CableType.active_standby: @@ -93,14 +97,15 @@ def test_active_tor_heartbeat_failure_downstream_active( def test_active_tor_heartbeat_failure_downstream_standby( toggle_all_simulator_ports_to_upper_tor, upper_tor_host, lower_tor_host, # noqa F811 - send_t1_to_server_with_action, shutdown_tor_heartbeat): # noqa F811 + send_t1_to_server_with_action, shutdown_tor_heartbeat, skip_traffic_test): # noqa F811 """ Send downstream traffic from T1 to the standby ToR and stop the LinkProber module on the active ToR. Confirm switchover and disruption lasts < 1 second. """ send_t1_to_server_with_action( lower_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: shutdown_tor_heartbeat(upper_tor_host) + action=lambda: shutdown_tor_heartbeat(upper_tor_host), + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -110,14 +115,15 @@ def test_active_tor_heartbeat_failure_downstream_standby( def test_standby_tor_heartbeat_failure_upstream( toggle_all_simulator_ports_to_upper_tor, upper_tor_host, lower_tor_host, # noqa F811 - send_server_to_t1_with_action, shutdown_tor_heartbeat): # noqa F811 + send_server_to_t1_with_action, shutdown_tor_heartbeat, skip_traffic_test): # noqa F811 """ Send upstream traffic and stop the LinkProber module on the standby ToR. Confirm no switchover and no disruption. """ send_server_to_t1_with_action( upper_tor_host, verify=True, - action=lambda: shutdown_tor_heartbeat(lower_tor_host) + action=lambda: shutdown_tor_heartbeat(lower_tor_host), + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, @@ -127,14 +133,15 @@ def test_standby_tor_heartbeat_failure_upstream( def test_standby_tor_heartbeat_failure_downstream_active( toggle_all_simulator_ports_to_upper_tor, upper_tor_host, lower_tor_host, # noqa F811 - send_t1_to_server_with_action, shutdown_tor_heartbeat): # noqa F811 + send_t1_to_server_with_action, shutdown_tor_heartbeat, skip_traffic_test): # noqa F811 """ Send downstream traffic from T1 to the active ToR and stop the LinkProber module on the standby ToR. Confirm no switchover and no disruption. """ send_t1_to_server_with_action( upper_tor_host, verify=True, - action=lambda: shutdown_tor_heartbeat(lower_tor_host) + action=lambda: shutdown_tor_heartbeat(lower_tor_host), + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, @@ -144,14 +151,15 @@ def test_standby_tor_heartbeat_failure_downstream_active( def test_standby_tor_heartbeat_failure_downstream_standby( toggle_all_simulator_ports_to_upper_tor, upper_tor_host, lower_tor_host, # noqa F811 - send_t1_to_server_with_action, shutdown_tor_heartbeat): # noqa F811 + send_t1_to_server_with_action, shutdown_tor_heartbeat, skip_traffic_test): # noqa F811 """ Send downstream traffic from T1 to the standby ToR and stop the LinkProber module on the standby ToR. Confirm no switchover and no disruption. """ send_t1_to_server_with_action( lower_tor_host, verify=True, - action=lambda: shutdown_tor_heartbeat(lower_tor_host) + action=lambda: shutdown_tor_heartbeat(lower_tor_host), + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, diff --git a/tests/dualtor_io/test_link_drop.py b/tests/dualtor_io/test_link_drop.py index 08ed22b668..27909f1302 100644 --- a/tests/dualtor_io/test_link_drop.py +++ b/tests/dualtor_io/test_link_drop.py @@ -18,13 +18,13 @@ from tests.common.fixtures.ptfhost_utils import run_icmp_responder, run_garp_service # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.dualtor.constants import MUX_SIM_ALLOWED_DISRUPTION_SEC from tests.common.dualtor.dual_tor_common import ActiveActivePortID from tests.common.dualtor.dual_tor_common import active_active_ports # noqa F401 from tests.common.dualtor.dual_tor_common import active_standby_ports # noqa F401 from tests.common.dualtor.dual_tor_common import cable_type # noqa F401 from tests.common.dualtor.dual_tor_common import CableType -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -97,7 +97,7 @@ def _drop_flow_upper_tor_active_active(): def test_active_link_drop_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, drop_flow_upper_tor_all, # noqa F811 - drop_flow_upper_tor_active_active, cable_type # noqa F811 + drop_flow_upper_tor_active_active, cable_type, skip_traffic_test # noqa F811 ): """ Send traffic from servers to T1 and remove the flow between the servers and the active ToR. @@ -109,7 +109,8 @@ def test_active_link_drop_upstream( verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, allowed_disruption=3, - action=drop_flow_upper_tor_all + action=drop_flow_upper_tor_all, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -124,7 +125,8 @@ def test_active_link_drop_upstream( verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, allowed_disruption=1, - action=drop_flow_upper_tor_active_active + action=drop_flow_upper_tor_active_active, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -139,7 +141,7 @@ def test_active_link_drop_upstream( def test_active_link_drop_downstream_active( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, drop_flow_upper_tor_all, # noqa F811 - drop_flow_upper_tor_active_active, cable_type # noqa F811 + drop_flow_upper_tor_active_active, cable_type, skip_traffic_test # noqa F811 ): """ Send traffic from the T1s to the servers via the active Tor and remove the flow between the @@ -152,7 +154,8 @@ def test_active_link_drop_downstream_active( verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, allowed_disruption=3, - action=drop_flow_upper_tor_all + action=drop_flow_upper_tor_all, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -167,7 +170,8 @@ def test_active_link_drop_downstream_active( verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, allowed_disruption=1, - action=drop_flow_upper_tor_active_active + action=drop_flow_upper_tor_active_active, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -180,7 +184,8 @@ def test_active_link_drop_downstream_active( def test_active_link_drop_downstream_standby( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 - toggle_all_simulator_ports_to_upper_tor, drop_flow_upper_tor_all # noqa F811 + toggle_all_simulator_ports_to_upper_tor, drop_flow_upper_tor_all, # noqa F811 + skip_traffic_test # noqa F811 ): """ Send traffic from the T1s to the servers via the standby Tor and remove the flow between the @@ -192,7 +197,8 @@ def test_active_link_drop_downstream_standby( verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, allowed_disruption=3, - action=drop_flow_upper_tor_all + action=drop_flow_upper_tor_all, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -204,7 +210,7 @@ def test_active_link_drop_downstream_standby( def test_standby_link_drop_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811 check_simulator_flap_counter, drop_flow_lower_tor_all, # noqa F811 - toggle_all_simulator_ports_to_upper_tor # noqa F811 + toggle_all_simulator_ports_to_upper_tor, skip_traffic_test # noqa F811 ): """ Send traffic from servers to T1 and remove the flow between the servers and the standby ToR. @@ -215,7 +221,8 @@ def test_standby_link_drop_upstream( verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, allowed_disruption=2, - action=drop_flow_lower_tor_all + action=drop_flow_lower_tor_all, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, @@ -228,7 +235,7 @@ def test_standby_link_drop_upstream( def test_standby_link_drop_downstream_active( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 check_simulator_flap_counter, drop_flow_lower_tor_all, # noqa F811 - toggle_all_simulator_ports_to_upper_tor # noqa F811 + toggle_all_simulator_ports_to_upper_tor, skip_traffic_test # noqa F811 ): """ Send traffic from the T1s to the servers via the active Tor and remove the flow between the @@ -240,7 +247,8 @@ def test_standby_link_drop_downstream_active( verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, allowed_disruption=2, - action=drop_flow_lower_tor_all + action=drop_flow_lower_tor_all, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, @@ -253,7 +261,7 @@ def test_standby_link_drop_downstream_active( def test_standby_link_drop_downstream_standby( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 check_simulator_flap_counter, drop_flow_lower_tor_all, # noqa F811 - toggle_all_simulator_ports_to_upper_tor # noqa F811 + toggle_all_simulator_ports_to_upper_tor, skip_traffic_test # noqa F811 ): """ Send traffic from the T1s to the servers via the standby Tor and remove the flow between the @@ -265,7 +273,8 @@ def test_standby_link_drop_downstream_standby( verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, allowed_disruption=2, - action=drop_flow_lower_tor_all + action=drop_flow_lower_tor_all, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, diff --git a/tests/dualtor_io/test_link_failure.py b/tests/dualtor_io/test_link_failure.py index 02cb85667f..580f73d805 100644 --- a/tests/dualtor_io/test_link_failure.py +++ b/tests/dualtor_io/test_link_failure.py @@ -11,12 +11,12 @@ from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_upper_tor # noqa F401 from tests.common.fixtures.ptfhost_utils import run_icmp_responder, run_garp_service, \ copy_ptftests_directory, change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.dualtor.constants import MUX_SIM_ALLOWED_DISRUPTION_SEC from tests.common.dualtor.dual_tor_common import active_active_ports # noqa F401 from tests.common.dualtor.dual_tor_common import cable_type # noqa F401 from tests.common.dualtor.dual_tor_common import CableType from tests.common.config_reload import config_reload -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -28,7 +28,7 @@ def test_active_link_down_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_fanout_upper_tor_intfs, cable_type # noqa F811 + shutdown_fanout_upper_tor_intfs, cable_type, skip_traffic_test # noqa F811 ): """ Send traffic from server to T1 and shutdown the active ToR link. @@ -37,7 +37,8 @@ def test_active_link_down_upstream( if cable_type == CableType.active_active: send_server_to_t1_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=1, action=shutdown_fanout_upper_tor_intfs + allowed_disruption=1, action=shutdown_fanout_upper_tor_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -50,7 +51,8 @@ def test_active_link_down_upstream( if cable_type == CableType.active_standby: send_server_to_t1_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=3, action=shutdown_fanout_upper_tor_intfs + allowed_disruption=3, action=shutdown_fanout_upper_tor_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( @@ -65,7 +67,7 @@ def test_active_link_down_upstream( def test_active_link_down_downstream_active( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_fanout_upper_tor_intfs, cable_type # noqa F811 + shutdown_fanout_upper_tor_intfs, cable_type, skip_traffic_test # noqa F811 ): """ Send traffic from T1 to active ToR and shutdown the active ToR link. @@ -74,7 +76,8 @@ def test_active_link_down_downstream_active( if cable_type == CableType.active_standby: send_t1_to_server_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=3, action=shutdown_fanout_upper_tor_intfs + allowed_disruption=3, action=shutdown_fanout_upper_tor_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -85,7 +88,8 @@ def test_active_link_down_downstream_active( if cable_type == CableType.active_active: send_t1_to_server_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=1, action=shutdown_fanout_upper_tor_intfs + allowed_disruption=1, action=shutdown_fanout_upper_tor_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -99,7 +103,7 @@ def test_active_link_down_downstream_active( def test_active_link_down_downstream_standby( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_fanout_upper_tor_intfs # noqa F811 + shutdown_fanout_upper_tor_intfs, skip_traffic_test # noqa F811 ): """ Send traffic from T1 to standby ToR and shutdown the active ToR link. @@ -107,7 +111,8 @@ def test_active_link_down_downstream_standby( """ send_t1_to_server_with_action( lower_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=3, action=shutdown_fanout_upper_tor_intfs + allowed_disruption=3, action=shutdown_fanout_upper_tor_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -119,7 +124,7 @@ def test_active_link_down_downstream_standby( def test_standby_link_down_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_fanout_lower_tor_intfs # noqa F811 + shutdown_fanout_lower_tor_intfs, skip_traffic_test # noqa F811 ): """ Send traffic from server to T1 and shutdown the standby ToR link. @@ -127,7 +132,8 @@ def test_standby_link_down_upstream( """ send_server_to_t1_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=2, action=shutdown_fanout_lower_tor_intfs + allowed_disruption=2, action=shutdown_fanout_lower_tor_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, @@ -139,7 +145,7 @@ def test_standby_link_down_upstream( def test_standby_link_down_downstream_active( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_fanout_lower_tor_intfs # noqa F811 + shutdown_fanout_lower_tor_intfs, skip_traffic_test # noqa F811 ): """ Send traffic from T1 to active ToR and shutdown the standby ToR link. @@ -147,7 +153,8 @@ def test_standby_link_down_downstream_active( """ send_t1_to_server_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=2, action=shutdown_fanout_lower_tor_intfs + allowed_disruption=2, action=shutdown_fanout_lower_tor_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, @@ -159,7 +166,7 @@ def test_standby_link_down_downstream_active( def test_standby_link_down_downstream_standby( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_fanout_lower_tor_intfs # noqa F811 + shutdown_fanout_lower_tor_intfs, skip_traffic_test # noqa F811 ): """ Send traffic from T1 to standby ToR and shutdwon the standby ToR link. @@ -167,7 +174,8 @@ def test_standby_link_down_downstream_standby( """ send_t1_to_server_with_action( lower_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=2, action=shutdown_fanout_lower_tor_intfs + allowed_disruption=2, action=shutdown_fanout_lower_tor_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, @@ -179,7 +187,7 @@ def test_standby_link_down_downstream_standby( def test_active_tor_downlink_down_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_upper_tor_downlink_intfs # noqa F811 + shutdown_upper_tor_downlink_intfs, skip_traffic_test # noqa F811 ): """ Send traffic from server to T1 and shutdown the active ToR downlink on DUT. @@ -187,7 +195,8 @@ def test_active_tor_downlink_down_upstream( """ send_server_to_t1_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=1, action=shutdown_upper_tor_downlink_intfs + allowed_disruption=1, action=shutdown_upper_tor_downlink_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -199,7 +208,7 @@ def test_active_tor_downlink_down_upstream( def test_active_tor_downlink_down_downstream_active( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_upper_tor_downlink_intfs # noqa F811 + shutdown_upper_tor_downlink_intfs, skip_traffic_test # noqa F811 ): """ Send traffic from T1 to active ToR and shutdown the active ToR downlink on DUT. @@ -207,7 +216,8 @@ def test_active_tor_downlink_down_downstream_active( """ send_t1_to_server_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=1, action=shutdown_upper_tor_downlink_intfs + allowed_disruption=1, action=shutdown_upper_tor_downlink_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -219,7 +229,7 @@ def test_active_tor_downlink_down_downstream_active( def test_active_tor_downlink_down_downstream_standby( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_upper_tor_downlink_intfs # noqa F811 + shutdown_upper_tor_downlink_intfs, skip_traffic_test # noqa F811 ): """ Send traffic from T1 to standby ToR and shutdown the active ToR downlink on DUT. @@ -227,7 +237,8 @@ def test_active_tor_downlink_down_downstream_standby( """ send_t1_to_server_with_action( lower_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=1, action=shutdown_upper_tor_downlink_intfs + allowed_disruption=1, action=shutdown_upper_tor_downlink_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -239,7 +250,7 @@ def test_active_tor_downlink_down_downstream_standby( def test_standby_tor_downlink_down_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_lower_tor_downlink_intfs # noqa F811 + shutdown_lower_tor_downlink_intfs, skip_traffic_test # noqa F811 ): """ Send traffic from server to T1 and shutdown the standby ToR downlink on DUT. @@ -247,7 +258,8 @@ def test_standby_tor_downlink_down_upstream( """ send_server_to_t1_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=1, action=shutdown_lower_tor_downlink_intfs + allowed_disruption=1, action=shutdown_lower_tor_downlink_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, @@ -259,7 +271,7 @@ def test_standby_tor_downlink_down_upstream( def test_standby_tor_downlink_down_downstream_active( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_lower_tor_downlink_intfs # noqa F811 + shutdown_lower_tor_downlink_intfs, skip_traffic_test # noqa F811 ): """ Send traffic from T1 to active ToR and shutdown the standby ToR downlink on DUT. @@ -267,7 +279,8 @@ def test_standby_tor_downlink_down_downstream_active( """ send_t1_to_server_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=1, action=shutdown_lower_tor_downlink_intfs + allowed_disruption=1, action=shutdown_lower_tor_downlink_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, @@ -279,7 +292,7 @@ def test_standby_tor_downlink_down_downstream_active( def test_standby_tor_downlink_down_downstream_standby( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_lower_tor_downlink_intfs # noqa F811 + shutdown_lower_tor_downlink_intfs, skip_traffic_test # noqa F811 ): """ Send traffic from T1 to standby ToR and shutdwon the standby ToR downlink on DUT. @@ -287,7 +300,8 @@ def test_standby_tor_downlink_down_downstream_standby( """ send_t1_to_server_with_action( lower_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - allowed_disruption=1, action=shutdown_lower_tor_downlink_intfs + allowed_disruption=1, action=shutdown_lower_tor_downlink_intfs, + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, diff --git a/tests/dualtor_io/test_normal_op.py b/tests/dualtor_io/test_normal_op.py index 90916d8c49..20781585e6 100644 --- a/tests/dualtor_io/test_normal_op.py +++ b/tests/dualtor_io/test_normal_op.py @@ -14,10 +14,10 @@ from tests.common.dualtor.dual_tor_utils import check_simulator_flap_counter # noqa F401 from tests.common.fixtures.ptfhost_utils import run_icmp_responder, run_garp_service, \ copy_ptftests_directory, change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.dualtor.constants import MUX_SIM_ALLOWED_DISRUPTION_SEC, CONFIG_RELOAD_ALLOWED_DISRUPTION_SEC from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -29,16 +29,19 @@ def test_normal_op_upstream(upper_tor_host, lower_tor_host, # noqa F811 send_server_to_t1_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - cable_type): # noqa F811 + cable_type, # noqa F811 + skip_traffic_test): # noqa F811 """Send upstream traffic and confirm no disruption or switchover occurs""" if cable_type == CableType.active_standby: - send_server_to_t1_with_action(upper_tor_host, verify=True, stop_after=60) + send_server_to_t1_with_action(upper_tor_host, verify=True, + stop_after=60, skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host, skip_tunnel_route=False) if cable_type == CableType.active_active: - send_server_to_t1_with_action(upper_tor_host, verify=True, stop_after=60) + send_server_to_t1_with_action(upper_tor_host, verify=True, + stop_after=60, skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=[upper_tor_host, lower_tor_host], expected_standby_host=None, cable_type=cable_type, @@ -49,18 +52,21 @@ def test_normal_op_upstream(upper_tor_host, lower_tor_host, # noqa F def test_normal_op_downstream_upper_tor(upper_tor_host, lower_tor_host, # noqa F811 send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - cable_type): # noqa F811 + cable_type, # noqa F811 + skip_traffic_test): # noqa F811 """ Send downstream traffic to the upper ToR and confirm no disruption or switchover occurs """ if cable_type == CableType.active_standby: - send_t1_to_server_with_action(upper_tor_host, verify=True, stop_after=60) + send_t1_to_server_with_action(upper_tor_host, verify=True, + stop_after=60, skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host) if cable_type == CableType.active_active: - send_t1_to_server_with_action(upper_tor_host, verify=True, stop_after=60) + send_t1_to_server_with_action(upper_tor_host, verify=True, + stop_after=60, skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=[upper_tor_host, lower_tor_host], expected_standby_host=None, cable_type=cable_type) @@ -68,20 +74,23 @@ def test_normal_op_downstream_upper_tor(upper_tor_host, lower_tor_host, @pytest.mark.enable_active_active def test_normal_op_downstream_lower_tor(upper_tor_host, lower_tor_host, # noqa F811 - send_t1_to_server_with_action, # noqa F811 - toggle_all_simulator_ports_to_upper_tor, # noqa F811 - cable_type): # noqa F811 + send_t1_to_server_with_action, # noqa F811 + toggle_all_simulator_ports_to_upper_tor, # noqa F811 + cable_type, # noqa F811 + skip_traffic_test): # noqa F811 """ Send downstream traffic to the lower ToR and confirm no disruption or switchover occurs """ if cable_type == CableType.active_standby: - send_t1_to_server_with_action(lower_tor_host, verify=True, stop_after=60) + send_t1_to_server_with_action(lower_tor_host, verify=True, + stop_after=60, skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host) if cable_type == CableType.active_active: - send_t1_to_server_with_action(lower_tor_host, verify=True, stop_after=60) + send_t1_to_server_with_action(lower_tor_host, verify=True, + stop_after=60, skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=[upper_tor_host, lower_tor_host], expected_standby_host=None, cable_type=cable_type) @@ -92,7 +101,8 @@ def test_normal_op_active_server_to_active_server(upper_tor_host, lower_tor_host send_server_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 cable_type, # noqa F811 - select_test_mux_ports): # noqa F811 + select_test_mux_ports, # noqa F811 + skip_traffic_test): # noqa F811 """ Send server to server traffic in active-active setup and confirm no disruption or switchover occurs. """ @@ -100,13 +110,15 @@ def test_normal_op_active_server_to_active_server(upper_tor_host, lower_tor_host test_mux_ports = select_test_mux_ports(cable_type, 2) if cable_type == CableType.active_standby: - send_server_to_server_with_action(upper_tor_host, test_mux_ports, verify=True, stop_after=60) + send_server_to_server_with_action(upper_tor_host, test_mux_ports, verify=True, + stop_after=60, skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host, skip_tunnel_route=False) if cable_type == CableType.active_active: - send_server_to_server_with_action(upper_tor_host, test_mux_ports, verify=True, stop_after=60) + send_server_to_server_with_action(upper_tor_host, test_mux_ports, verify=True, + stop_after=60, skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=[upper_tor_host, lower_tor_host], expected_standby_host=None, cable_type=cable_type, @@ -118,7 +130,8 @@ def test_normal_op_active_server_to_standby_server(upper_tor_host, lower_tor_hos send_server_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 cable_type, force_standby_tor, # noqa F811 - select_test_mux_ports): # noqa F811 + select_test_mux_ports, # noqa F811 + skip_traffic_test): # noqa F811 """ Send server to server traffic in active-standby setup and confirm no disruption or switchover occurs. """ @@ -134,10 +147,12 @@ def _is_mux_port_standby(duthost, mux_port): "failed to toggle mux port %s to standby on DUT %s" % (tx_mux_port, upper_tor_host.hostname)) if cable_type == CableType.active_standby: - send_server_to_server_with_action(upper_tor_host, test_mux_ports, verify=True, stop_after=60) + send_server_to_server_with_action(upper_tor_host, test_mux_ports, verify=True, + stop_after=60, skip_traffic_test=skip_traffic_test) if cable_type == CableType.active_active: - send_server_to_server_with_action(upper_tor_host, test_mux_ports, verify=True, stop_after=60) + send_server_to_server_with_action(upper_tor_host, test_mux_ports, verify=True, + stop_after=60, skip_traffic_test=skip_traffic_test) # TODO: Add per-port db check @@ -147,7 +162,8 @@ def _is_mux_port_standby(duthost, mux_port): def test_upper_tor_config_reload_upstream(upper_tor_host, lower_tor_host, # noqa F811 send_server_to_t1_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - cable_type): # noqa F811 + cable_type, # noqa F811 + skip_traffic_test): # noqa F811 """ Send upstream traffic and `config reload` the active ToR. Confirm switchover occurs and disruption lasted < 1 second for active-standby ports. @@ -155,13 +171,15 @@ def test_upper_tor_config_reload_upstream(upper_tor_host, lower_tor_host, """ if cable_type == CableType.active_standby: send_server_to_t1_with_action(upper_tor_host, verify=True, delay=CONFIG_RELOAD_ALLOWED_DISRUPTION_SEC, - action=lambda: config_reload(upper_tor_host, wait=0)) + action=lambda: config_reload(upper_tor_host, wait=0), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host) if cable_type == CableType.active_active: send_server_to_t1_with_action(upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: config_reload(upper_tor_host, wait=0)) + action=lambda: config_reload(upper_tor_host, wait=0), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=[upper_tor_host, lower_tor_host], expected_standby_host=None, cable_type=cable_type) @@ -171,14 +189,16 @@ def test_upper_tor_config_reload_upstream(upper_tor_host, lower_tor_host, def test_lower_tor_config_reload_upstream(upper_tor_host, lower_tor_host, # noqa F811 send_server_to_t1_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - cable_type): # noqa F811 + cable_type, # noqa F811 + skip_traffic_test): # noqa F811 """ Send upstream traffic and `config reload` the lower ToR. Confirm no switchover occurs and no disruption. """ if cable_type == CableType.active_standby: send_server_to_t1_with_action(upper_tor_host, verify=True, - action=lambda: config_reload(lower_tor_host, wait=0)) + action=lambda: config_reload(lower_tor_host, wait=0), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host) @@ -188,20 +208,23 @@ def test_lower_tor_config_reload_upstream(upper_tor_host, lower_tor_host, def test_lower_tor_config_reload_downstream_upper_tor(upper_tor_host, lower_tor_host, # noqa F811 send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - cable_type): # noqa F811 + cable_type, # noqa F811 + skip_traffic_test): # noqa F811 """ Send downstream traffic to the upper ToR and `config reload` the lower ToR. Confirm no switchover occurs and no disruption """ if cable_type == CableType.active_standby: send_t1_to_server_with_action(upper_tor_host, verify=True, - action=lambda: config_reload(lower_tor_host, wait=0)) + action=lambda: config_reload(lower_tor_host, wait=0), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host) if cable_type == CableType.active_active: send_t1_to_server_with_action(upper_tor_host, verify=True, - action=lambda: config_reload(lower_tor_host, wait=0)) + action=lambda: config_reload(lower_tor_host, wait=0), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=[upper_tor_host, lower_tor_host], expected_standby_host=None, cable_type=cable_type) @@ -211,7 +234,8 @@ def test_lower_tor_config_reload_downstream_upper_tor(upper_tor_host, lower_tor_ def test_upper_tor_config_reload_downstream_lower_tor(upper_tor_host, lower_tor_host, # noqa F811 send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - cable_type): # noqa F811 + cable_type, # noqa F811 + skip_traffic_test): # noqa F811 """ Send downstream traffic to the lower ToR and `config reload` the upper ToR. Confirm switchover occurs and disruption lasts < 1 second for active-standby ports. @@ -219,7 +243,8 @@ def test_upper_tor_config_reload_downstream_lower_tor(upper_tor_host, lower_tor_ """ if cable_type == CableType.active_standby: send_t1_to_server_with_action(lower_tor_host, verify=True, delay=CONFIG_RELOAD_ALLOWED_DISRUPTION_SEC, - action=lambda: config_reload(upper_tor_host, wait=0)) + action=lambda: config_reload(upper_tor_host, wait=0), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host) @@ -229,7 +254,8 @@ def test_tor_switch_upstream(upper_tor_host, lower_tor_host, # no send_server_to_t1_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 force_active_tor, force_standby_tor, # noqa F811 - cable_type): # noqa F811 + cable_type, # noqa F811 + skip_traffic_test): # noqa F811 """ Send upstream traffic and perform switchover via CLI. Confirm switchover occurs and disruption lasts < 1 second for active-standby ports. @@ -237,13 +263,15 @@ def test_tor_switch_upstream(upper_tor_host, lower_tor_host, # no """ if cable_type == CableType.active_standby: send_server_to_t1_with_action(upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: force_active_tor(lower_tor_host, 'all')) + action=lambda: force_active_tor(lower_tor_host, 'all'), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host) if cable_type == CableType.active_active: send_server_to_t1_with_action(upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: force_standby_tor(upper_tor_host, 'all')) + action=lambda: force_standby_tor(upper_tor_host, 'all'), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host, expected_standby_health="healthy", @@ -255,7 +283,8 @@ def test_tor_switch_downstream_active(upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 force_active_tor, force_standby_tor, # noqa F811 - cable_type): # noqa F811 + cable_type, # noqa F811 + skip_traffic_test): # noqa F811 """ Send downstream traffic to the upper ToR and perform switchover via CLI. Confirm switchover occurs and disruption lasts < 1 second for active-standby ports. @@ -263,13 +292,15 @@ def test_tor_switch_downstream_active(upper_tor_host, lower_tor_host, """ if cable_type == CableType.active_standby: send_t1_to_server_with_action(upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: force_active_tor(lower_tor_host, 'all')) + action=lambda: force_active_tor(lower_tor_host, 'all'), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host) if cable_type == CableType.active_active: send_t1_to_server_with_action(upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: force_standby_tor(upper_tor_host, 'all')) + action=lambda: force_standby_tor(upper_tor_host, 'all'), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host, expected_standby_health="healthy", @@ -281,7 +312,8 @@ def test_tor_switch_downstream_standby(upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 force_active_tor, force_standby_tor, # noqa F811 - cable_type): # noqa F811 + cable_type, # noqa F811 + skip_traffic_test): # noqa F811 """ Send downstream traffic to the lower ToR and perform switchover via CLI. Confirm switchover occurs and disruption lasts < 1 second for active-standby ports. @@ -289,13 +321,15 @@ def test_tor_switch_downstream_standby(upper_tor_host, lower_tor_host, """ if cable_type == CableType.active_standby: send_t1_to_server_with_action(lower_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: force_active_tor(lower_tor_host, 'all')) + action=lambda: force_active_tor(lower_tor_host, 'all'), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host) if cable_type == CableType.active_active: send_t1_to_server_with_action(lower_tor_host, verify=True, - action=lambda: force_standby_tor(upper_tor_host, 'all')) + action=lambda: force_standby_tor(upper_tor_host, 'all'), + skip_traffic_test=skip_traffic_test) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host, expected_standby_health="healthy", @@ -323,7 +357,7 @@ def _is_mux_port_standby(duthost, mux_port): send_server_to_server_with_action(upper_tor_host, test_mux_ports, verify=True, action=lambda: force_standby_tor(upper_tor_host, [tx_mux_port]), send_interval=0.0035, - stop_after=60) + stop_after=60,) pytest_assert(_is_mux_port_standby(upper_tor_host, tx_mux_port), "mux port %s on DUT %s failed to toggle to standby" % (upper_tor_host.hostname, tx_mux_port)) diff --git a/tests/dualtor_io/test_tor_bgp_failure.py b/tests/dualtor_io/test_tor_bgp_failure.py index 03f37be0ed..91783bd14f 100644 --- a/tests/dualtor_io/test_tor_bgp_failure.py +++ b/tests/dualtor_io/test_tor_bgp_failure.py @@ -11,11 +11,11 @@ from tests.common.dualtor.tor_failure_utils import shutdown_bgp_sessions_on_duthost from tests.common.fixtures.ptfhost_utils import run_icmp_responder, run_garp_service, \ copy_ptftests_directory, change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.dualtor.tunnel_traffic_utils import tunnel_traffic_monitor # noqa F401 from tests.common.dualtor.constants import MUX_SIM_ALLOWED_DISRUPTION_SEC from tests.common.dualtor.dual_tor_common import cable_type # noqa F401 from tests.common.dualtor.dual_tor_common import CableType -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -64,7 +64,11 @@ def ignore_expected_loganalyzer_exception(loganalyzer, duthosts): ignore_errors = [ r".* ERR bgp#bgpmon: \*ERROR\* Failed with rc:1 when execute: vtysh -c 'show bgp summary json'", - r".* ERR bgp#bgpmon: \*ERROR\* Failed with rc:1 when execute: \['vtysh', '-c', 'show bgp summary json'\]" + r".* ERR bgp#bgpmon: \*ERROR\* Failed with rc:1 when execute: \['vtysh', '-c', 'show bgp summary json'\]", + r".* ERR syncd#syncd: .*SAI_API_TUNNEL:_brcm_sai_mptnl_tnl_route_event_add:\d+ ecmp table entry lookup " + "failed with error.*", + r".* ERR syncd#syncd: .*SAI_API_TUNNEL:_brcm_sai_mptnl_process_route_add_mode_default_and_host:\d+ " + "_brcm_sai_mptnl_tnl_route_event_add failed with error.*" ] if loganalyzer: @@ -76,7 +80,7 @@ def ignore_expected_loganalyzer_exception(loganalyzer, duthosts): def test_active_tor_kill_bgpd_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811 - toggle_all_simulator_ports_to_upper_tor, kill_bgpd): # noqa F811 + toggle_all_simulator_ports_to_upper_tor, kill_bgpd, skip_traffic_test): # noqa F811 ''' Case: Server -> ToR -> T1 (Active ToR BGP Down) Action: Shutdown all BGP sessions on the active ToR @@ -88,7 +92,8 @@ def test_active_tor_kill_bgpd_upstream( ''' send_server_to_t1_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: kill_bgpd(upper_tor_host) + action=lambda: kill_bgpd(upper_tor_host), + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -98,7 +103,7 @@ def test_active_tor_kill_bgpd_upstream( def test_standby_tor_kill_bgpd_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811 - toggle_all_simulator_ports_to_upper_tor, kill_bgpd): # noqa F811 + toggle_all_simulator_ports_to_upper_tor, kill_bgpd, skip_traffic_test): # noqa F811 ''' Case: Server -> ToR -> T1 (Standby ToR BGP Down) Action: Shutdown all BGP sessions on the standby ToR @@ -109,7 +114,8 @@ def test_standby_tor_kill_bgpd_upstream( ''' send_server_to_t1_with_action( upper_tor_host, verify=True, - action=lambda: kill_bgpd(lower_tor_host) + action=lambda: kill_bgpd(lower_tor_host), + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, @@ -120,7 +126,7 @@ def test_standby_tor_kill_bgpd_upstream( def test_standby_tor_kill_bgpd_downstream_active( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, kill_bgpd, # noqa F811 - tunnel_traffic_monitor): # noqa F811 + tunnel_traffic_monitor, skip_traffic_test): # noqa F811 ''' Case: T1 -> Active ToR -> Server (Standby ToR BGP Down) Action: Shutdown all BGP sessions on the standby ToR @@ -131,7 +137,8 @@ def test_standby_tor_kill_bgpd_downstream_active( with tunnel_traffic_monitor(lower_tor_host, existing=False): send_t1_to_server_with_action( upper_tor_host, verify=True, - action=lambda: kill_bgpd(lower_tor_host) + action=lambda: kill_bgpd(lower_tor_host), + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=upper_tor_host, @@ -142,7 +149,7 @@ def test_standby_tor_kill_bgpd_downstream_active( def test_active_tor_kill_bgpd_downstream_standby( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, kill_bgpd, # noqa F811 - tunnel_traffic_monitor): # noqa F811 + tunnel_traffic_monitor, skip_traffic_test): # noqa F811 ''' Case: T1 -> Standby ToR -> Server (Active ToR BGP Down) Action: Shutdown all BGP sessions on the active ToR @@ -153,7 +160,8 @@ def test_active_tor_kill_bgpd_downstream_standby( ''' send_t1_to_server_with_action( lower_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: kill_bgpd(upper_tor_host) + action=lambda: kill_bgpd(upper_tor_host), + skip_traffic_test=skip_traffic_test ) verify_tor_states( expected_active_host=lower_tor_host, @@ -165,7 +173,7 @@ def test_active_tor_kill_bgpd_downstream_standby( def test_active_tor_shutdown_bgp_sessions_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, # noqa F811 toggle_all_simulator_ports_to_upper_tor, # noqa F811 - shutdown_bgp_sessions, cable_type # noqa F811 + shutdown_bgp_sessions, cable_type, skip_traffic_test # noqa F811 ): """ Case: Server -> ToR -> T1 (Active ToR BGP Down) @@ -179,13 +187,15 @@ def test_active_tor_shutdown_bgp_sessions_upstream( if cable_type == CableType.active_standby: send_server_to_t1_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: shutdown_bgp_sessions(upper_tor_host) + action=lambda: shutdown_bgp_sessions(upper_tor_host), + skip_traffic_test=skip_traffic_test ) if cable_type == CableType.active_active: send_server_to_t1_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, - action=lambda: shutdown_bgp_sessions(upper_tor_host) + action=lambda: shutdown_bgp_sessions(upper_tor_host), + skip_traffic_test=skip_traffic_test ) if cable_type == CableType.active_active: diff --git a/tests/dualtor_io/test_tor_failure.py b/tests/dualtor_io/test_tor_failure.py index ca3d0499cf..98d3f4b18c 100644 --- a/tests/dualtor_io/test_tor_failure.py +++ b/tests/dualtor_io/test_tor_failure.py @@ -20,7 +20,6 @@ from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/dualtor_mgmt/test_dualtor_bgp_update_delay.py b/tests/dualtor_mgmt/test_dualtor_bgp_update_delay.py index a893dc2482..009e0ee07d 100644 --- a/tests/dualtor_mgmt/test_dualtor_bgp_update_delay.py +++ b/tests/dualtor_mgmt/test_dualtor_bgp_update_delay.py @@ -14,7 +14,6 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ diff --git a/tests/dualtor_mgmt/test_egress_drop_nvidia.py b/tests/dualtor_mgmt/test_egress_drop_nvidia.py index 6bc8022235..e91b0c2eb5 100644 --- a/tests/dualtor_mgmt/test_egress_drop_nvidia.py +++ b/tests/dualtor_mgmt/test_egress_drop_nvidia.py @@ -15,7 +15,6 @@ from tests.common.utilities import wait_until from ptf.testutils import simple_tcp_packet, simple_ipv4ip_packet from tests.common.plugins.allure_wrapper import allure_step_wrapper as allure -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ pytest.mark.topology('dualtor') diff --git a/tests/dualtor_mgmt/test_grpc_periodical_sync.py b/tests/dualtor_mgmt/test_grpc_periodical_sync.py index d553bb1352..0290cd1b72 100644 --- a/tests/dualtor_mgmt/test_grpc_periodical_sync.py +++ b/tests/dualtor_mgmt/test_grpc_periodical_sync.py @@ -17,7 +17,6 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ diff --git a/tests/dualtor_mgmt/test_ingress_drop.py b/tests/dualtor_mgmt/test_ingress_drop.py index 05c7e88de6..c98be9db04 100644 --- a/tests/dualtor_mgmt/test_ingress_drop.py +++ b/tests/dualtor_mgmt/test_ingress_drop.py @@ -16,10 +16,10 @@ from tests.common.dualtor.nic_simulator_control import mux_status_from_nic_simulator # noqa F401 from tests.common.dualtor.nic_simulator_control import stop_nic_simulator # noqa F401 from tests.common.fixtures.ptfhost_utils import run_icmp_responder # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -104,7 +104,7 @@ def selected_mux_port(cable_type, active_active_ports, active_standby_ports): @pytest.mark.enable_active_active -def test_ingress_drop(cable_type, ptfadapter, setup_mux, tbinfo, selected_mux_port, upper_tor_host): # noqa F811 +def test_ingress_drop(cable_type, ptfadapter, setup_mux, tbinfo, selected_mux_port, upper_tor_host, skip_traffic_test): # noqa F811 """ Aims to verify if orchagent installs ingress drop ACL when the port comes to standby. @@ -131,7 +131,7 @@ def test_ingress_drop(cable_type, ptfadapter, setup_mux, tbinfo, selected_mux_po if cable_type == CableType.active_active: verify_upstream_traffic(upper_tor_host, ptfadapter, tbinfo, selected_mux_port, - server_ip, pkt_num=10, drop=False) + server_ip, pkt_num=10, drop=False, skip_traffic_test=skip_traffic_test) elif cable_type == CableType.active_standby: verify_upstream_traffic(upper_tor_host, ptfadapter, tbinfo, selected_mux_port, - server_ip, pkt_num=10, drop=True) + server_ip, pkt_num=10, drop=True, skip_traffic_test=skip_traffic_test) diff --git a/tests/dualtor_mgmt/test_server_failure.py b/tests/dualtor_mgmt/test_server_failure.py index dd5e2f5d57..2bbdc2b014 100644 --- a/tests/dualtor_mgmt/test_server_failure.py +++ b/tests/dualtor_mgmt/test_server_failure.py @@ -17,11 +17,10 @@ from tests.common.fixtures.ptfhost_utils import change_mac_addresses, run_garp_service, \ run_icmp_responder # noqa: F401 from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ - pytest.mark.topology('t0'), + pytest.mark.topology('dualtor'), pytest.mark.usefixtures('run_garp_service', 'run_icmp_responder') ] diff --git a/tests/dualtor_mgmt/test_toggle_mux.py b/tests/dualtor_mgmt/test_toggle_mux.py index cfcdde47e7..996711ee62 100644 --- a/tests/dualtor_mgmt/test_toggle_mux.py +++ b/tests/dualtor_mgmt/test_toggle_mux.py @@ -7,11 +7,10 @@ from tests.common.dualtor.mux_simulator_control import check_mux_status, validate_check_result from tests.common.dualtor.dual_tor_utils import recover_linkmgrd_probe_interval, update_linkmgrd_probe_interval from tests.common.utilities import wait_until -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ - pytest.mark.topology("t0") + pytest.mark.topology("dualtor") ] logger = logging.getLogger(__name__) @@ -37,8 +36,9 @@ def restore_mux_auto_mode(duthosts): def get_interval_v4(duthosts): mux_linkmgr_output = duthosts.shell('sonic-cfggen -d --var-json MUX_LINKMGR') mux_linkmgr = list(mux_linkmgr_output.values())[0]['stdout'] - if len(mux_linkmgr) != 0: - cur_interval_v4 = json.loads(mux_linkmgr)['LINK_PROBER']['interval_v4'] + mux_linkmgr_json = json.loads(mux_linkmgr) + if len(mux_linkmgr) != 0 and 'LINK_PROBER' in mux_linkmgr_json: + cur_interval_v4 = mux_linkmgr_json['LINK_PROBER']['interval_v4'] return cur_interval_v4 else: return None diff --git a/tests/dut_console/test_console_baud_rate.py b/tests/dut_console/test_console_baud_rate.py index 08dd80ae5b..39189b3d11 100644 --- a/tests/dut_console/test_console_baud_rate.py +++ b/tests/dut_console/test_console_baud_rate.py @@ -3,7 +3,6 @@ from tests.common.helpers.assertions import pytest_assert, pytest_require from tests.common.helpers.console_helper import assert_expect_text, create_ssh_client, ensure_console_session_up from tests.common.reboot import reboot -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 pytestmark = [ @@ -22,7 +21,7 @@ def is_sonic_console(conn_graph_facts, dut_hostname): - return conn_graph_facts['device_console_info'][dut_hostname]["Os"] == "sonic" + return conn_graph_facts['device_console_info'][dut_hostname].get("Os", "") == "sonic" def test_console_baud_rate_config(duthost): diff --git a/tests/dut_console/test_escape_character.py b/tests/dut_console/test_escape_character.py index bf7f34f98a..74c2f8ec40 100644 --- a/tests/dut_console/test_escape_character.py +++ b/tests/dut_console/test_escape_character.py @@ -1,6 +1,5 @@ import logging import pytest -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 TOTAL_PACKETS = 100 diff --git a/tests/dut_console/test_idle_timeout.py b/tests/dut_console/test_idle_timeout.py index f5605e02a4..85d50d4102 100644 --- a/tests/dut_console/test_idle_timeout.py +++ b/tests/dut_console/test_idle_timeout.py @@ -3,7 +3,6 @@ import pytest from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.tacacs import tacacs_creds, setup_tacacs # noqa F401 logger = logging.getLogger(__name__) diff --git a/tests/ecmp/inner_hashing/test_inner_hashing.py b/tests/ecmp/inner_hashing/test_inner_hashing.py index 3b2f0aa181..fe45fe6616 100644 --- a/tests/ecmp/inner_hashing/test_inner_hashing.py +++ b/tests/ecmp/inner_hashing/test_inner_hashing.py @@ -13,12 +13,12 @@ from tests.ptf_runner import ptf_runner from tests.ecmp.inner_hashing.conftest import get_src_dst_ip_range, FIB_INFO_FILE_DST,\ VXLAN_PORT, PTF_QLEN, check_pbh_counters, OUTER_ENCAP_FORMATS, NVGRE_TNI, IP_VERSIONS_LIST, config_pbh +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 logger = logging.getLogger(__name__) pytestmark = [ - pytest.mark.topology('t0'), - pytest.mark.asic('mellanox') + pytest.mark.topology('t0') ] update_outer_ipver = random.choice(IP_VERSIONS_LIST) @@ -35,7 +35,7 @@ def setup_dynamic_pbh(self, duthost, vlan_ptf_ports, tbinfo): def test_inner_hashing(self, request, hash_keys, ptfhost, outer_ipver, inner_ipver, router_mac, vlan_ptf_ports, symmetric_hashing, duthost, lag_mem_ptf_ports_groups, - get_function_completeness_level): + get_function_completeness_level, skip_traffic_test): # noqa F811 logging.info("Executing dynamic inner hash test for outer {} and inner {} with symmetric_hashing set to {}" .format(outer_ipver, inner_ipver, str(symmetric_hashing))) with allure.step('Run ptf test InnerHashTest'): @@ -47,7 +47,7 @@ def test_inner_hashing(self, request, hash_keys, ptfhost, outer_ipver, inner_ipv outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver) inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver) - normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough' + normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug' if normalize_level == 'thorough': balancing_test_times = 120 @@ -73,21 +73,22 @@ def test_inner_hashing(self, request, hash_keys, ptfhost, outer_ipver, inner_ipv "symmetric_hashing": symmetric_hashing} duthost.shell("sonic-clear pbh statistics") - ptf_runner(ptfhost, - "ptftests", - "inner_hash_test.InnerHashTest", - platform_dir="ptftests", - params=ptf_params, - log_file=log_file, - qlen=PTF_QLEN, - socket_recv_size=16384, - is_python3=True) + if not skip_traffic_test: + ptf_runner(ptfhost, + "ptftests", + "inner_hash_test.InnerHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=log_file, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True) - retry_call(check_pbh_counters, - fargs=[duthost, outer_ipver, inner_ipver, balancing_test_times, - symmetric_hashing, hash_keys, lag_mem_ptf_ports_groups], - tries=5, - delay=5) + retry_call(check_pbh_counters, + fargs=[duthost, outer_ipver, inner_ipver, balancing_test_times, + symmetric_hashing, hash_keys, lag_mem_ptf_ports_groups], + tries=5, + delay=5) if update_outer_ipver == outer_ipver and update_inner_ipver == inner_ipver: logging.info("Validate dynamic inner hash Edit Flow for outer {} and inner {} ip versions with" @@ -104,6 +105,8 @@ def test_inner_hashing(self, request, hash_keys, ptfhost, outer_ipver, inner_ipv with allure.step('Run again the ptf test InnerHashTest after updating the rules'): logging.info('Run again the ptf test InnerHashTest after updating the rules') duthost.shell("sonic-clear pbh statistics") + if skip_traffic_test is True: + return ptf_runner(ptfhost, "ptftests", "inner_hash_test.InnerHashTest", @@ -125,7 +128,7 @@ def test_inner_hashing(self, request, hash_keys, ptfhost, outer_ipver, inner_ipv class TestStaticInnerHashing(): def test_inner_hashing(self, hash_keys, ptfhost, outer_ipver, inner_ipver, router_mac, - vlan_ptf_ports, symmetric_hashing, lag_mem_ptf_ports_groups): + vlan_ptf_ports, symmetric_hashing, lag_mem_ptf_ports_groups, skip_traffic_test): # noqa F811 logging.info("Executing static inner hash test for outer {} and inner {} with symmetric_hashing set to {}" .format(outer_ipver, inner_ipver, str(symmetric_hashing))) timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') @@ -135,6 +138,8 @@ def test_inner_hashing(self, hash_keys, ptfhost, outer_ipver, inner_ipver, route outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver) inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver) + if skip_traffic_test is True: + return ptf_runner(ptfhost, "ptftests", "inner_hash_test.InnerHashTest", diff --git a/tests/ecmp/inner_hashing/test_inner_hashing_lag.py b/tests/ecmp/inner_hashing/test_inner_hashing_lag.py index 39df82b398..ed61656986 100644 --- a/tests/ecmp/inner_hashing/test_inner_hashing_lag.py +++ b/tests/ecmp/inner_hashing/test_inner_hashing_lag.py @@ -12,12 +12,12 @@ from tests.ptf_runner import ptf_runner from tests.ecmp.inner_hashing.conftest import get_src_dst_ip_range, FIB_INFO_FILE_DST,\ VXLAN_PORT, PTF_QLEN, check_pbh_counters, OUTER_ENCAP_FORMATS, NVGRE_TNI, setup_lag_config, config_pbh_lag +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 logger = logging.getLogger(__name__) pytestmark = [ - pytest.mark.topology('t0'), - pytest.mark.asic('mellanox') + pytest.mark.topology('t0') ] @@ -33,7 +33,7 @@ def setup_dynamic_pbh(self, duthost, lag_port_map, lag_ip_map): def test_inner_hashing(self, hash_keys, ptfhost, outer_ipver, inner_ipver, router_mac, vlan_ptf_ports, symmetric_hashing, duthost, lag_mem_ptf_ports_groups, - get_function_completeness_level): + get_function_completeness_level, skip_traffic_test): # noqa F811 logging.info("Executing dynamic inner hash test for outer {} and inner {} with symmetric_hashing set to {}" .format(outer_ipver, inner_ipver, str(symmetric_hashing))) with allure.step('Run ptf test InnerHashTest'): @@ -45,7 +45,7 @@ def test_inner_hashing(self, hash_keys, ptfhost, outer_ipver, inner_ipver, route outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver) inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver) - normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough' + normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug' if normalize_level == 'thorough': balancing_test_times = 120 @@ -54,6 +54,8 @@ def test_inner_hashing(self, hash_keys, ptfhost, outer_ipver, inner_ipver, route balancing_test_times = 20 balancing_range = 0.5 + if skip_traffic_test is True: + return ptf_runner(ptfhost, "ptftests", "inner_hash_test.InnerHashTest", diff --git a/tests/ecmp/inner_hashing/test_wr_inner_hashing.py b/tests/ecmp/inner_hashing/test_wr_inner_hashing.py index 3e090c3f75..02d697cea3 100644 --- a/tests/ecmp/inner_hashing/test_wr_inner_hashing.py +++ b/tests/ecmp/inner_hashing/test_wr_inner_hashing.py @@ -9,13 +9,13 @@ from tests.ecmp.inner_hashing.conftest import get_src_dst_ip_range, FIB_INFO_FILE_DST, VXLAN_PORT,\ PTF_QLEN, OUTER_ENCAP_FORMATS, NVGRE_TNI, config_pbh from tests.ptf_runner import ptf_runner +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 logger = logging.getLogger(__name__) pytestmark = [ pytest.mark.disable_loganalyzer, - pytest.mark.topology('t0'), - pytest.mark.asic('mellanox') + pytest.mark.topology('t0') ] @@ -29,7 +29,7 @@ def setup_dynamic_pbh(self, duthost, vlan_ptf_ports, tbinfo): def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipver, router_mac, vlan_ptf_ports, symmetric_hashing, localhost, lag_mem_ptf_ports_groups, - get_function_completeness_level): + get_function_completeness_level, skip_traffic_test): # noqa F811 logging.info("Executing warm boot dynamic inner hash test for outer {} and inner {} with symmetric_hashing" " set to {}".format(outer_ipver, inner_ipver, str(symmetric_hashing))) with allure.step('Run ptf test InnerHashTest and warm-reboot in parallel'): @@ -45,7 +45,7 @@ def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipv outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver) inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver) - normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough' + normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug' if normalize_level == 'thorough': balancing_test_times = 200 @@ -57,29 +57,30 @@ def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipv reboot_thr = threading.Thread(target=reboot, args=(duthost, localhost, 'warm', 10, 0, 0, True, True,)) reboot_thr.start() - ptf_runner(ptfhost, - "ptftests", - "inner_hash_test.InnerHashTest", - platform_dir="ptftests", - params={"fib_info": FIB_INFO_FILE_DST, - "router_mac": router_mac, - "src_ports": vlan_ptf_ports, - "exp_port_groups": lag_mem_ptf_ports_groups, - "hash_keys": hash_keys, - "vxlan_port": VXLAN_PORT, - "inner_src_ip_range": ",".join(inner_src_ip_range), - "inner_dst_ip_range": ",".join(inner_dst_ip_range), - "outer_src_ip_range": ",".join(outer_src_ip_range), - "outer_dst_ip_range": ",".join(outer_dst_ip_range), - "balancing_test_times": balancing_test_times, - "balancing_range": balancing_range, - "outer_encap_formats": outer_encap_format, - "nvgre_tni": NVGRE_TNI, - "symmetric_hashing": symmetric_hashing}, - log_file=log_file, - qlen=PTF_QLEN, - socket_recv_size=16384, - is_python3=True) + if not skip_traffic_test: + ptf_runner(ptfhost, + "ptftests", + "inner_hash_test.InnerHashTest", + platform_dir="ptftests", + params={"fib_info": FIB_INFO_FILE_DST, + "router_mac": router_mac, + "src_ports": vlan_ptf_ports, + "exp_port_groups": lag_mem_ptf_ports_groups, + "hash_keys": hash_keys, + "vxlan_port": VXLAN_PORT, + "inner_src_ip_range": ",".join(inner_src_ip_range), + "inner_dst_ip_range": ",".join(inner_dst_ip_range), + "outer_src_ip_range": ",".join(outer_src_ip_range), + "outer_dst_ip_range": ",".join(outer_dst_ip_range), + "balancing_test_times": balancing_test_times, + "balancing_range": balancing_range, + "outer_encap_formats": outer_encap_format, + "nvgre_tni": NVGRE_TNI, + "symmetric_hashing": symmetric_hashing}, + log_file=log_file, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True) reboot_thr.join() @@ -87,7 +88,7 @@ def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipv class TestWRStaticInnerHashing(): def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipver, router_mac, - vlan_ptf_ports, symmetric_hashing, localhost, lag_mem_ptf_ports_groups): + vlan_ptf_ports, symmetric_hashing, localhost, lag_mem_ptf_ports_groups, skip_traffic_test): # noqa F811 logging.info("Executing static inner hash test for outer {} and inner {} with symmetric_hashing set to {}" .format(outer_ipver, inner_ipver, str(symmetric_hashing))) timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') @@ -101,24 +102,25 @@ def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipv reboot_thr = threading.Thread(target=reboot, args=(duthost, localhost, 'warm', 10, 0, 0, True, True,)) reboot_thr.start() - ptf_runner(ptfhost, - "ptftests", - "inner_hash_test.InnerHashTest", - platform_dir="ptftests", - params={"fib_info": FIB_INFO_FILE_DST, - "router_mac": router_mac, - "src_ports": vlan_ptf_ports, - "exp_port_groups": lag_mem_ptf_ports_groups, - "hash_keys": hash_keys, - "vxlan_port": VXLAN_PORT, - "inner_src_ip_range": ",".join(inner_src_ip_range), - "inner_dst_ip_range": ",".join(inner_dst_ip_range), - "outer_src_ip_range": ",".join(outer_src_ip_range), - "outer_dst_ip_range": ",".join(outer_dst_ip_range), - "outer_encap_formats": OUTER_ENCAP_FORMATS, - "symmetric_hashing": symmetric_hashing}, - log_file=log_file, - qlen=PTF_QLEN, - socket_recv_size=16384, - is_python3=True) + if not skip_traffic_test: + ptf_runner(ptfhost, + "ptftests", + "inner_hash_test.InnerHashTest", + platform_dir="ptftests", + params={"fib_info": FIB_INFO_FILE_DST, + "router_mac": router_mac, + "src_ports": vlan_ptf_ports, + "exp_port_groups": lag_mem_ptf_ports_groups, + "hash_keys": hash_keys, + "vxlan_port": VXLAN_PORT, + "inner_src_ip_range": ",".join(inner_src_ip_range), + "inner_dst_ip_range": ",".join(inner_dst_ip_range), + "outer_src_ip_range": ",".join(outer_src_ip_range), + "outer_dst_ip_range": ",".join(outer_dst_ip_range), + "outer_encap_formats": OUTER_ENCAP_FORMATS, + "symmetric_hashing": symmetric_hashing}, + log_file=log_file, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True) reboot_thr.join() diff --git a/tests/ecmp/inner_hashing/test_wr_inner_hashing_lag.py b/tests/ecmp/inner_hashing/test_wr_inner_hashing_lag.py index d34ddf6f5c..6ce69b57d7 100644 --- a/tests/ecmp/inner_hashing/test_wr_inner_hashing_lag.py +++ b/tests/ecmp/inner_hashing/test_wr_inner_hashing_lag.py @@ -9,13 +9,13 @@ from tests.ecmp.inner_hashing.conftest import get_src_dst_ip_range, FIB_INFO_FILE_DST, VXLAN_PORT,\ PTF_QLEN, OUTER_ENCAP_FORMATS, NVGRE_TNI, setup_lag_config, config_pbh_lag from tests.ptf_runner import ptf_runner +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 logger = logging.getLogger(__name__) pytestmark = [ pytest.mark.disable_loganalyzer, - pytest.mark.topology('t0'), - pytest.mark.asic('mellanox') + pytest.mark.topology('t0') ] @@ -31,7 +31,7 @@ def setup_dynamic_pbh(self, duthost, lag_port_map, lag_ip_map): def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipver, router_mac, vlan_ptf_ports, symmetric_hashing, localhost, lag_mem_ptf_ports_groups, - get_function_completeness_level): + get_function_completeness_level, skip_traffic_test): # noqa F811 logging.info("Executing warm boot dynamic inner hash test for outer {} and inner {} with symmetric_hashing" " set to {}".format(outer_ipver, inner_ipver, str(symmetric_hashing))) timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') @@ -46,7 +46,7 @@ def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipv outer_src_ip_range, outer_dst_ip_range = get_src_dst_ip_range(outer_ipver) inner_src_ip_range, inner_dst_ip_range = get_src_dst_ip_range(inner_ipver) - normalize_level = get_function_completeness_level if get_function_completeness_level else 'thorough' + normalize_level = get_function_completeness_level if get_function_completeness_level else 'debug' if normalize_level == 'thorough': balancing_test_times = 200 @@ -58,27 +58,28 @@ def test_inner_hashing(self, duthost, hash_keys, ptfhost, outer_ipver, inner_ipv reboot_thr = threading.Thread(target=reboot, args=(duthost, localhost, 'warm', 10, 0, 0, True, True,)) reboot_thr.start() - ptf_runner(ptfhost, - "ptftests", - "inner_hash_test.InnerHashTest", - platform_dir="ptftests", - params={"fib_info": FIB_INFO_FILE_DST, - "router_mac": router_mac, - "src_ports": vlan_ptf_ports, - "exp_port_groups": lag_mem_ptf_ports_groups, - "hash_keys": hash_keys, - "vxlan_port": VXLAN_PORT, - "inner_src_ip_range": ",".join(inner_src_ip_range), - "inner_dst_ip_range": ",".join(inner_dst_ip_range), - "outer_src_ip_range": ",".join(outer_src_ip_range), - "outer_dst_ip_range": ",".join(outer_dst_ip_range), - "balancing_test_times": balancing_test_times, - "balancing_range": balancing_range, - "outer_encap_formats": outer_encap_format, - "nvgre_tni": NVGRE_TNI, - "symmetric_hashing": symmetric_hashing}, - log_file=log_file, - qlen=PTF_QLEN, - socket_recv_size=16384, - is_python3=True) + if not skip_traffic_test: + ptf_runner(ptfhost, + "ptftests", + "inner_hash_test.InnerHashTest", + platform_dir="ptftests", + params={"fib_info": FIB_INFO_FILE_DST, + "router_mac": router_mac, + "src_ports": vlan_ptf_ports, + "exp_port_groups": lag_mem_ptf_ports_groups, + "hash_keys": hash_keys, + "vxlan_port": VXLAN_PORT, + "inner_src_ip_range": ",".join(inner_src_ip_range), + "inner_dst_ip_range": ",".join(inner_dst_ip_range), + "outer_src_ip_range": ",".join(outer_src_ip_range), + "outer_dst_ip_range": ",".join(outer_dst_ip_range), + "balancing_test_times": balancing_test_times, + "balancing_range": balancing_range, + "outer_encap_formats": outer_encap_format, + "nvgre_tni": NVGRE_TNI, + "symmetric_hashing": symmetric_hashing}, + log_file=log_file, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True) reboot_thr.join() diff --git a/tests/ecmp/test_ecmp_sai_value.py b/tests/ecmp/test_ecmp_sai_value.py index 795b22d46f..05c02e51d5 100644 --- a/tests/ecmp/test_ecmp_sai_value.py +++ b/tests/ecmp/test_ecmp_sai_value.py @@ -126,10 +126,14 @@ def check_config_bcm_file(duthost, topo_type): logging.info("sai_hash_seed_config_hash_offset_enable={}".format(value)) else: logging.info("sai_hash_seed_config_hash_offset_enable not found in the file.") - if topo_type == "t0": - pytest_assert(not cat_output, "sai_hash_seed_config_hash_offset_enable should not set for T0") - if topo_type == "t1": - pytest_assert(cat_output and value == "1", "sai_hash_seed_config_hash_offset_enable is not set to 1") + # with code change https://github.com/sonic-net/sonic-buildimage/pull/18912, + # the sai_hash_seed_config_hash_offset_enable is not set in config.bcm, + # it's set by swss config on 202311 and later image + if "20230531" in duthost.os_version: + if topo_type == "t0": + pytest_assert(not cat_output, "sai_hash_seed_config_hash_offset_enable should not set for T0") + if topo_type == "t1": + pytest_assert(cat_output and value == "1", "sai_hash_seed_config_hash_offset_enable is not set to 1") else: pytest.fail("Config bcm file not found.") @@ -276,7 +280,7 @@ def test_ecmp_offset_value(localhost, duthosts, tbinfo, enum_rand_one_per_hwsku_ elif parameter == "reload": logging.info("Run config reload on DUT") config_reload(duthost, safe_reload=True, check_intf_up_ports=True) - check_hash_seed_value(duthost, asic_name, topo_type) + check_ecmp_offset_value(duthost, asic_name, topo_type, hwsku) elif parameter == "reboot": logging.info("Run cold reboot on DUT") reboot(duthost, localhost, reboot_type=REBOOT_TYPE_COLD, reboot_helper=None, diff --git a/tests/ecmp/test_fgnhg.py b/tests/ecmp/test_fgnhg.py index e3f7aa06c7..9688c78786 100644 --- a/tests/ecmp/test_fgnhg.py +++ b/tests/ecmp/test_fgnhg.py @@ -66,8 +66,8 @@ def configure_interfaces(cfg_facts, duthost, ptfhost, vlan_ip): ptf_to_dut_port_map[ptf_port_id] = port port_list.sort() - bank_0_port = port_list[:len(port_list)/2] - bank_1_port = port_list[len(port_list)/2:] + bank_0_port = port_list[:len(port_list)//2] + bank_1_port = port_list[len(port_list)//2:] # Create vlan if duthost.command('config interface ip add Vlan' + str(DEFAULT_VLAN_ID) + ' ' + str(vlan_ip)) @@ -111,6 +111,9 @@ def generate_fgnhg_config(duthost, ip_to_port, bank_0_port, bank_1_port): def setup_neighbors(duthost, ptfhost, ip_to_port): + duthost.shell("sonic-clear fdb all") + duthost.shell("sonic-clear arp") + duthost.shell("sonic-clear ndp") vlan_name = "Vlan" + str(DEFAULT_VLAN_ID) neigh_entries = {} neigh_entries['NEIGH'] = {} diff --git a/tests/everflow/everflow_test_utilities.py b/tests/everflow/everflow_test_utilities.py index f78a4ce448..97e80743fe 100644 --- a/tests/everflow/everflow_test_utilities.py +++ b/tests/everflow/everflow_test_utilities.py @@ -753,7 +753,8 @@ def send_and_check_mirror_packets(self, src_port=None, dest_ports=None, expect_recv=True, - valid_across_namespace=True): + valid_across_namespace=True, + skip_traffic_test=False): # In Below logic idea is to send traffic in such a way so that mirror traffic # will need to go across namespaces and within namespace. If source and mirror destination @@ -788,6 +789,9 @@ def send_and_check_mirror_packets(self, src_port_set.add(dest_ports[0]) src_port_metadata_map[dest_ports[0]] = (None, 2) + if skip_traffic_test is True: + logging.info("Skipping traffic test") + return # Loop through Source Port Set and send traffic on each source port of the set for src_port in src_port_set: expected_mirror_packet = BaseEverflowTest.get_expected_mirror_packet(mirror_session, @@ -856,9 +860,12 @@ def get_expected_mirror_packet(mirror_session, setup, duthost, direction, mirror payload = binascii.unhexlify("0" * 44) + str(payload) else: payload = binascii.unhexlify("0" * 44) + bytes(payload) - - if duthost.facts["asic_type"] in ["barefoot", "cisco-8000", "innovium"] or duthost.facts.get( - "platform_asic") in ["broadcom-dnx"]: + if ( + duthost.facts["asic_type"] in ["barefoot", "cisco-8000", "innovium"] + or duthost.facts.get("platform_asic") in ["broadcom-dnx"] + or duthost.facts["hwsku"] + in ["rd98DX35xx", "rd98DX35xx_cn9131", "Nokia-7215-A1"] + ): if six.PY2: payload = binascii.unhexlify("0" * 24) + str(payload) else: @@ -884,6 +891,7 @@ def get_expected_mirror_packet(mirror_session, setup, duthost, direction, mirror expected_packet.set_do_not_care_scapy(packet.IP, "chksum") if duthost.facts["asic_type"] == 'marvell': expected_packet.set_do_not_care_scapy(packet.IP, "id") + expected_packet.set_do_not_care_scapy(packet.GRE, "seqnum_present") if duthost.facts["asic_type"] in ["cisco-8000", "innovium"] or \ duthost.facts.get("platform_asic") in ["broadcom-dnx"]: expected_packet.set_do_not_care_scapy(packet.GRE, "seqnum_present") diff --git a/tests/everflow/test_everflow_ipv6.py b/tests/everflow/test_everflow_ipv6.py index d986a484a1..6ffd720f28 100644 --- a/tests/everflow/test_everflow_ipv6.py +++ b/tests/everflow/test_everflow_ipv6.py @@ -1,13 +1,20 @@ """Test cases to support the Everflow IPv6 Mirroring feature in SONiC.""" +import threading import time import pytest import ptf.testutils as testutils +import ipaddress +from ptf import packet +from ptf.mask import Mask +import ptf.packet as scapy from . import everflow_test_utilities as everflow_utils from .everflow_test_utilities import BaseEverflowTest, DOWN_STREAM, UP_STREAM - +import random # Module-level fixtures from .everflow_test_utilities import setup_info # noqa: F401 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa: F401 + pytestmark = [ pytest.mark.topology("t0", "t1", "t2", "m0") ] @@ -83,9 +90,73 @@ def everflow_direction(self, setup_info): # noqa F811 yield direction + @pytest.fixture(scope='function', autouse=True) + def background_traffic(self, ptfadapter, everflow_direction, setup_info): # noqa F811 + stop_thread = threading.Event() + src_port = EverflowIPv6Tests.rx_port_ptf_id + + neigh_ipv6 = {entry['name']: entry['addr'].lower() for entry in ptfadapter.mg_facts['minigraph_bgp'] if + 'ASIC' not in entry['name'] and isinstance(ipaddress.ip_address(entry['addr']), + ipaddress.IPv6Address)} + if len(neigh_ipv6) > 1: + def background_traffic(run_count=None): + selected_addrs1 = random.sample(list(neigh_ipv6.values()), 2) + selected_addrs2 = random.sample(list(neigh_ipv6.values()), 2) + selected_addrs3 = random.sample(list(neigh_ipv6.values()), 2) + + inner_pkt2 = self._base_tcpv6_packet( + everflow_direction, + ptfadapter, + setup_info, + src_ip=selected_addrs1[1], + dst_ip=selected_addrs1[0] + ).getlayer(scapy.IPv6) + + packets = [ + testutils.simple_ipv6ip_packet(ipv6_src=selected_addrs2[0], + eth_src=ptfadapter.dataplane.get_mac(0, 0), + eth_dst=setup_info[everflow_direction]["ingress_router_mac"], + ipv6_dst=selected_addrs2[1], + inner_frame=inner_pkt2), + self._base_tcpv6_packet( + everflow_direction, + ptfadapter, + setup_info, + src_ip=selected_addrs3[0], + dst_ip=selected_addrs3[1] + ) + ] + + count = 0 + while (run_count is None and not stop_thread.is_set()) or (run_count is not None and count < run_count): + time.sleep(.1) + for bkg_trf in packets: + testutils.send(ptfadapter, src_port, bkg_trf) + # Verify packet if run_count is specified + if run_count is not None: + exp_pkt = bkg_trf + exp_pkt = Mask(exp_pkt) + exp_pkt.set_do_not_care_scapy(packet.Ether, 'dst') + exp_pkt.set_do_not_care_scapy(packet.Ether, 'src') + exp_pkt.set_do_not_care_packet(scapy.IPv6, "hlim") + testutils.verify_packet_any_port(ptfadapter, exp_pkt, ports=ptfadapter.ptf_port_set) + count += 1 + + background_traffic(run_count=1) + background_thread = threading.Thread(target=background_traffic) + background_thread.daemon = True + background_thread.start() + + yield + stop_thread.set() + # Wait for the background thread to finish + background_thread.join() + background_traffic(run_count=1) + def test_src_ipv6_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match on Source IPv6 addresses.""" test_packet = self._base_tcpv6_packet( everflow_direction, @@ -99,11 +170,13 @@ def test_src_ipv6_mirroring(self, setup_info, setup_mirror_session, ptfadapter, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_dst_ipv6_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match on Destination IPv6 addresses.""" test_packet = self._base_tcpv6_packet( everflow_direction, @@ -117,11 +190,13 @@ def test_dst_ipv6_mirroring(self, setup_info, setup_mirror_session, ptfadapter, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_next_header_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match on the Next Header field.""" test_packet = self._base_tcpv6_packet(everflow_direction, ptfadapter, setup_info, next_header=0x7E) @@ -130,11 +205,13 @@ def test_next_header_mirroring(self, setup_info, setup_mirror_session, ptfadapte ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_l4_src_port_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match on the L4 Source Port.""" test_packet = self._base_tcpv6_packet(everflow_direction, ptfadapter, setup_info, sport=9000) @@ -143,11 +220,13 @@ def test_l4_src_port_mirroring(self, setup_info, setup_mirror_session, ptfadapte ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_l4_dst_port_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match on the L4 Destination Port.""" test_packet = self._base_tcpv6_packet(everflow_direction, ptfadapter, setup_info, dport=9001) @@ -156,12 +235,14 @@ def test_l4_dst_port_mirroring(self, setup_info, setup_mirror_session, ptfadapte ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_l4_src_port_range_mirroring(self, setup_info, setup_mirror_session, # noqa F811 ptfadapter, everflow_dut, everflow_direction, setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match on a range of L4 Source Ports.""" test_packet = self._base_tcpv6_packet(everflow_direction, ptfadapter, setup_info, sport=10200) @@ -170,12 +251,14 @@ def test_l4_src_port_range_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_l4_dst_port_range_mirroring(self, setup_info, setup_mirror_session, # noqa F811 ptfadapter, everflow_dut, everflow_direction, setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match on a range of L4 Destination Ports.""" test_packet = self._base_tcpv6_packet(everflow_direction, ptfadapter, setup_info, dport=10700) @@ -184,11 +267,13 @@ def test_l4_dst_port_range_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_tcp_flags_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match on TCP Flags.""" test_packet = self._base_tcpv6_packet(everflow_direction, ptfadapter, setup_info, flags=0x1B) @@ -197,11 +282,13 @@ def test_tcp_flags_mirroring(self, setup_info, setup_mirror_session, ptfadapter, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_dscp_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match on DSCP.""" test_packet = self._base_tcpv6_packet(everflow_direction, ptfadapter, setup_info, dscp=37) @@ -210,11 +297,13 @@ def test_dscp_mirroring(self, setup_info, setup_mirror_session, ptfadapter, ever ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_l4_range_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match from a source port to a range of destination ports and vice-versa.""" test_packet = self._base_tcpv6_packet( everflow_direction, @@ -231,7 +320,8 @@ def test_l4_range_mirroring(self, setup_info, setup_mirror_session, ptfadapter, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) test_packet = self._base_tcpv6_packet( everflow_direction, @@ -248,11 +338,13 @@ def test_l4_range_mirroring(self, setup_info, setup_mirror_session, ptfadapter, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_tcp_response_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match a SYN -> SYN-ACK pattern.""" test_packet = self._base_tcpv6_packet( everflow_direction, @@ -268,7 +360,8 @@ def test_tcp_response_mirroring(self, setup_info, setup_mirror_session, ptfadapt ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) test_packet = self._base_tcpv6_packet( everflow_direction, @@ -284,12 +377,14 @@ def test_tcp_response_mirroring(self, setup_info, setup_mirror_session, ptfadapt ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_tcp_application_mirroring(self, setup_info, setup_mirror_session, # noqa F811 ptfadapter, everflow_dut, everflow_direction, setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match a TCP handshake between a client and server.""" test_packet = self._base_tcpv6_packet( everflow_direction, @@ -307,7 +402,8 @@ def test_tcp_application_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) test_packet = self._base_tcpv6_packet( everflow_direction, @@ -325,12 +421,14 @@ def test_tcp_application_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_udp_application_mirroring(self, setup_info, setup_mirror_session, # noqa F811 ptfadapter, everflow_dut, everflow_direction, setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match UDP traffic between a client and server application.""" test_packet = self._base_udpv6_packet( everflow_direction, @@ -348,7 +446,8 @@ def test_udp_application_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) test_packet = self._base_udpv6_packet( everflow_direction, ptfadapter, @@ -365,11 +464,13 @@ def test_udp_application_mirroring(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_any_protocol(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that the protocol number is ignored if it is not specified in the ACL rule.""" test_packet = self._base_tcpv6_packet( everflow_direction, @@ -384,7 +485,8 @@ def test_any_protocol(self, setup_info, setup_mirror_session, ptfadapter, everfl ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) test_packet = self._base_udpv6_packet( everflow_direction, @@ -399,7 +501,8 @@ def test_any_protocol(self, setup_info, setup_mirror_session, ptfadapter, everfl ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) test_packet = self._base_udpv6_packet( everflow_direction, @@ -415,12 +518,14 @@ def test_any_protocol(self, setup_info, setup_mirror_session, ptfadapter, everfl ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_any_transport_protocol(self, setup_info, setup_mirror_session, # noqa F811 ptfadapter, everflow_dut, everflow_direction, setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that src port and dst port rules match regardless of whether TCP or UDP traffic is sent.""" test_packet = self._base_tcpv6_packet( everflow_direction, @@ -437,7 +542,8 @@ def test_any_transport_protocol(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) test_packet = self._base_udpv6_packet( everflow_direction, @@ -454,11 +560,13 @@ def test_any_transport_protocol(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_invalid_tcp_rule(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that the ASIC does not reject rules with TCP flags if the protocol is not TCP.""" pass @@ -469,7 +577,8 @@ def test_invalid_tcp_rule(self, setup_info, setup_mirror_session, ptfadapter, ev def test_source_subnet(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match packets with a Source IPv6 Subnet.""" test_packet = self._base_tcpv6_packet( everflow_direction, @@ -486,11 +595,13 @@ def test_source_subnet(self, setup_info, setup_mirror_session, ptfadapter, everf ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_dest_subnet(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match packets with a Destination IPv6 Subnet.""" test_packet = self._base_tcpv6_packet( everflow_direction, @@ -507,11 +618,13 @@ def test_dest_subnet(self, setup_info, setup_mirror_session, ptfadapter, everflo ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_both_subnets(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match packets with both source and destination subnets.""" test_packet = self._base_tcpv6_packet( everflow_direction, @@ -528,11 +641,13 @@ def test_both_subnets(self, setup_info, setup_mirror_session, ptfadapter, everfl ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def test_fuzzy_subnets(self, setup_info, setup_mirror_session, ptfadapter, everflow_dut, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 - everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + everflow_direction, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that we can match packets with non-standard subnet sizes.""" test_packet = self._base_tcpv6_packet( everflow_direction, @@ -549,7 +664,8 @@ def test_fuzzy_subnets(self, setup_info, setup_mirror_session, ptfadapter, everf ptfadapter, everflow_dut, test_packet, everflow_direction, src_port=EverflowIPv6Tests.rx_port_ptf_id, - dest_ports=EverflowIPv6Tests.tx_port_ids) + dest_ports=EverflowIPv6Tests.tx_port_ids, + skip_traffic_test=skip_traffic_test) def _base_tcpv6_packet(self, direction, diff --git a/tests/everflow/test_everflow_per_interface.py b/tests/everflow/test_everflow_per_interface.py index 6daf90e206..820513d667 100644 --- a/tests/everflow/test_everflow_per_interface.py +++ b/tests/everflow/test_everflow_per_interface.py @@ -11,8 +11,9 @@ DUT_RUN_DIR, EVERFLOW_RULE_CREATE_FILE, UP_STREAM from tests.common.helpers.assertions import pytest_require -from .everflow_test_utilities import setup_info, EVERFLOW_DSCP_RULES # noqa: F401 +from .everflow_test_utilities import setup_info, EVERFLOW_DSCP_RULES # noqa: F401 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor # noqa: F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa: F401 pytestmark = [ pytest.mark.topology("any") @@ -180,7 +181,8 @@ def send_and_verify_packet(ptfadapter, packet, expected_packet, tx_port, rx_port def test_everflow_per_interface(ptfadapter, setup_info, apply_acl_rule, tbinfo, # noqa F811 - toggle_all_simulator_ports_to_rand_selected_tor, ip_ver): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, ip_ver, # noqa F811 + skip_traffic_test): # noqa F811 """Verify packet ingress from candidate ports are captured by EVERFLOW, while packets ingress from unselected ports are not captured """ @@ -190,6 +192,9 @@ def test_everflow_per_interface(ptfadapter, setup_info, apply_acl_rule, tbinfo, setup_info[UP_STREAM]['ingress_router_mac'], setup_info, ip_ver) uplink_ports = everflow_config["monitor_port_ptf_ids"] + if skip_traffic_test: + return + # Verify that packet ingressed from INPUT_PORTS (candidate ports) are mirrored for port, ptf_idx in list(everflow_config['candidate_ports'].items()): logger.info("Verifying packet ingress from {} is mirrored".format(port)) diff --git a/tests/everflow/test_everflow_testbed.py b/tests/everflow/test_everflow_testbed.py index 7aa33e14ec..1e0777dcd5 100644 --- a/tests/everflow/test_everflow_testbed.py +++ b/tests/everflow/test_everflow_testbed.py @@ -1,17 +1,22 @@ """Test cases to support the Everflow Mirroring feature in SONiC.""" import logging +import random import time import pytest - +import ipaddress +import threading import ptf.testutils as testutils +from ptf.mask import Mask +import ptf.packet as packet from . import everflow_test_utilities as everflow_utils - +import ptf.packet as scapy from tests.ptf_runner import ptf_runner from .everflow_test_utilities import TARGET_SERVER_IP, BaseEverflowTest, DOWN_STREAM, UP_STREAM, DEFAULT_SERVER_IP # Module-level fixtures from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa: F401 from tests.common.fixtures.ptfhost_utils import copy_acstests_directory # noqa: F401 -from .everflow_test_utilities import setup_info, setup_arp_responder, EVERFLOW_DSCP_RULES # noqa: F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa: F401 +from .everflow_test_utilities import setup_info, setup_arp_responder, EVERFLOW_DSCP_RULES # noqa: F401 from tests.common.fixtures.ptfhost_utils import copy_arp_responder_py # noqa: F401 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor # noqa: F401 @@ -130,7 +135,8 @@ def add_dest_routes(self, setup_info, tbinfo, dest_port_type): # noqa F811 def test_everflow_basic_forwarding(self, setup_info, setup_mirror_session, # noqa F811 dest_port_type, ptfadapter, tbinfo, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 - setup_standby_ports_on_rand_unselected_tor_unconditionally): # noqa F811 + setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 + skip_traffic_test): # noqa F811 """ Verify basic forwarding scenarios for the Everflow feature. @@ -164,7 +170,8 @@ def test_everflow_basic_forwarding(self, setup_info, setup_mirror_session, everflow_dut, rx_port_ptf_id, [tx_port_ptf_id], - dest_port_type + dest_port_type, + skip_traffic_test=skip_traffic_test ) # Add a (better) unresolved route to the mirror session destination IP @@ -181,7 +188,8 @@ def test_everflow_basic_forwarding(self, setup_info, setup_mirror_session, everflow_dut, rx_port_ptf_id, [tx_port_ptf_id], - dest_port_type + dest_port_type, + skip_traffic_test=skip_traffic_test ) # Remove the unresolved route @@ -204,7 +212,8 @@ def test_everflow_basic_forwarding(self, setup_info, setup_mirror_session, everflow_dut, rx_port_ptf_id, [tx_port_ptf_id], - dest_port_type + dest_port_type, + skip_traffic_test=skip_traffic_test ) # Remove the better route. @@ -221,7 +230,8 @@ def test_everflow_basic_forwarding(self, setup_info, setup_mirror_session, everflow_dut, rx_port_ptf_id, [tx_port_ptf_id], - dest_port_type + dest_port_type, + skip_traffic_test=skip_traffic_test ) remote_dut.shell(remote_dut.get_vtysh_cmd_for_namespace( @@ -231,7 +241,8 @@ def test_everflow_basic_forwarding(self, setup_info, setup_mirror_session, def test_everflow_neighbor_mac_change(self, setup_info, setup_mirror_session, # noqa F811 dest_port_type, ptfadapter, tbinfo, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 - setup_standby_ports_on_rand_unselected_tor_unconditionally): # noqa F811 + setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that session destination MAC address is changed after neighbor MAC address update.""" everflow_dut = setup_info[dest_port_type]['everflow_dut'] @@ -254,7 +265,8 @@ def test_everflow_neighbor_mac_change(self, setup_info, setup_mirror_session, everflow_dut, rx_port_ptf_id, [tx_port_ptf_id], - dest_port_type + dest_port_type, + skip_traffic_test=skip_traffic_test ) # Update the MAC on the neighbor interface for the route we installed @@ -274,7 +286,8 @@ def test_everflow_neighbor_mac_change(self, setup_info, setup_mirror_session, everflow_dut, rx_port_ptf_id, [tx_port_ptf_id], - dest_port_type + dest_port_type, + skip_traffic_test=skip_traffic_test ) finally: @@ -295,13 +308,15 @@ def test_everflow_neighbor_mac_change(self, setup_info, setup_mirror_session, everflow_dut, rx_port_ptf_id, [tx_port_ptf_id], - dest_port_type + dest_port_type, + skip_traffic_test=skip_traffic_test ) def test_everflow_remove_unused_ecmp_next_hop(self, setup_info, setup_mirror_session, # noqa F811 dest_port_type, ptfadapter, tbinfo, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 - setup_standby_ports_on_rand_unselected_tor_unconditionally): # noqa F811 + setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that session is still active after removal of next hop from ECMP route that was not in use.""" everflow_dut = setup_info[dest_port_type]['everflow_dut'] @@ -333,7 +348,8 @@ def test_everflow_remove_unused_ecmp_next_hop(self, setup_info, setup_mirror_ses everflow_dut, rx_port_ptf_id, tx_port_ptf_ids, - dest_port_type + dest_port_type, + skip_traffic_test=skip_traffic_test ) # Remaining Scenario not applicable for this topology @@ -358,7 +374,8 @@ def test_everflow_remove_unused_ecmp_next_hop(self, setup_info, setup_mirror_ses [tx_port_ptf_id], dest_port_type, expect_recv=False, - valid_across_namespace=False + valid_across_namespace=False, + skip_traffic_test=skip_traffic_test ) # Remove the extra hop @@ -376,7 +393,8 @@ def test_everflow_remove_unused_ecmp_next_hop(self, setup_info, setup_mirror_ses [tx_port_ptf_id], dest_port_type, expect_recv=False, - valid_across_namespace=False + valid_across_namespace=False, + skip_traffic_test=skip_traffic_test ) # Verify that mirrored traffic is still sent to one of the original next hops @@ -387,13 +405,15 @@ def test_everflow_remove_unused_ecmp_next_hop(self, setup_info, setup_mirror_ses everflow_dut, rx_port_ptf_id, tx_port_ptf_ids, - dest_port_type + dest_port_type, + skip_traffic_test=skip_traffic_test ) def test_everflow_remove_used_ecmp_next_hop(self, setup_info, setup_mirror_session, # noqa F811 dest_port_type, ptfadapter, tbinfo, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 - setup_standby_ports_on_rand_unselected_tor_unconditionally): # noqa F811 + setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that session is still active after removal of next hop from ECMP route that was in use.""" everflow_dut = setup_info[dest_port_type]['everflow_dut'] @@ -424,7 +444,8 @@ def test_everflow_remove_used_ecmp_next_hop(self, setup_info, setup_mirror_sessi everflow_dut, rx_port_ptf_id, [tx_port_ptf_id], - dest_port_type + dest_port_type, + skip_traffic_test=skip_traffic_test ) # Add two new ECMP next hops @@ -448,7 +469,8 @@ def test_everflow_remove_used_ecmp_next_hop(self, setup_info, setup_mirror_sessi rx_port_ptf_id, [tx_port_ptf_id], dest_port_type, - valid_across_namespace=False + valid_across_namespace=False, + skip_traffic_test=skip_traffic_test ) # Verify that traffic is not sent along either of the new next hops @@ -465,7 +487,8 @@ def test_everflow_remove_used_ecmp_next_hop(self, setup_info, setup_mirror_sessi tx_port_ptf_ids, dest_port_type, expect_recv=False, - valid_across_namespace=False + valid_across_namespace=False, + skip_traffic_test=skip_traffic_test ) # Remove the original next hop @@ -482,7 +505,8 @@ def test_everflow_remove_used_ecmp_next_hop(self, setup_info, setup_mirror_sessi rx_port_ptf_id, [tx_port_ptf_id], dest_port_type, - expect_recv=False + expect_recv=False, + skip_traffic_test=skip_traffic_test ) # Verify that mirrored traffis is now sent along either of the new next hops @@ -493,7 +517,8 @@ def test_everflow_remove_used_ecmp_next_hop(self, setup_info, setup_mirror_sessi everflow_dut, rx_port_ptf_id, tx_port_ptf_ids, - dest_port_type + dest_port_type, + skip_traffic_test=skip_traffic_test ) def test_everflow_dscp_with_policer( @@ -506,6 +531,7 @@ def test_everflow_dscp_with_policer( tbinfo, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 setup_standby_ports_on_rand_unselected_tor_unconditionally, # noqa F811 + skip_traffic_test # noqa F811 ): """Verify that we can rate-limit mirrored traffic from the MIRROR_DSCP table. This tests single rate three color policer mode and specifically checks CIR value @@ -591,6 +617,9 @@ def test_everflow_dscp_with_policer( config_method, rules=EVERFLOW_DSCP_RULES) + if skip_traffic_test is True: + return + # Run test with expected CIR/CBS in packets/sec and tolerance % partial_ptf_runner(setup_info, dest_port_type, @@ -606,7 +635,8 @@ def test_everflow_dscp_with_policer( cir=rate_limit, cbs=rate_limit, send_time=send_time, - tolerance=everflow_tolerance) + tolerance=everflow_tolerance, + skip_traffic_test=skip_traffic_test) finally: # Clean up ACL rules and routes BaseEverflowTest.remove_acl_rule_config(everflow_dut, table_name, config_method) @@ -618,8 +648,180 @@ def test_everflow_dscp_with_policer( everflow_utils.remove_route(everflow_dut, self.DEFAULT_DST_IP + "/32", default_traffic_peer_ip, setup_info[default_tarffic_port_type]["remote_namespace"]) + def test_everflow_frwd_with_bkg_trf(self, + setup_info, # noqa F811 + setup_mirror_session, + dest_port_type, ptfadapter, tbinfo, + skip_traffic_test # noqa F811 + ): + """ + Verify basic forwarding scenarios for the Everflow feature with background traffic. + Background Traffic PKT1 IP in IP with same ports & macs but with dummy ips + Background Traffic PKT2 Packer with same ports & macs but with dummy ips + """ + everflow_dut = setup_info[dest_port_type]['everflow_dut'] + remote_dut = setup_info[dest_port_type]['remote_dut'] + remote_dut.shell(remote_dut.get_vtysh_cmd_for_namespace( + "vtysh -c \"configure terminal\" -c \"no ip nht resolve-via-default\"", + setup_info[dest_port_type]["remote_namespace"])) + + # Add a route to the mirror session destination IP + tx_port = setup_info[dest_port_type]["dest_port"][0] + peer_ip = everflow_utils.get_neighbor_info(remote_dut, tx_port, tbinfo) + everflow_utils.add_route(remote_dut, setup_mirror_session["session_prefixes"][0], peer_ip, + setup_info[dest_port_type]["remote_namespace"]) + + time.sleep(15) + + # Verify that mirrored traffic is sent along the route we installed + rx_port_ptf_id = setup_info[dest_port_type]["src_port_ptf_id"] + tx_port_ptf_id = setup_info[dest_port_type]["dest_port_ptf_id"][0] + + # Events to control the thread + stop_thread = threading.Event() + neigh_ipv4 = {entry['name']: entry['addr'].lower() for entry in ptfadapter.mg_facts['minigraph_bgp'] if + 'ASIC' not in entry['name'] and isinstance(ipaddress.ip_address(entry['addr']), + ipaddress.IPv4Address)} + if len(neigh_ipv4) < 2: + pytest.skip("Skipping as Less than 2 Neigbhour") + + def background_traffic(run_count=None): + selected_addrs1 = random.sample(list(neigh_ipv4.values()), 2) + selected_addrs2 = random.sample(list(neigh_ipv4.values()), 2) + selected_addrs3 = random.sample(list(neigh_ipv4.values()), 2) + router_mac = setup_info[dest_port_type]["ingress_router_mac"] + inner_pkt2 = self._base_tcp_packet(ptfadapter, setup_info, router_mac, src_ip=selected_addrs1[1], + dst_ip=selected_addrs1[0]) + packets = [ + testutils.simple_ipv4ip_packet( + ip_src=selected_addrs2[0], + eth_dst=router_mac, + ip_dst=selected_addrs2[1], + eth_src=ptfadapter.dataplane.get_mac(0, 0), + inner_frame=inner_pkt2[scapy.IP] + ), + self._base_tcp_packet(ptfadapter, setup_info, router_mac, src_ip=selected_addrs3[0], + dst_ip=selected_addrs3[1]) + ] + + count = 0 + if run_count is None: + logger.debug("Background Traffic Started") + # Run the loop either for a specified count or until stop_thread is set + while (run_count is None and not stop_thread.is_set()) or (run_count is not None and count < run_count): + for bkg_trf in packets: + # Send packet + time.sleep(.1) + testutils.send(ptfadapter, rx_port_ptf_id, bkg_trf) + + # Verify packet if run_count is specified + if run_count is not None: + exp_pkt = Mask(bkg_trf) + exp_pkt.set_do_not_care_scapy(scapy.IP, "id") + exp_pkt.set_do_not_care_scapy(packet.Ether, 'dst') + exp_pkt.set_do_not_care_scapy(packet.Ether, 'src') + exp_pkt.set_do_not_care_scapy(packet.IP, "chksum") + exp_pkt.set_do_not_care_scapy(packet.IP, 'ttl') + testutils.verify_packet_any_port( + ptfadapter, + exp_pkt, + ports=ptfadapter.ptf_port_set + ) + + count += 1 + + background_traffic(run_count=1) + background_thread = threading.Thread(target=background_traffic) + background_thread.daemon = True + background_thread.start() + + try: + self._run_everflow_test_scenarios( + ptfadapter, + setup_info, + setup_mirror_session, + everflow_dut, + rx_port_ptf_id, + [tx_port_ptf_id], + dest_port_type, + skip_traffic_test=skip_traffic_test + ) + + # Add a (better) unresolved route to the mirror session destination IP + peer_ip = everflow_utils.get_neighbor_info(remote_dut, tx_port, tbinfo, resolved=False) + everflow_utils.add_route(remote_dut, setup_mirror_session["session_prefixes"][1], peer_ip, + setup_info[dest_port_type]["remote_namespace"]) + time.sleep(15) + background_traffic(run_count=1) + + # Verify that mirrored traffic is still sent along the original route + self._run_everflow_test_scenarios( + ptfadapter, + setup_info, + setup_mirror_session, + everflow_dut, + rx_port_ptf_id, + [tx_port_ptf_id], + dest_port_type, + skip_traffic_test=skip_traffic_test + ) + + # Remove the unresolved route + everflow_utils.remove_route(remote_dut, setup_mirror_session["session_prefixes"][1], + peer_ip, setup_info[dest_port_type]["remote_namespace"]) + + # Add a better route to the mirror session destination IP + tx_port = setup_info[dest_port_type]["dest_port"][1] + peer_ip = everflow_utils.get_neighbor_info(remote_dut, tx_port, tbinfo) + everflow_utils.add_route(remote_dut, setup_mirror_session['session_prefixes'][1], peer_ip, + setup_info[dest_port_type]["remote_namespace"]) + time.sleep(15) + background_traffic(run_count=1) + # Verify that mirrored traffic uses the new route + tx_port_ptf_id = setup_info[dest_port_type]["dest_port_ptf_id"][1] + self._run_everflow_test_scenarios( + ptfadapter, + setup_info, + setup_mirror_session, + everflow_dut, + rx_port_ptf_id, + [tx_port_ptf_id], + dest_port_type, + skip_traffic_test=skip_traffic_test + ) + + # Remove the better route. + everflow_utils.remove_route(remote_dut, setup_mirror_session["session_prefixes"][1], peer_ip, + setup_info[dest_port_type]["remote_namespace"]) + time.sleep(15) + background_traffic(run_count=1) + # Verify that mirrored traffic switches back to the original route + tx_port_ptf_id = setup_info[dest_port_type]["dest_port_ptf_id"][0] + self._run_everflow_test_scenarios( + ptfadapter, + setup_info, + setup_mirror_session, + everflow_dut, + rx_port_ptf_id, + [tx_port_ptf_id], + dest_port_type, + skip_traffic_test=skip_traffic_test + ) + + remote_dut.shell(remote_dut.get_vtysh_cmd_for_namespace( + "vtysh -c \"configure terminal\" -c \"ip nht resolve-via-default\"", + setup_info[dest_port_type]["remote_namespace"])) + finally: + # Signal the background thread to stop + logger.debug("Thread_stop") + stop_thread.set() + # Wait for the background thread to finish + background_thread.join() + background_traffic(run_count=1) + def _run_everflow_test_scenarios(self, ptfadapter, setup, mirror_session, duthost, rx_port, - tx_ports, direction, expect_recv=True, valid_across_namespace=True): + tx_ports, direction, expect_recv=True, valid_across_namespace=True, + skip_traffic_test=False): # noqa F811 # FIXME: In the ptf_runner version of these tests, LAGs were passed down to the tests # as comma-separated strings of LAG member port IDs (e.g. portchannel0001 -> "2,3"). # Because the DSCP test is still using ptf_runner we will preserve this for now, @@ -658,6 +860,7 @@ def _run_everflow_test_scenarios(self, ptfadapter, setup, mirror_session, duthos dest_ports=tx_port_ids, expect_recv=expect_recv, valid_across_namespace=valid_across_namespace, + skip_traffic_test=skip_traffic_test, ) def _base_tcp_packet( diff --git a/tests/fdb/conftest.py b/tests/fdb/conftest.py index 54eb2a4cef..af8d5cdb3f 100644 --- a/tests/fdb/conftest.py +++ b/tests/fdb/conftest.py @@ -36,7 +36,7 @@ def set_polling_interval(duthost): @pytest.fixture(scope='module') -def get_function_conpleteness_level(pytestconfig, duthost): +def get_function_completeness_level(pytestconfig, duthost): asic_name = duthost.get_asic_name() if asic_name in ['td2']: return None diff --git a/tests/fdb/test_fdb.py b/tests/fdb/test_fdb.py index 0254dce95a..7fd165aeab 100644 --- a/tests/fdb/test_fdb.py +++ b/tests/fdb/test_fdb.py @@ -57,7 +57,8 @@ def get_dummay_mac_count(tbinfo, duthosts, rand_one_dut_hostname): # t0-116 will take 90m with DUMMY_MAC_COUNT, so use DUMMY_MAC_COUNT_SLIM for t0-116 to reduce running time # Use DUMMY_MAC_COUNT_SLIM on dualtor-64 to reduce running time - REQUIRED_TOPO = ["t0-116", "dualtor-64", "dualtor-120"] + REQUIRED_TOPO = ["t0-116", "dualtor-64", "dualtor-120", + "t0-standalone-64", "t0-standalone-128", "t0-standalone-256", "t0-standalone-512"] if tbinfo["topo"]["name"] in REQUIRED_TOPO: # To reduce the case running time logger.info("Use dummy mac count {} on topo {}\n".format(DUMMY_MAC_COUNT_SLIM, tbinfo["topo"]["name"])) diff --git a/tests/fdb/test_fdb_mac_learning.py b/tests/fdb/test_fdb_mac_learning.py new file mode 100644 index 0000000000..a8bf4243e8 --- /dev/null +++ b/tests/fdb/test_fdb_mac_learning.py @@ -0,0 +1,273 @@ +import logging +import pytest +import time +from tests.common import config_reload +from tests.common.utilities import wait_until + +from tests.common.helpers.assertions import pytest_assert +from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.ptf_runner import ptf_runner +from .utils import fdb_table_has_dummy_mac_for_interface + +pytestmark = [ + pytest.mark.topology('t0') +] +logger = logging.getLogger(__name__) + + +class TestFdbMacLearning: + """ + TestFdbMacLearning verifies that stale MAC entries are not present in MAC table after doing sonic-clear fdb all + -shut down all ports + -config save + -config reload + -bring up 1 port. populate fdb + -bring up 3 more ports. populate fdb. + -shut down 3 ports added in last step + -verify that MAC entries connected to shutdown ports are gone from MAC table + -sonic-clear fdb all + -populate fdb for the UP port + -verify that MAC entries connected to shutdown ports not present in MAC table + """ + DUMMY_MAC_PREFIX = "00:11:22:33:55" + TEST_MAC = "00:11:22:33:55:66" + FDB_POPULATE_SLEEP_TIMEOUT = 5 + PTF_HOST_IP = "20.0.0.2" + PTF_HOST_NETMASK = "24" + DUT_INTF_IP = "20.0.0.1" + DUT_INTF_NETMASK = "24" + + def configureInterfaceIp(self, duthost, dut_intf, action=None): + """ + Configure interface IP address on the DUT + + Args: + duthost (AnsibleHost): Device Under Test (DUT) + action (str): action to perform, add/remove interface IP + + Returns: + None + """ + + logger.info("{0} an ip entry {1} for {2}".format(action, self.DUT_INTF_IP, dut_intf)) + interfaceIp = "{}/{}".format(self.DUT_INTF_IP, self.DUT_INTF_NETMASK) + duthost.shell(argv=[ + "config", + "interface", + "ip", + action, + dut_intf, + interfaceIp + ]) + + def configureNeighborIp(self, ptfhost, ptf_intf, action=None): + """ + Configure interface and set IP address on the PTF host + + Args: + ptfhost (PTF host): PTF instance used + action (str): action to perform, add/remove interface IP + + Returns: + None + """ + ptfhost.shell("ip addr {} {}/{} dev {}".format(action, self.PTF_HOST_IP, self.PTF_HOST_NETMASK, ptf_intf)) + logger.info("{0} an ip entry {1}/{2} for {3} on ptf" + .format(action, self.PTF_HOST_IP, self.PTF_HOST_NETMASK, ptf_intf)) + + def __runPtfTest(self, ptfhost, testCase='', testParams={}): + """ + Runs FDB MAC Learning test case on PTF host + + Args: + ptfhost (AnsibleHost): Packet Test Framework (PTF) + testCase (str): FDB tests test case name + testParams (dict): Map of test params required by testCase + + Returns: + None + + Raises: + RunAnsibleModuleFail if ptf test fails + """ + logger.info("Running PTF test case '{0}' on '{1}'".format(testCase, ptfhost.hostname)) + ptf_runner( + ptfhost, + "ptftests", + testCase, + platform_dir="ptftests", + params=testParams, + log_file="/tmp/{0}".format(testCase), + is_python3=True + ) + + @pytest.fixture(scope="class", autouse=True) + def prepare_test(self, duthosts, rand_one_dut_hostname, ptfhost): + """ + Select DUT ports which will be used for the testcase + Get a mapping of selected DUT ports and ptf ports + shut down all DUT ports, congit save and config reload the DUT before starting the testcases + + Args: + duthosts: Devices under test + rand_one_dut_hostname: selected device index + ptfhost: PTF instance used + + Yields: + DUT to PTF port mapping + PTF ports available in topology + conf_facts + """ + duthost = duthosts[rand_one_dut_hostname] + # Get 4 UP ports which will be used for the test + up_interfaces = [] + ifs_status = duthost.get_interfaces_status() + logging.info("ifs_status {} ".format(ifs_status)) + for _, interface_info in ifs_status.items(): + if (r'N\/A' != interface_info['alias']) and (r'N\/A' != interface_info['type']) \ + and ('up' == interface_info['oper']) and (interface_info['vlan'] == 'trunk'): + up_interfaces.append(interface_info['interface']) + + if len(up_interfaces) < 4: + pytest.skip('Test FDB MAC Learning: cannot get enough target port to test: {}'.format(up_interfaces)) + up_interface_numbers = [] + for intf in up_interfaces: + up_interface_numbers.append(int(intf[8:])) + up_interface_numbers.sort() + target_ports = [] + for i in range(0, 4): + target_ports.append("Ethernet"+str(up_interface_numbers[i])) + logging.info("DUT interfaces selected for running the test are {} ".format(target_ports)) + + # Get a mapping between selected DUT ports and PTF ports + conf_facts = duthost.config_facts(host=duthost.hostname, source="persistent")['ansible_facts'] + port_index_to_name = {v: k for k, v in list(conf_facts['port_index_map'].items())} + ptf_ports_available_in_topo = ptfhost.host.options['variable_manager'].extra_vars.get("ifaces_map") + target_ports_to_ptf_mapping = [[] for _ in range(len(target_ports))] + for i in range(len(target_ports)): + target_ports_to_ptf_mapping[i].append(target_ports[i]) + for idx, name in list(ptf_ports_available_in_topo.items()): + if (idx in port_index_to_name and port_index_to_name[idx] in target_ports): + target_ports_to_ptf_mapping[target_ports.index(port_index_to_name[idx])].append(idx) + logging.info("DUT to PTF port mapping is {}".format(target_ports_to_ptf_mapping)) + + # shut down all ports on DUT, config save and config reload + dut_ports = conf_facts['port_index_map'].keys() + logging.info("shutdown all interfaces on DUT") + for port in dut_ports: + duthost.shell("sudo config interface shutdown {}".format(port)) + duthost.command('sudo config save -y') + config_reload(duthost, config_source='config_db', safe_reload=True) + yield target_ports_to_ptf_mapping, ptf_ports_available_in_topo, conf_facts + + def dynamic_fdb_oper(self, duthost, tbinfo, ptfhost, dut_ptf_ports): + """function to populate fdb for given dut/ptf ports""" + testParams = { + "testbed_type": tbinfo["topo"]["name"], + "router_mac": duthost.facts["router_mac"], + "dut_ptf_ports": dut_ptf_ports, + "dummy_mac_prefix": self.DUMMY_MAC_PREFIX, + } + self.__runPtfTest(ptfhost, "fdb_mac_learning_test.FdbMacLearningTest", testParams) + + res = duthost.command('show mac') + logging.info("show mac {}".format(res['stdout_lines'])) + + def testFdbMacLearning(self, ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, tbinfo, request, prepare_test): + """ + TestFdbMacLearning verifies stale MAC entries are not present in MAC table after doing sonic-clear fdb all + -shut down all ports + -config save + -config reload + -bring up 1 port. populate fdb + -bring up 3 more ports. populate fdb. + -shut down 3 ports added in last step + -verify that MAC entries connected to shutdown ports are gone from MAC table + -sonic-clear fdb all + -populate fdb for the UP port + -verify that MAC entries connected to shutdown ports not present in MAC table + """ + target_ports_to_ptf_mapping, ptf_ports_available_in_topo, _ = prepare_test + + # Find MAC addresses for different PTF interfaces to be used in Testcase + ptf_interfaces_mac_addresses = [] + for i in range(len(target_ports_to_ptf_mapping)): + ptf_port = ptf_ports_available_in_topo[target_ports_to_ptf_mapping[i][1]] + res = ptfhost.shell('cat /sys/class/net/{}/address'.format(ptf_port)) + ptf_interfaces_mac_addresses.append(res['stdout'].upper()) + + # unshut 1 port and populate fdb for that port. make sure fdb entry is populated in mac table + duthost = duthosts[rand_one_dut_hostname] + duthost.shell("sudo config interface startup {}".format(target_ports_to_ptf_mapping[0][0])) + time.sleep(10) + self.dynamic_fdb_oper(duthost, tbinfo, ptfhost, [target_ports_to_ptf_mapping[0]]) + pytest_assert(wait_until(300, 2, 1, fdb_table_has_dummy_mac_for_interface, duthost, + target_ports_to_ptf_mapping[0][0], self.DUMMY_MAC_PREFIX), "After starting {}" + " and populating fdb, corresponding mac address entry not seen in mac table" + .format(target_ports_to_ptf_mapping[0][0])) + + # unshut 3 more ports and populate fdb for those ports + duthost.shell("sudo config interface startup {}-{}".format(target_ports_to_ptf_mapping[1][0], + target_ports_to_ptf_mapping[3][0][8:])) + time.sleep(10) + self.dynamic_fdb_oper(duthost, tbinfo, ptfhost, target_ports_to_ptf_mapping[1:]) + for i in range(1, len(target_ports_to_ptf_mapping)): + pytest_assert(wait_until(300, 2, 1, fdb_table_has_dummy_mac_for_interface, duthost, + target_ports_to_ptf_mapping[i][0], self.DUMMY_MAC_PREFIX), + "After starting {} and populating fdb, corresponding mac address entry" + "not seen in mac table".format(target_ports_to_ptf_mapping[i][0])) + + # shutdown last 3 ports and make sure corresponding entries are gone from MAC address table + for i in range(1, len(target_ports_to_ptf_mapping)): + duthost.shell("sudo config interface shutdown {}".format(target_ports_to_ptf_mapping[i][0])) + time.sleep(10) + for i in range(1, len(target_ports_to_ptf_mapping)): + pytest_assert(not (fdb_table_has_dummy_mac_for_interface(duthost, target_ports_to_ptf_mapping[i][0])), + "mac entry present when interface {} is down" + .format(target_ports_to_ptf_mapping[i][0])) + + # clear all fdb entries on DUT + duthost.shell("sonic-clear fdb all") + + # after clearing fdb, make sure that no stale entries are present in MAC address table + self.dynamic_fdb_oper(duthost, tbinfo, ptfhost, [target_ports_to_ptf_mapping[0]]) + if wait_until(100, 1, 1, fdb_table_has_dummy_mac_for_interface, duthost, target_ports_to_ptf_mapping[0][0], + self.DUMMY_MAC_PREFIX): + for i in range(1, len(target_ports_to_ptf_mapping)): + pytest_assert(not (fdb_table_has_dummy_mac_for_interface(duthost, target_ports_to_ptf_mapping[i][0])), + "mac entry present when interface {} is down even after sonic-clear fdb all" + .format(target_ports_to_ptf_mapping[i][0])) + + def testARPCompleted(self, ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, tbinfo, request, prepare_test): + """ + Select a DUT interface and corresponding PTF interface + If DUT interface is in VLAN, remove it from the vlan + Configure ip addresses on DUT interface and PTF interface and do a ping test + Check if the ARP entry on DUT has all details + + """ + target_ports_to_ptf_mapping, ptf_ports_available_in_topo, conf_facts = prepare_test + duthost = duthosts[rand_one_dut_hostname] + dut_interface, ptf_port_index = target_ports_to_ptf_mapping[0] + duthost.shell("sudo config interface startup {}".format(dut_interface)) + for vlan in conf_facts['VLAN_MEMBER']: + for member_interface in conf_facts['VLAN_MEMBER'][vlan]: + if (member_interface == dut_interface): + duthost.shell("sudo config vlan member del {} {}".format(vlan[4:], member_interface)) + try: + self.configureInterfaceIp(duthost, dut_interface, action="add") + self.configureNeighborIp(ptfhost, ptf_ports_available_in_topo[ptf_port_index], action="add") + ptfhost.shell("ping {} -c 3 -I {}".format(self.DUT_INTF_IP, self.PTF_HOST_IP), module_ignore_errors=True) + + finally: + show_arp = duthost.command('show arp') + arp_found = False + for arp_entry in show_arp['stdout_lines']: + items = arp_entry.split() + if (items[0] == self.PTF_HOST_IP): + arp_found = True + pytest_assert(items[2] == dut_interface, "ARP entry for ip address {}" + " is incomplete. Interface is missing".format(self.PTF_HOST_IP)) + pytest_assert(arp_found, "ARP entry not found for ip address {}".format(self.PTF_HOST_IP)) + self.configureInterfaceIp(duthost, dut_interface, action="remove") + self.configureNeighborIp(ptfhost, ptf_ports_available_in_topo[ptf_port_index], action="del") diff --git a/tests/fdb/test_fdb_mac_move.py b/tests/fdb/test_fdb_mac_move.py index dddb90fded..ce097b7799 100644 --- a/tests/fdb/test_fdb_mac_move.py +++ b/tests/fdb/test_fdb_mac_move.py @@ -27,50 +27,6 @@ ] -def modify_rsyslog_severity_level(dut): - logger.info('Modify rsyslog severity level to error on dut: {}'.format(dut.hostname)) - dut.shell("sudo mv /etc/rsyslog.d /etc/rsyslog.d.bak") - dut.shell("sudo mkdir /etc/rsyslog.d") - dut.shell("sudo echo '*.err /var/log/syslog' > /etc/rsyslog.d/50_debug.conf") - dut.shell("sudo systemctl restart rsyslog") - time.sleep(5) - - -def revert_rsyslog_severity_level(dut): - logger.info('Revert rsyslog severity level to error on dut: {}'.format(dut.hostname)) - dut.shell("sudo rm -rf /etc/rsyslog.d") - dut.shell("sudo mv /etc/rsyslog.d.bak /etc/rsyslog.d") - dut.shell("sudo systemctl restart rsyslog") - time.sleep(5) - - -@pytest.fixture(scope="function") -def fixture_rsyslog_conf_setup_teardown(duthosts, rand_one_dut_hostname): - duthost = duthosts[rand_one_dut_hostname] - - # workaround for small disk devices which also has limitation on /var/log size - cmd = "df -m" - cmd_response_lines = duthost.shell(cmd, module_ignore_errors=True).get('stdout_lines', []) - logger.debug("cmd {} rsp {}".format(cmd, cmd_response_lines)) - - available_size = 0 - for line in cmd_response_lines: - if "/var/log" in line: - available_size = int(line.split()[3]) - break - - if available_size < 400: - modify_rsyslog_severity_level(duthost) - rsyslog_severity_level_modified = True - else: - rsyslog_severity_level_modified = False - - yield - - if rsyslog_severity_level_modified: - revert_rsyslog_severity_level(duthost) - - def get_fdb_dict(ptfadapter, vlan_table, dummay_mac_count): """ :param ptfadapter: PTF adapter object @@ -100,14 +56,14 @@ def get_fdb_dict(ptfadapter, vlan_table, dummay_mac_count): return fdb -def test_fdb_mac_move(ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, get_function_conpleteness_level, - fixture_rsyslog_conf_setup_teardown): +def test_fdb_mac_move(ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, get_function_completeness_level, + rotate_syslog): # Perform FDB clean up before each test fdb_cleanup(duthosts, rand_one_dut_hostname) - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: - normalized_level = "basic" + normalized_level = "debug" loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level] duthost = duthosts[rand_one_dut_hostname] diff --git a/tests/fdb/utils.py b/tests/fdb/utils.py index 9c0763607e..2b93b5bf46 100644 --- a/tests/fdb/utils.py +++ b/tests/fdb/utils.py @@ -48,6 +48,15 @@ def get_fdb_dynamic_mac_count(duthost): return total_mac_count +def fdb_table_has_dummy_mac_for_interface(duthost, interface, dummy_mac_prefix=""): + res = duthost.command('show mac') + logger.info('"show mac" output on DUT:\n{}'.format(pprint.pformat(res['stdout_lines']))) + for output_mac in res['stdout_lines']: + if (interface in output_mac and (dummy_mac_prefix in output_mac or dummy_mac_prefix == "")): + return True + return False + + def fdb_table_has_no_dynamic_macs(duthost): return (get_fdb_dynamic_mac_count(duthost) == 0) diff --git a/tests/fib/test_fib.py b/tests/fib/test_fib.py index baebbeb1cc..c1e8f47bfb 100644 --- a/tests/fib/test_fib.py +++ b/tests/fib/test_fib.py @@ -10,6 +10,7 @@ from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 from tests.common.fixtures.ptfhost_utils import set_ptf_port_mapping_mode # noqa F401 from tests.common.fixtures.ptfhost_utils import ptf_test_port_map_active_active, ptf_test_port_map +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.ptf_runner import ptf_runner from tests.common.dualtor.mux_simulator_control import mux_server_url # noqa F401 @@ -83,7 +84,8 @@ def test_basic_fib(duthosts, ptfhost, ipv4, ipv6, mtu, mux_status_from_nic_simulator, ignore_ttl, single_fib_for_duts, # noqa F401 duts_running_config_facts, duts_minigraph_facts, - validate_active_active_dualtor_setup): # noqa F401 + validate_active_active_dualtor_setup, # noqa F401 + skip_traffic_test): # noqa F811 if 'dualtor' in updated_tbinfo['topo']['name']: wait(30, 'Wait some time for mux active/standby state to be stable after toggled mux state') @@ -103,6 +105,8 @@ def test_basic_fib(duthosts, ptfhost, ipv4, ipv6, mtu, log_file = "/tmp/fib_test.FibTest.ipv4.{}.ipv6.{}.{}.log".format( ipv4, ipv6, timestamp) logging.info("PTF log file: %s" % log_file) + if skip_traffic_test is True: + return ptf_runner( ptfhost, "ptftests", @@ -315,7 +319,7 @@ def test_hash(add_default_route_to_dut, duthosts, fib_info_files_per_function, s hash_keys, ptfhost, ipver, toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 updated_tbinfo, mux_server_url, mux_status_from_nic_simulator, ignore_ttl, # noqa F811 single_fib_for_duts, duts_running_config_facts, duts_minigraph_facts, # noqa F811 - setup_active_active_ports, active_active_ports): # noqa F811 + setup_active_active_ports, active_active_ports, skip_traffic_test): # noqa F811 if 'dualtor' in updated_tbinfo['topo']['name']: wait(30, 'Wait some time for mux active/standby state to be stable after toggled mux state') @@ -331,6 +335,8 @@ def test_hash(add_default_route_to_dut, duthosts, fib_info_files_per_function, s else: src_ip_range = SRC_IPV6_RANGE dst_ip_range = DST_IPV6_RANGE + if skip_traffic_test is True: + return ptf_runner( ptfhost, "ptftests", @@ -365,7 +371,7 @@ def test_hash(add_default_route_to_dut, duthosts, fib_info_files_per_function, s def test_ipinip_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files_per_function, # noqa F811 hash_keys, ptfhost, ipver, tbinfo, mux_server_url, # noqa F811 ignore_ttl, single_fib_for_duts, duts_running_config_facts, # noqa F811 - duts_minigraph_facts): + duts_minigraph_facts, skip_traffic_test): # noqa F811 # Skip test on none T1 testbed pytest_require('t1' == tbinfo['topo']['type'], "The test case runs on T1 topology") @@ -379,6 +385,8 @@ def test_ipinip_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files else: src_ip_range = SRC_IPV6_RANGE dst_ip_range = DST_IPV6_RANGE + if skip_traffic_test is True: + return ptf_runner(ptfhost, "ptftests", "hash_test.IPinIPHashTest", @@ -396,7 +404,8 @@ def test_ipinip_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files }, log_file=log_file, qlen=PTF_QLEN, - socket_recv_size=16384) + socket_recv_size=16384, + is_python3=True) # The test is to verify the hashing logic is not using unexpected field as keys # Only inner frame length is tested at this moment @@ -404,7 +413,8 @@ def test_ipinip_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files def test_ipinip_hash_negative(add_default_route_to_dut, duthosts, fib_info_files_per_function, # noqa F811 ptfhost, ipver, tbinfo, mux_server_url, ignore_ttl, single_fib_for_duts, # noqa F811 - duts_running_config_facts, duts_minigraph_facts, mux_status_from_nic_simulator): + duts_running_config_facts, duts_minigraph_facts, mux_status_from_nic_simulator, + skip_traffic_test): # noqa F811 hash_keys = ['inner_length'] timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') log_file = "/tmp/hash_test.IPinIPHashTest.{}.{}.log".format( @@ -416,6 +426,8 @@ def test_ipinip_hash_negative(add_default_route_to_dut, duthosts, fib_info_files else: src_ip_range = SRC_IPV6_RANGE dst_ip_range = DST_IPV6_RANGE + if skip_traffic_test is True: + return ptf_runner(ptfhost, "ptftests", "hash_test.IPinIPHashTest", @@ -436,4 +448,5 @@ def test_ipinip_hash_negative(add_default_route_to_dut, duthosts, fib_info_files }, log_file=log_file, qlen=PTF_QLEN, - socket_recv_size=16384) + socket_recv_size=16384, + is_python3=True) diff --git a/tests/generic_config_updater/conftest.py b/tests/generic_config_updater/conftest.py index 2a697341b1..6e14fd1374 100644 --- a/tests/generic_config_updater/conftest.py +++ b/tests/generic_config_updater/conftest.py @@ -3,8 +3,8 @@ from tests.common.utilities import skip_release from tests.common.config_reload import config_reload -from tests.generic_config_updater.gu_utils import apply_patch -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import apply_patch +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile CONFIG_DB = "/etc/sonic/config_db.json" CONFIG_DB_BACKUP = "/etc/sonic/config_db.json.before_gcu_test" diff --git a/tests/generic_config_updater/gu_utils.py b/tests/generic_config_updater/gu_utils.py index ed7b3525c5..6032b26145 100644 --- a/tests/generic_config_updater/gu_utils.py +++ b/tests/generic_config_updater/gu_utils.py @@ -1,377 +1,14 @@ -import json -import logging -import pytest import os -from jsonpointer import JsonPointer -from tests.common.helpers.assertions import pytest_assert -from tests.common.utilities import wait_until -from tests.common.config_reload import config_reload - -logger = logging.getLogger(__name__) +import logging +import json +from tests.common.gu_utils import apply_patch, generate_tmpfile, delete_tmpfile -CONTAINER_SERVICES_LIST = ["swss", "syncd", "radv", "lldp", "dhcp_relay", "teamd", "bgp", "pmon", "telemetry", "acms"] -DEFAULT_CHECKPOINT_NAME = "test" -GCU_FIELD_OPERATION_CONF_FILE = "gcu_field_operation_validators.conf.json" -GET_HWSKU_CMD = "sonic-cfggen -d -v DEVICE_METADATA.localhost.hwsku" BASE_DIR = os.path.dirname(os.path.realpath(__file__)) -FILES_DIR = os.path.join(BASE_DIR, "files") -TEMPLATES_DIR = os.path.join(BASE_DIR, "templates") +TEMPLATES_DIR = os.path.join(BASE_DIR, "../generic_config_updater/templates") TMP_DIR = '/tmp' - -def generate_tmpfile(duthost): - """Generate temp file - """ - return duthost.shell('mktemp')['stdout'] - - -def delete_tmpfile(duthost, tmpfile): - """Delete temp file - """ - duthost.file(path=tmpfile, state='absent') - - -def apply_patch(duthost, json_data, dest_file): - """Run apply-patch on target duthost - - Args: - duthost: Device Under Test (DUT) - json_data: Source json patch to apply - dest_file: Destination file on duthost - """ - duthost.copy(content=json.dumps(json_data, indent=4), dest=dest_file) - - cmds = 'config apply-patch {}'.format(dest_file) - - logger.info("Commands: {}".format(cmds)) - output = duthost.shell(cmds, module_ignore_errors=True) - - return output - - -def expect_op_success(duthost, output): - """Expected success from apply-patch output - """ - pytest_assert(not output['rc'], "Command is not running successfully") - pytest_assert( - "Patch applied successfully" in output['stdout'], - "Please check if json file is validate" - ) - - -def expect_op_success_and_reset_check(duthost, output, service_name, timeout, interval, delay): - """Add contianer reset check after op success - - Args: - duthost: Device Under Test (DUT) - output: Command couput - service_name: Service to reset - timeout: Maximum time to wait - interval: Poll interval - delay: Delay time - """ - expect_op_success(duthost, output) - if start_limit_hit(duthost, service_name): - reset_start_limit_hit(duthost, service_name, timeout, interval, delay) - - -def expect_res_success(duthost, output, expected_content_list, unexpected_content_list): - """Check output success with expected and unexpected content - - Args: - duthost: Device Under Test (DUT) - output: Command output - expected_content_list: Expected content from output - unexpected_content_list: Unexpected content from output - """ - for expected_content in expected_content_list: - pytest_assert( - expected_content in output['stdout'], - "{} is expected content".format(expected_content) - ) - - for unexpected_content in unexpected_content_list: - pytest_assert( - unexpected_content not in output['stdout'], - "{} is unexpected content".format(unexpected_content) - ) - - -def expect_op_failure(output): - """Expected failure from apply-patch output - """ - logger.info("return code {}".format(output['rc'])) - pytest_assert( - output['rc'], - "The command should fail with non zero return code" - ) - - -def start_limit_hit(duthost, service_name): - """If start-limit-hit is hit, the service will not start anyway. - - Args: - service_name: Service to reset - """ - service_status = duthost.shell("systemctl status {}.service | grep 'Active'".format(service_name)) - pytest_assert( - not service_status['rc'], - "{} service status cannot be found".format(service_name) - ) - - for line in service_status["stdout_lines"]: - if "start-limit-hit" in line: - return True - - return False - - -def reset_start_limit_hit(duthost, service_name, timeout, interval, delay): - """Reset service if hit start-limit-hit - - Args: - duthost: Device Under Test (DUT) - service_name: Service to reset - timeout: Maximum time to wait - interval: Poll interval - delay: Delay time - """ - logger.info("Reset service '{}' due to start-limit-hit".format(service_name)) - - service_reset_failed = duthost.shell("systemctl reset-failed {}.service".format(service_name)) - pytest_assert( - not service_reset_failed['rc'], - "{} systemctl reset-failed service fails" - ) - - service_start = duthost.shell("systemctl start {}.service".format(service_name)) - pytest_assert( - not service_start['rc'], - "{} systemctl start service fails" - ) - - if service_name not in CONTAINER_SERVICES_LIST: - return - - reset_service = wait_until(timeout, - interval, - delay, - duthost.is_service_fully_started, - service_name) - pytest_assert( - reset_service, - "Failed to reset service '{}' due to start-limit-hit".format(service_name) - ) - - -def list_checkpoints(duthost): - """List checkpoint on target duthost - - Args: - duthost: Device Under Test (DUT) - cp: checkpoint filename - """ - cmds = 'config list-checkpoints' - - logger.info("Commands: {}".format(cmds)) - output = duthost.shell(cmds, module_ignore_errors=True) - - pytest_assert( - not output['rc'], - "Failed to list all checkpoint file" - ) - - return output - - -def verify_checkpoints_exist(duthost, cp): - """Check if checkpoint file exist in duthost - """ - output = list_checkpoints(duthost) - return '"{}"'.format(cp) in output['stdout'] - - -def create_checkpoint(duthost, cp=DEFAULT_CHECKPOINT_NAME): - """Run checkpoint on target duthost - - Args: - duthost: Device Under Test (DUT) - cp: checkpoint filename - """ - cmds = 'config checkpoint {}'.format(cp) - - logger.info("Commands: {}".format(cmds)) - output = duthost.shell(cmds, module_ignore_errors=True) - - pytest_assert( - not output['rc'] - and "Checkpoint created successfully" in output['stdout'] - and verify_checkpoints_exist(duthost, cp), - "Failed to config a checkpoint file: {}".format(cp) - ) - - -def delete_checkpoint(duthost, cp=DEFAULT_CHECKPOINT_NAME): - """Run checkpoint on target duthost - - Args: - duthost: Device Under Test (DUT) - cp: checkpoint filename - """ - pytest_assert( - verify_checkpoints_exist(duthost, cp), - "Failed to find the checkpoint file: {}".format(cp) - ) - - cmds = 'config delete-checkpoint {}'.format(cp) - - logger.info("Commands: {}".format(cmds)) - output = duthost.shell(cmds, module_ignore_errors=True) - - pytest_assert( - not output['rc'] and "Checkpoint deleted successfully" in output['stdout'], - "Failed to delete a checkpoint file: {}".format(cp) - ) - - -def rollback(duthost, cp=DEFAULT_CHECKPOINT_NAME): - """Run rollback on target duthost - - Args: - duthost: Device Under Test (DUT) - cp: rollback filename - """ - cmds = 'config rollback {}'.format(cp) - - logger.info("Commands: {}".format(cmds)) - output = duthost.shell(cmds, module_ignore_errors=True) - - return output - - -def rollback_or_reload(duthost, cp=DEFAULT_CHECKPOINT_NAME): - """Run rollback on target duthost. config_reload if rollback failed. - - Args: - duthost: Device Under Test (DUT) - """ - output = rollback(duthost, cp) - - if output['rc'] or "Config rolled back successfully" not in output['stdout']: - config_reload(duthost) - pytest.fail("config rollback failed. Restored by config_reload") - - -def create_path(tokens): - return JsonPointer.from_parts(tokens).path - - -def check_show_ip_intf(duthost, intf_name, expected_content_list, unexpected_content_list, is_ipv4=True): - """Check lo interface status by show command - - Sample output: - admin@vlab-01:~$ show ip interfaces | grep -w Vlan1000 - Vlan1000 192.168.0.1/21 up/up N/A N/A - admin@vlab-01:~$ show ipv6 interfaces | grep -w Vlan1000 - Vlan1000 fc02:1000::1/64 up/up N/A N/A - fe80::5054:ff:feda:c6af%Vlan1000/64 N/A N/A - """ - address_family = "ip" if is_ipv4 else "ipv6" - output = duthost.shell("show {} interfaces | grep -w {} || true".format(address_family, intf_name)) - - expect_res_success(duthost, output, expected_content_list, unexpected_content_list) - - -def check_vrf_route_for_intf(duthost, vrf_name, intf_name, is_ipv4=True): - """Check ip route for specific vrf - - Sample output: - admin@vlab-01:~$ show ip route vrf Vrf_01 | grep -w Loopback0 - C>* 10.1.0.32/32 is directly connected, Loopback0, 00:00:13 - """ - address_family = "ip" if is_ipv4 else "ipv6" - output = duthost.shell("show {} route vrf {} | grep -w {}".format(address_family, vrf_name, intf_name)) - - pytest_assert(not output['rc'], "Route not found for {} in vrf {}".format(intf_name, vrf_name)) - - -def get_gcu_field_operations_conf(duthost): - get_gcu_dir_path_cmd = 'python3 -c \"import generic_config_updater ; print(generic_config_updater.__path__)\"' - gcu_dir_path = duthost.shell("{}".format(get_gcu_dir_path_cmd))['stdout'].replace("[", "").replace("]", "") - gcu_conf = duthost.shell('cat {}/{}'.format(gcu_dir_path, GCU_FIELD_OPERATION_CONF_FILE))['stdout'] - gcu_conf_json = json.loads(gcu_conf) - return gcu_conf_json - - -def get_asic_name(duthost): - asic_type = duthost.facts["asic_type"] - asic = "unknown" - gcu_conf = get_gcu_field_operations_conf(duthost) - asic_mapping = gcu_conf["helper_data"]["rdma_config_update_validator"] - - def _get_asic_name(asic_type): - cur_hwsku = duthost.shell(GET_HWSKU_CMD)['stdout'].rstrip('\n') - # The key name is like "mellanox_asics" or "broadcom_asics" - asic_key_name = asic_type + "_asics" - if asic_key_name not in asic_mapping: - return "unknown" - asic_hwskus = asic_mapping[asic_key_name] - for asic_name, hwskus in asic_hwskus.items(): - if cur_hwsku.lower() in [hwsku.lower() for hwsku in hwskus]: - return asic_name - return "unknown" - - if asic_type == 'cisco-8000': - asic = "cisco-8000" - elif asic_type in ('mellanox', 'broadcom'): - asic = _get_asic_name(asic_type) - elif asic_type == 'vs': - # We need to check both mellanox and broadcom asics for vs platform - dummy_asic_list = ['broadcom', 'mellanox', 'cisco-8000'] - for dummy_asic in dummy_asic_list: - tmp_asic = _get_asic_name(dummy_asic) - if tmp_asic != "unknown": - asic = tmp_asic - break - - return asic - - -def is_valid_platform_and_version(duthost, table, scenario, operation, field_value=None): - asic = get_asic_name(duthost) - os_version = duthost.os_version - if asic == "unknown": - return False - gcu_conf = get_gcu_field_operations_conf(duthost) - - if operation == "add": - if field_value: - operation = "replace" - - # Ensure that the operation is supported by comparing with conf - try: - valid_ops = gcu_conf["tables"][table]["validator_data"]["rdma_config_update_validator"][scenario]["operations"] - if operation not in valid_ops: - return False - except KeyError: - return False - except IndexError: - return False - - # Ensure that the version is suported by comparing with conf - if "master" in os_version or "internal" in os_version: - return True - try: - version_required = gcu_conf["tables"][table]["validator_data"]["rdma_config_update_validator"][scenario]["platforms"][asic] # noqa E501 - if version_required == "": - return False - # os_version is in format "20220531.04", version_required is in format "20220500" - return os_version[0:8] >= version_required[0:8] - except KeyError: - return False - except IndexError: - return False +logger = logging.getLogger(__name__) def format_and_apply_template(duthost, template_name, extra_vars, setup): @@ -419,102 +56,3 @@ def load_and_apply_json_patch(duthost, file_name, setup): delete_tmpfile(dut, tmpfile) return outputs - - -def apply_formed_json_patch(duthost, json_patch, setup): - - duts_to_apply = [duthost] - outputs = [] - if setup["is_dualtor"]: - duts_to_apply.append(setup["rand_unselected_dut"]) - - for dut in duts_to_apply: - tmpfile = generate_tmpfile(dut) - logger.info("tmpfile {}".format(tmpfile)) - - try: - output = apply_patch(dut, json_data=json_patch, dest_file=tmpfile) - outputs.append(output) - finally: - delete_tmpfile(dut, tmpfile) - - return outputs - - -def expect_acl_table_match_multiple_bindings(duthost, - table_name, - expected_first_line_content, - expected_bindings, - setup): - """Check if acl table show as expected - Acl table with multiple bindings will show as such - - Table_Name Table_Type Ethernet4 Table_Description ingress - Ethernet8 - Ethernet12 - Ethernet16 - - So we must have separate checks for first line and bindings - """ - - cmds = "show acl table {}".format(table_name) - - duts_to_check = [duthost] - if setup["is_dualtor"]: - duts_to_check.append(setup["rand_unselected_dut"]) - - for dut in duts_to_check: - - output = dut.show_and_parse(cmds) - pytest_assert(len(output) > 0, "'{}' is not a table on this device".format(table_name)) - - first_line = output[0] - pytest_assert(set(first_line.values()) == set(expected_first_line_content)) - table_bindings = [first_line["binding"]] - for i in range(len(output)): - table_bindings.append(output[i]["binding"]) - pytest_assert(set(table_bindings) == set(expected_bindings), "ACL Table bindings don't fully match") - - -def expect_acl_rule_match(duthost, rulename, expected_content_list, setup): - """Check if acl rule shows as expected""" - - cmds = "show acl rule DYNAMIC_ACL_TABLE {}".format(rulename) - - duts_to_check = [duthost] - if setup["is_dualtor"]: - duts_to_check.append(setup["rand_unselected_dut"]) - - for dut in duts_to_check: - - output = dut.show_and_parse(cmds) - - rule_lines = len(output) - - pytest_assert(rule_lines >= 1, "'{}' is not a rule on this device".format(rulename)) - - first_line = output[0].values() - - pytest_assert(set(first_line) <= set(expected_content_list), "ACL Rule details do not match!") - - if rule_lines > 1: - for i in range(1, rule_lines): - pytest_assert(output[i]["match"] in expected_content_list, - "Unexpected match condition found: " + str(output[i]["match"])) - - -def expect_acl_rule_removed(duthost, rulename, setup): - """Check if ACL rule has been successfully removed""" - - cmds = "show acl rule DYNAMIC_ACL_TABLE {}".format(rulename) - - duts_to_check = [duthost] - if setup["is_dualtor"]: - duts_to_check.append(setup["rand_unselected_dut"]) - - for dut in duts_to_check: - output = dut.show_and_parse(cmds) - - removed = len(output) == 0 - - pytest_assert(removed, "'{}' showed a rule, this following rule should have been removed".format(cmds)) diff --git a/tests/generic_config_updater/test_aaa.py b/tests/generic_config_updater/test_aaa.py index c1fae19a6a..52bdaeffa5 100644 --- a/tests/generic_config_updater/test_aaa.py +++ b/tests/generic_config_updater/test_aaa.py @@ -2,9 +2,10 @@ import pytest from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.fixtures.tacacs import get_aaa_sub_options_value +from tests.common.gu_utils import apply_patch, expect_op_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload pytestmark = [ pytest.mark.topology('any'), @@ -51,19 +52,6 @@ def setup_env(duthosts, rand_one_dut_hostname): delete_checkpoint(duthost) -def get_aaa_sub_options_value(duthost, aaa_type, option): - r""" Verify if AAA sub type's options match with expected value - - Sample output: - admin@vlab-01:~$ show aaa | grep -Po "AAA authentication login \K.*" - local (default) - """ - output = duthost.shell(r'show aaa | grep -Po "AAA {} {} \K.*"'.format(aaa_type, option)) - - pytest_assert(not output['rc'], "Failed to grep AAA {}".format(option)) - return output['stdout'] - - def aaa_add_init_config_without_table(duthost): """ Add initial config not containing AAA table @@ -431,6 +419,7 @@ def test_tc2_tacacs_global_suite(rand_selected_dut): """ This test is for default setting when configDB doesn't contian TACACS table. So we remove TACACS config at first. """ + aaa_add_init_config_without_table(rand_selected_dut) tacacs_add_init_config_without_table(rand_selected_dut) tacacs_global_tc2_add_config(rand_selected_dut) tacacs_global_tc2_invalid_input(rand_selected_dut) diff --git a/tests/generic_config_updater/test_bgp_prefix.py b/tests/generic_config_updater/test_bgp_prefix.py index 52612e0ef9..3f40de54ed 100644 --- a/tests/generic_config_updater/test_bgp_prefix.py +++ b/tests/generic_config_updater/test_bgp_prefix.py @@ -3,9 +3,9 @@ import re from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_failure, expect_op_success -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_op_failure, expect_op_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload pytestmark = [ pytest.mark.topology('t1'), # It is a t1 only feature @@ -22,6 +22,20 @@ PREFIXES_V6_RE = r"ipv6 prefix-list PL_ALLOW_LIST_DEPLOYMENT_ID_0_COMMUNITY_{}_V6 seq \d+ permit {}" +@pytest.fixture(autouse=True) +def _ignore_allow_list_errlogs(duthosts, rand_one_dut_hostname, loganalyzer): + """Ignore expected failures logs during test execution.""" + if loganalyzer: + IgnoreRegex = [ + ".*ERR bgp#bgpcfgd: BGPAllowListMgr::Default action community value is not found.*", + ] + duthost = duthosts[rand_one_dut_hostname] + """Cisco 8111-O64 has different allow list config""" + if duthost.facts['hwsku'] == 'Cisco-8111-O64': + loganalyzer[rand_one_dut_hostname].ignore_regex.extend(IgnoreRegex) + return + + def get_bgp_prefix_runningconfig(duthost): """ Get bgp prefix config """ diff --git a/tests/generic_config_updater/test_bgp_sentinel.py b/tests/generic_config_updater/test_bgp_sentinel.py index fa84a62f78..be35f1a13b 100644 --- a/tests/generic_config_updater/test_bgp_sentinel.py +++ b/tests/generic_config_updater/test_bgp_sentinel.py @@ -4,9 +4,9 @@ import ipaddress from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_op_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload pytestmark = [ diff --git a/tests/generic_config_updater/test_bgp_speaker.py b/tests/generic_config_updater/test_bgp_speaker.py index af141cc61f..6dcdeda193 100644 --- a/tests/generic_config_updater/test_bgp_speaker.py +++ b/tests/generic_config_updater/test_bgp_speaker.py @@ -4,9 +4,9 @@ import ipaddress from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_op_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload pytestmark = [ pytest.mark.topology('t0'), # BGP Speaker is limited to t0 only diff --git a/tests/generic_config_updater/test_bgpl.py b/tests/generic_config_updater/test_bgpl.py index edc4fa5500..b42ad26e66 100644 --- a/tests/generic_config_updater/test_bgpl.py +++ b/tests/generic_config_updater/test_bgpl.py @@ -5,9 +5,9 @@ from netaddr import IPNetwork from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.generators import generate_ip_through_default_route -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_op_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload pytestmark = [ pytest.mark.topology('t0', 'm0', 'mx'), diff --git a/tests/generic_config_updater/test_cacl.py b/tests/generic_config_updater/test_cacl.py index 5764bb01f2..2631db4459 100644 --- a/tests/generic_config_updater/test_cacl.py +++ b/tests/generic_config_updater/test_cacl.py @@ -4,9 +4,9 @@ import difflib from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_res_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_op_success, expect_res_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload # Test on t0 topo to verify functionality and to choose predefined variable # admin@vlab-01:~$ show acl table @@ -18,7 +18,7 @@ # SSH_ONLY CTRLPLANE SSH SSH_ONLY ingress pytestmark = [ - pytest.mark.topology('t0', 'm0', 'mx'), + pytest.mark.topology('t0', 'm0', 'mx', 't1'), ] logger = logging.getLogger(__name__) @@ -134,24 +134,24 @@ def expect_res_success_acl_rule(duthost, expected_content_list, unexpected_conte expect_res_success(duthost, output, expected_content_list, unexpected_content_list) -def cacl_tc1_add_new_table(duthost): +def cacl_tc1_add_new_table(duthost, protocol): """ Add acl table for test Sample output admin@vlab-01:~$ show acl table - Name Type Binding Description Stage - ------ --------- --------- ------------- ------- - ... - TEST_1 CTRLPLANE SNMP Test_Table_1 ingress + Name Type Binding Description Stage Status + ---------------------- --------- --------------- ---------------------------- ------- -------- + SNMP_TEST_1 CTRLPLANE SNMP SNMP_Test_Table_1 ingress Active """ + table = "{}_TEST_1".format(protocol) json_patch = [ { "op": "add", - "path": "/ACL_TABLE/TEST_1", + "path": "/ACL_TABLE/{}".format(table), "value": { - "policy_desc": "Test_Table_1", + "policy_desc": "{}_Test_Table_1".format(protocol), "services": [ - "SNMP" + protocol ], "stage": "ingress", "type": "CTRLPLANE" @@ -166,23 +166,27 @@ def cacl_tc1_add_new_table(duthost): output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) expect_op_success(duthost, output) - expected_content_list = ["TEST_1", "CTRLPLANE", "SNMP", "Test_Table_1", "ingress"] - expect_acl_table_match(duthost, "TEST_1", expected_content_list) + expected_content_list = [table, "CTRLPLANE", protocol, "{}_Test_Table_1".format(protocol), "ingress"] + expect_acl_table_match(duthost, table, expected_content_list) finally: delete_tmpfile(duthost, tmpfile) -def cacl_tc1_add_duplicate_table(duthost): +def cacl_tc1_add_duplicate_table(duthost, protocol): """ Add duplicate acl table """ + if protocol == 'SSH': + table_name = "SSH_ONLY" + else: + table_name = "{}_ACL".format(protocol) json_patch = [ { "op": "add", - "path": "/ACL_TABLE/SNMP_ACL", + "path": "/ACL_TABLE/{}".format(table_name), "value": { - "policy_desc": "SNMP_ACL", + "policy_desc": table_name, "services": [ - "SNMP" + protocol ], "stage": "ingress", "type": "CTRLPLANE" @@ -200,32 +204,53 @@ def cacl_tc1_add_duplicate_table(duthost): delete_tmpfile(duthost, tmpfile) -def cacl_tc1_replace_table_variable(duthost): +def cacl_tc1_replace_table_variable(duthost, protocol): """ Replace acl table with SSH service Expected output admin@vlab-01:~$ show acl table Name Type Binding Description Stage ---------- --------- --------------- ------------- ------- - SNMP_ACL CTRLPLANE SSH SNMP_TO_SSH egress + SNMP_ACL CTRLPLANE SNMP SNMP_TO_SSH egress """ - json_patch = [ - { - "op": "replace", - "path": "/ACL_TABLE/SNMP_ACL/stage", - "value": "egress" - }, - { - "op": "replace", - "path": "/ACL_TABLE/SNMP_ACL/services/0", - "value": "SSH" - }, - { - "op": "replace", - "path": "/ACL_TABLE/SNMP_ACL/policy_desc", - "value": "SNMP_TO_SSH" - } - ] + if protocol == 'SSH': + table_name = "SSH_ONLY" + json_patch = [ + { + "op": "replace", + "path": "/ACL_TABLE/{}/stage".format(table_name), + "value": "egress" + }, + { + "op": "replace", + "path": "/ACL_TABLE/{}/services/0".format(table_name), + "value": "NTP" + }, + { + "op": "replace", + "path": "/ACL_TABLE/{}/policy_desc".format(table_name), + "value": "{}_TO_NTP".format(protocol) + } + ] + else: + table_name = "{}_ACL".format(protocol) + json_patch = [ + { + "op": "replace", + "path": "/ACL_TABLE/{}/stage".format(table_name), + "value": "egress" + }, + { + "op": "replace", + "path": "/ACL_TABLE/{}/services/0".format(table_name), + "value": "SSH" + }, + { + "op": "replace", + "path": "/ACL_TABLE/{}/policy_desc".format(table_name), + "value": "{}_TO_SSH".format(protocol) + } + ] tmpfile = generate_tmpfile(duthost) logger.info("tmpfile {}".format(tmpfile)) @@ -233,23 +258,26 @@ def cacl_tc1_replace_table_variable(duthost): try: output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) expect_op_success(duthost, output) - - expected_content_list = ["SNMP_ACL", "CTRLPLANE", "SSH", - "SNMP_TO_SSH", "egress"] - expect_acl_table_match(duthost, "SNMP_ACL", expected_content_list) + if protocol == 'SSH': + expected_content_list = [table_name, "CTRLPLANE", "NTP", + "{}_TO_NTP".format(protocol), "egress"] + else: + expected_content_list = [table_name, "CTRLPLANE", "SSH", + "{}_TO_SSH".format(protocol), "egress"] + expect_acl_table_match(duthost, table_name, expected_content_list) finally: delete_tmpfile(duthost, tmpfile) -def cacl_tc1_add_invalid_table(duthost): +def cacl_tc1_add_invalid_table(duthost, protocol): """ Add invalid acl table {"service": "SSH", "stage": "ogress", "type": "CTRLPLANE"}, # wrong stage {"service": "SSH", "stage": "ingress", "type": "TRLPLANE"} # wrong type """ invalid_table = [ - {"service": "SSH", "stage": "ogress", "type": "CTRLPLANE"}, - {"service": "SSH", "stage": "ingress", "type": "TRLPLANE"} + {"service": protocol, "stage": "ogress", "type": "CTRLPLANE"}, + {"service": protocol, "stage": "ingress", "type": "TRLPLANE"} ] for ele in invalid_table: @@ -297,13 +325,17 @@ def cacl_tc1_remove_unexisted_table(duthost): delete_tmpfile(duthost, tmpfile) -def cacl_tc1_remove_table(duthost): +def cacl_tc1_remove_table(duthost, protocol): """ Remove acl table test """ + if protocol == 'SSH': + table_name = "SSH_ONLY" + else: + table_name = "{}_ACL".format(protocol) json_patch = [ { "op": "remove", - "path": "/ACL_TABLE/SSH_ONLY" + "path": "/ACL_TABLE/{}".format(table_name) } ] @@ -314,21 +346,12 @@ def cacl_tc1_remove_table(duthost): output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) expect_op_success(duthost, output) - expect_acl_table_match(duthost, "SSH_ONLY", []) + expect_acl_table_match(duthost, table_name, []) finally: delete_tmpfile(duthost, tmpfile) -def test_cacl_tc1_acl_table_suite(rand_selected_dut): - cacl_tc1_add_new_table(rand_selected_dut) - cacl_tc1_add_duplicate_table(rand_selected_dut) - cacl_tc1_replace_table_variable(rand_selected_dut) - cacl_tc1_add_invalid_table(rand_selected_dut) - cacl_tc1_remove_unexisted_table(rand_selected_dut) - cacl_tc1_remove_table(rand_selected_dut) - - -def cacl_tc2_add_init_rule(duthost): +def cacl_tc2_add_init_rule(duthost, protocol): """ Add acl rule for test Check 'ip tables' to make sure rule is actually being applied @@ -342,51 +365,93 @@ def cacl_tc2_add_init_rule(duthost): SRC_IP: 9.9.9.9/32 """ + params_dict = {} + + if protocol == 'SSH': + params_dict["table"] = "SSH_ONLY" + params_dict["IP_PROTOCOL"] = "6" + params_dict["L4_DST_PORT"] = "22" + elif protocol == 'SNMP': + params_dict["table"] = "SNMP_ACL" + params_dict["IP_PROTOCOL"] = "17" + params_dict["L4_DST_PORT"] = "161" + elif protocol == 'NTP': + params_dict["table"] = "NTP_ACL" + params_dict["IP_PROTOCOL"] = "17" + params_dict["L4_DST_PORT"] = "123" + elif protocol == 'EXTERNAL_CLIENT': + params_dict["table"] = "EXTERNAL_CLIENT_ACL" + params_dict["IP_PROTOCOL"] = "6" + params_dict["L4_DST_PORT"] = "8081" json_patch = [ { "op": "add", "path": "/ACL_RULE", "value": { - "SSH_ONLY|TEST_DROP": { - "L4_DST_PORT": "22", - "IP_PROTOCOL": "6", - "IP_TYPE": "IP", - "PACKET_ACTION": "DROP", - "PRIORITY": "9998", - "SRC_IP": "9.9.9.9/32" + "{}|TEST_DROP".format(params_dict["table"]): { + "IP_PROTOCOL": "{}".format(params_dict["IP_PROTOCOL"]), + "L4_DST_PORT": "{}".format(params_dict["L4_DST_PORT"]), + "IP_TYPE": "IP", + "PACKET_ACTION": "DROP", + "PRIORITY": "9998", + "SRC_IP": "9.9.9.9/32" } } } ] - tmpfile = generate_tmpfile(duthost) logger.info("tmpfile {}".format(tmpfile)) try: output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) expect_op_success(duthost, output) - - expected_content_list = ["-A INPUT -s 9.9.9.9/32 -p tcp -m tcp --dport 22 -j DROP"] + if protocol == 'SSH': + expected_content_list = ["-A INPUT -s 9.9.9.9/32 -p tcp -m tcp --dport 22 -j DROP"] + if protocol == 'NTP': + expected_content_list = ["-A INPUT -s 9.9.9.9/32 -p udp -m udp --dport 123 -j DROP"] + elif protocol == 'SNMP': + expected_content_list = ["-A INPUT -s 9.9.9.9/32 -p tcp -m tcp --dport 161 -j DROP", + "-A INPUT -s 9.9.9.9/32 -p udp -m udp --dport 161 -j DROP"] + elif protocol == 'EXTERNAL_CLIENT': + expected_content_list = ["-A INPUT -s 9.9.9.9/32 -p tcp -m tcp --dport 8081 -j DROP"] expect_res_success_acl_rule(duthost, expected_content_list, []) finally: delete_tmpfile(duthost, tmpfile) -def cacl_tc2_add_duplicate_rule(duthost): +def cacl_tc2_add_duplicate_rule(duthost, protocol): """ Add duplicate acl rule for test """ + params_dict = {} + + if protocol == 'SSH': + params_dict["table"] = "SSH_ONLY" + params_dict["IP_PROTOCOL"] = "6" + params_dict["L4_DST_PORT"] = "22" + elif protocol == 'SNMP': + params_dict["table"] = "SNMP_ACL" + params_dict["IP_PROTOCOL"] = "17" + params_dict["L4_DST_PORT"] = "161" + elif protocol == 'NTP': + params_dict["table"] = "NTP_ACL" + params_dict["IP_PROTOCOL"] = "6" + params_dict["L4_DST_PORT"] = "123" + elif protocol == 'EXTERNAL_CLIENT': + params_dict["table"] = "EXTERNAL_CLIENT_ACL" + params_dict["IP_PROTOCOL"] = "6" + params_dict["L4_DST_PORT"] = "8081" json_patch = [ { "op": "add", "path": "/ACL_RULE", "value": { - "SSH_ONLY|TEST_DROP": { - "L4_DST_PORT": "22", - "IP_PROTOCOL": "6", - "IP_TYPE": "IP", - "PACKET_ACTION": "DROP", - "PRIORITY": "9998", - "SRC_IP": "9.9.9.9/32" + "{}|TEST_DROP".format(params_dict["table"]): { + "IP_PROTOCOL": "{}".format(params_dict["IP_PROTOCOL"]), + "L4_DST_PORT": "{}".format(params_dict["L4_DST_PORT"]), + "IP_TYPE": "IP", + "PACKET_ACTION": "DROP", + "PRIORITY": "9998", + "SRC_IP": "9.9.9.9/32" } } } @@ -402,7 +467,7 @@ def cacl_tc2_add_duplicate_rule(duthost): delete_tmpfile(duthost, tmpfile) -def cacl_tc2_replace_rule(duthost): +def cacl_tc2_replace_rule(duthost, protocol): """ Replace a value from acl rule test Check 'ip tables' to make sure rule is actually being applied @@ -415,23 +480,41 @@ def cacl_tc2_replace_rule(duthost): L4_DST_PORT: 22 SRC_IP: 8.8.8.8/32 """ + if protocol == 'SSH': + table = 'SSH_ONLY' + elif protocol == 'SNMP': + table = 'SNMP_ACL' + elif protocol == 'NTP': + table = 'NTP_ACL' + elif protocol == 'EXTERNAL_CLIENT': + table = 'EXTERNAL_CLIENT_ACL' json_patch = [ { "op": "replace", - "path": "/ACL_RULE/SSH_ONLY|TEST_DROP/SRC_IP", + "path": "/ACL_RULE/{}|TEST_DROP/SRC_IP".format(table), "value": "8.8.8.8/32" } ] - tmpfile = generate_tmpfile(duthost) logger.info("tmpfile {}".format(tmpfile)) try: output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) expect_op_success(duthost, output) - - expected_content_list = ["-A INPUT -s 8.8.8.8/32 -p tcp -m tcp --dport 22 -j DROP"] - unexpected_content_list = ["-A INPUT -s 9.9.9.9/32 -p tcp -m tcp --dport 22 -j DROP"] + if protocol == 'SSH': + expected_content_list = ["-A INPUT -s 8.8.8.8/32 -p tcp -m tcp --dport 22 -j DROP"] + unexpected_content_list = ["-A INPUT -s 9.9.9.9/32 -p tcp -m tcp --dport 22 -j DROP"] + if protocol == 'NTP': + expected_content_list = ["-A INPUT -s 8.8.8.8/32 -p udp -m udp --dport 123 -j DROP"] + unexpected_content_list = ["-A INPUT -s 9.9.9.9/32 -p udp -m udp --dport 123 -j DROP"] + elif protocol == 'SNMP': + expected_content_list = ["-A INPUT -s 8.8.8.8/32 -p tcp -m tcp --dport 161 -j DROP", + "-A INPUT -s 8.8.8.8/32 -p udp -m udp --dport 161 -j DROP"] + unexpected_content_list = ["-A INPUT -s 9.9.9.9/32 -p tcp -m tcp --dport 161 -j DROP", + "-A INPUT -s 9.9.9.9/32 -p udp -m udp --dport 161 -j DROP"] + elif protocol == 'EXTERNAL_CLIENT': + expected_content_list = ["-A INPUT -s 8.8.8.8/32 -p tcp -m tcp --dport 8081 -j DROP"] + unexpected_content_list = ["-A INPUT -s 9.9.9.9/32 -p tcp -m tcp --dport 8081 -j DROP"] expect_res_success_acl_rule(duthost, expected_content_list, unexpected_content_list) finally: delete_tmpfile(duthost, tmpfile) @@ -465,13 +548,21 @@ def cacl_tc2_add_rule_to_unexisted_table(duthost): delete_tmpfile(duthost, tmpfile) -def cacl_tc2_remove_table_before_rule(duthost): +def cacl_tc2_remove_table_before_rule(duthost, protocol): """ Remove acl table before removing acl rule """ + if protocol == 'SSH': + table = 'SSH_ONLY' + elif protocol == 'SNMP': + table = 'SNMP_ACL' + elif protocol == 'NTP': + table = 'NTP_ACL' + elif protocol == 'EXTERNAL_CLIENT': + table = 'EXTERNAL_CLIENT_ACL' json_patch = [ { "op": "remove", - "path": "/ACL_TABLE/SSH_ONLY" + "path": "/ACL_TABLE/{}".format(table) } ] @@ -485,13 +576,21 @@ def cacl_tc2_remove_table_before_rule(duthost): delete_tmpfile(duthost, tmpfile) -def cacl_tc2_remove_unexist_rule(duthost): +def cacl_tc2_remove_unexist_rule(duthost, protocol): """ Remove unexisted acl rule """ + if protocol == 'SSH': + table = 'SSH_ONLY' + elif protocol == 'SNMP': + table = 'SNMP_ACL' + elif protocol == 'NTP': + table = 'NTP_ACL' + elif protocol == 'EXTERNAL_CLIENT': + table = 'EXTERNAL_CLIENT_ACL' json_patch = [ { "op": "remove", - "path": "/ACL_RULE/SSH_ONLY|TEST_DROP2" + "path": "/ACL_RULE/{}|TEST_DROP2".format(table) } ] tmpfile = generate_tmpfile(duthost) @@ -520,18 +619,80 @@ def cacl_tc2_remove_rule(duthost): output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) expect_op_success(duthost, output) - unexpected_content_list = ["-A INPUT -s 8.8.8.8/32 -p tcp -m tcp --dport 22 -j DROP"] + unexpected_content_list = ["-A INPUT -s 8.8.8.8/32 -p tcp -m tcp --dport 22 -j DROP", + "-A INPUT -s 8.8.8.8/32 -p tcp -m tcp --dport 161 -j DROP", + "-A INPUT -s 8.8.8.8/32 -p udp -m udp --dport 161 -j DROP", + "-A INPUT -s 8.8.8.8/32 -p tcp -m udp --dport 123 -j DROP", + "-A INPUT -s 8.8.8.8/32 -p tcp -m tcp --dport 8081 -j DROP"] expect_res_success_acl_rule(duthost, [], unexpected_content_list) finally: delete_tmpfile(duthost, tmpfile) +def cacl_external_client_add_new_table(duthost): + """ Add acl table for test + Sample output + admin@vlab-01:~$ show acl table + Name Type Binding Description Stage Status + ---------------------- --------- --------------- ---------------------------- ------- -------- + EXTERNAL_CLIENT_ACL CTRLPLANE EXTERNAL_CLIENT EXTERNAL_CLIENT_ACL ingress Active + """ + json_patch = [ + { + "op": "add", + "path": "/ACL_TABLE/EXTERNAL_CLIENT_ACL", + "value": { + "policy_desc": "EXTERNAL_CLIENT_ACL", + "services": [ + "EXTERNAL_CLIENT" + ], + "stage": "ingress", + "type": "CTRLPLANE" + } + } + ] + + tmpfile = generate_tmpfile(duthost) + logger.info("tmpfile {}".format(tmpfile)) + + try: + output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) + expect_op_success(duthost, output) + + expected_content_list = ["EXTERNAL_CLIENT_ACL", "CTRLPLANE", "EXTERNAL_CLIENT", + "EXTERNAL_CLIENT_ACL", "ingress"] + expect_acl_table_match(duthost, "EXTERNAL_CLIENT_ACL", expected_content_list) + finally: + delete_tmpfile(duthost, tmpfile) + + +@pytest.fixture(scope="module", params=["SSH", "NTP", "SNMP", "EXTERNAL_CLIENT"]) +def cacl_protocol(request): # noqa F811 + """ + Return the protocol to be tested + """ + return request.param + + +def test_cacl_tc1_acl_table_suite(cacl_protocol, rand_selected_dut): + logger.info("Test acl table for protocol {}".format(cacl_protocol)) + cacl_tc1_add_new_table(rand_selected_dut, cacl_protocol) + cacl_tc1_add_duplicate_table(rand_selected_dut, cacl_protocol) + cacl_tc1_replace_table_variable(rand_selected_dut, cacl_protocol) + cacl_tc1_add_invalid_table(rand_selected_dut, cacl_protocol) + cacl_tc1_remove_unexisted_table(rand_selected_dut) + cacl_tc1_remove_table(rand_selected_dut, cacl_protocol) + + # ACL_RULE tests are related. So group them into one test. -def test_cacl_tc2_acl_rule_test(rand_selected_dut): - cacl_tc2_add_init_rule(rand_selected_dut) - cacl_tc2_add_duplicate_rule(rand_selected_dut) - cacl_tc2_replace_rule(rand_selected_dut) +def test_cacl_tc2_acl_rule_test(cacl_protocol, rand_selected_dut): + logger.info("Test acl table for protocol {}".format(cacl_protocol)) + if cacl_protocol == 'EXTERNAL_CLIENT': + cacl_external_client_add_new_table(rand_selected_dut) + cacl_tc2_add_init_rule(rand_selected_dut, cacl_protocol) + cacl_tc2_add_duplicate_rule(rand_selected_dut, cacl_protocol) + cacl_tc2_replace_rule(rand_selected_dut, cacl_protocol) cacl_tc2_add_rule_to_unexisted_table(rand_selected_dut) - cacl_tc2_remove_table_before_rule(rand_selected_dut) - cacl_tc2_remove_unexist_rule(rand_selected_dut) + cacl_tc2_remove_table_before_rule(rand_selected_dut, cacl_protocol) + cacl_tc2_remove_unexist_rule(rand_selected_dut, cacl_protocol) cacl_tc2_remove_rule(rand_selected_dut) diff --git a/tests/generic_config_updater/test_dhcp_relay.py b/tests/generic_config_updater/test_dhcp_relay.py index 9fe7bc61ca..9dca17b1cb 100644 --- a/tests/generic_config_updater/test_dhcp_relay.py +++ b/tests/generic_config_updater/test_dhcp_relay.py @@ -3,11 +3,11 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until -from tests.common.fixtures.duthost_utils import utils_vlan_intfs_dict_orig, \ - utils_vlan_intfs_dict_add, utils_create_test_vlans # noqa F401 -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_res_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload, rollback +from tests.common.fixtures.duthost_utils import utils_vlan_intfs_dict_orig,\ + utils_vlan_intfs_dict_add, utils_create_test_vlans # noqa F401 +from tests.common.gu_utils import apply_patch, expect_op_success, expect_res_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload, rollback pytestmark = [ pytest.mark.topology('t0', 'm0'), diff --git a/tests/generic_config_updater/test_dynamic_acl.py b/tests/generic_config_updater/test_dynamic_acl.py index 6a7ad87d13..f7c86f056b 100644 --- a/tests/generic_config_updater/test_dynamic_acl.py +++ b/tests/generic_config_updater/test_dynamic_acl.py @@ -23,19 +23,19 @@ import ptf.testutils as testutils from ipaddress import ip_network, IPv6Network, IPv4Network -from tests.arp.arp_utils import increment_ipv6_addr, increment_ipv4_addr -from tests.common.fixtures.ptfhost_utils import remove_ip_addresses # noqa F401 -from tests.generic_config_updater.gu_utils import expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.fixtures.ptfhost_utils import remove_ip_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 +from tests.common.gu_utils import expect_op_success, expect_op_failure +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_formed_json_patch +from tests.common.gu_utils import expect_acl_rule_match, expect_acl_rule_removed +from tests.common.gu_utils import expect_acl_table_match_multiple_bindings from tests.generic_config_updater.gu_utils import format_and_apply_template, load_and_apply_json_patch -from tests.generic_config_updater.gu_utils import apply_formed_json_patch -from tests.generic_config_updater.gu_utils import expect_acl_rule_match, expect_acl_rule_removed -from tests.generic_config_updater.gu_utils import expect_acl_table_match_multiple_bindings from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor # noqa F401 from tests.common.dualtor.dual_tor_utils import setup_standby_ports_on_rand_unselected_tor # noqa F401 -from tests.common.utilities import get_upstream_neigh_type, get_downstream_neigh_type - +from tests.common.utilities import get_upstream_neigh_type, get_downstream_neigh_type, \ + increment_ipv4_addr, increment_ipv6_addr pytestmark = [ pytest.mark.topology('t0', 'm0'), @@ -122,9 +122,8 @@ class DHCP6OptClientLinkLayerAddr(_DHCP6OptGuessPayload): # RFC6939 ShortField("lltype", 1), # ethernet _LLAddrField("clladdr", ETHER_ANY)] -# Fixtures - +# Fixtures @pytest.fixture(scope="module") def setup(rand_selected_dut, rand_unselected_dut, tbinfo, vlan_name, topo_scenario, ptfadapter, ptfhost): """Setup various variables neede for different tests""" @@ -847,7 +846,8 @@ def dynamic_acl_create_dhcp_forward_rule(duthost, setup): expect_acl_rule_match(duthost, "DHCPV6_RULE", expected_v6_rule_content, setup) -def dynamic_acl_verify_packets(setup, ptfadapter, packets, packets_dropped, src_port=None): +def dynamic_acl_verify_packets(setup, ptfadapter, packets, packets_dropped, src_port=None, + skip_traffic_test=False): # noqa F811 """Verify that the given packets are either dropped/forwarded correctly Args: @@ -862,6 +862,9 @@ def dynamic_acl_verify_packets(setup, ptfadapter, packets, packets_dropped, src_ if src_port is None: src_port = setup["blocked_src_port_indice"] + if skip_traffic_test is True: + logger.info("Skipping traffic test") + return for rule, pkt in list(packets.items()): logger.info("Testing that {} packets are correctly {}".format(rule, action_type)) exp_pkt = build_exp_pkt(pkt) @@ -1066,7 +1069,8 @@ def test_gcu_acl_arp_rule_creation(rand_selected_dut, setup, dynamic_acl_create_table, prepare_ptf_intf_and_ip, - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Test that we can create a blanket ARP/NDP packet forwarding rule with GCU, and that ARP/NDP packets are correctly forwarded while all others are dropped.""" @@ -1101,7 +1105,8 @@ def test_gcu_acl_arp_rule_creation(rand_selected_dut, ptfadapter, packets=generate_packets(setup, DST_IP_BLOCKED, DST_IPV6_BLOCKED), packets_dropped=True, - src_port=ptf_intf_index) + src_port=ptf_intf_index, + skip_traffic_test=skip_traffic_test) def test_gcu_acl_dhcp_rule_creation(rand_selected_dut, @@ -1109,8 +1114,9 @@ def test_gcu_acl_dhcp_rule_creation(rand_selected_dut, ptfadapter, setup, dynamic_acl_create_table, - toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 - setup_standby_ports_on_rand_unselected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + setup_standby_ports_on_rand_unselected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Verify that DHCP and DHCPv6 forwarding rules can be created, and that dhcp packets are properly forwarded whereas others are dropped""" @@ -1125,7 +1131,8 @@ def test_gcu_acl_dhcp_rule_creation(rand_selected_dut, dynamic_acl_verify_packets(setup, ptfadapter, packets=generate_packets(setup, DST_IP_BLOCKED, DST_IPV6_BLOCKED), - packets_dropped=True) + packets_dropped=True, + skip_traffic_test=skip_traffic_test) def test_gcu_acl_drop_rule_creation(rand_selected_dut, @@ -1133,7 +1140,8 @@ def test_gcu_acl_drop_rule_creation(rand_selected_dut, ptfadapter, setup, dynamic_acl_create_table, - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Test that we can create a drop rule via GCU, and that once this drop rule is in place packets that match the drop rule are dropped and packets that do not match the drop rule are forwarded""" @@ -1142,12 +1150,14 @@ def test_gcu_acl_drop_rule_creation(rand_selected_dut, dynamic_acl_verify_packets(setup, ptfadapter, packets=generate_packets(setup, DST_IP_BLOCKED, DST_IPV6_BLOCKED), - packets_dropped=True) + packets_dropped=True, + skip_traffic_test=skip_traffic_test) dynamic_acl_verify_packets(setup, ptfadapter, packets=generate_packets(setup, DST_IP_BLOCKED, DST_IPV6_BLOCKED), packets_dropped=False, - src_port=setup["unblocked_src_port_indice"]) + src_port=setup["unblocked_src_port_indice"], + skip_traffic_test=skip_traffic_test) def test_gcu_acl_drop_rule_removal(rand_selected_dut, @@ -1155,7 +1165,8 @@ def test_gcu_acl_drop_rule_removal(rand_selected_dut, ptfadapter, setup, dynamic_acl_create_table, - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Test that once a drop rule is removed, packets that were previously being dropped are now forwarded""" dynamic_acl_create_three_drop_rules(rand_selected_dut, setup) @@ -1165,7 +1176,8 @@ def test_gcu_acl_drop_rule_removal(rand_selected_dut, ptfadapter, packets=generate_packets(setup, DST_IP_BLOCKED, DST_IPV6_BLOCKED), packets_dropped=False, - src_port=setup["scale_port_indices"][2]) + src_port=setup["scale_port_indices"][2], + skip_traffic_test=skip_traffic_test) def test_gcu_acl_forward_rule_priority_respected(rand_selected_dut, @@ -1173,7 +1185,8 @@ def test_gcu_acl_forward_rule_priority_respected(rand_selected_dut, ptfadapter, setup, dynamic_acl_create_table, - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Test that forward rules and drop rules can be created at the same time, with the forward rules having higher priority than drop. Then, perform a traffic test to confirm that packets that match both the forward and drop rules are correctly forwarded, as the forwarding rules have higher priority""" @@ -1181,10 +1194,11 @@ def test_gcu_acl_forward_rule_priority_respected(rand_selected_dut, dynamic_acl_create_forward_rules(rand_selected_dut, setup) dynamic_acl_create_secondary_drop_rule(rand_selected_dut, setup) - dynamic_acl_verify_packets(setup, ptfadapter, packets=generate_packets(setup), packets_dropped=False) + dynamic_acl_verify_packets(setup, ptfadapter, packets=generate_packets(setup), + packets_dropped=False, skip_traffic_test=skip_traffic_test) dynamic_acl_verify_packets(setup, ptfadapter, packets=generate_packets(setup, DST_IP_BLOCKED, DST_IPV6_BLOCKED), - packets_dropped=True) + packets_dropped=True, skip_traffic_test=skip_traffic_test) def test_gcu_acl_forward_rule_replacement(rand_selected_dut, @@ -1192,7 +1206,8 @@ def test_gcu_acl_forward_rule_replacement(rand_selected_dut, ptfadapter, setup, dynamic_acl_create_table, - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Test that forward rules can be created, and then afterwards can have their match pattern updated to a new value. Confirm that packets sent that match this new value are correctly forwarded, and that packets that are sent that match the old, replaced value are correctly dropped.""" @@ -1206,8 +1221,10 @@ def test_gcu_acl_forward_rule_replacement(rand_selected_dut, packets=generate_packets(setup, DST_IP_FORWARDED_REPLACEMENT, DST_IPV6_FORWARDED_REPLACEMENT), - packets_dropped=False) - dynamic_acl_verify_packets(setup, ptfadapter, packets=generate_packets(setup), packets_dropped=True) + packets_dropped=False, + skip_traffic_test=skip_traffic_test) + dynamic_acl_verify_packets(setup, ptfadapter, packets=generate_packets(setup), packets_dropped=True, + skip_traffic_test=skip_traffic_test) @pytest.mark.parametrize("ip_type", ["IPV4", "IPV6"]) @@ -1217,7 +1234,8 @@ def test_gcu_acl_forward_rule_removal(rand_selected_dut, setup, ip_type, dynamic_acl_create_table, - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Test that if a forward rule is created, and then removed, that packets associated with that rule are properly no longer forwarded, and packets associated with the remaining rule are forwarded""" @@ -1234,12 +1252,15 @@ def test_gcu_acl_forward_rule_removal(rand_selected_dut, # generate_packets returns ipv4 and ipv6 packets. remove vals from two dicts so that only correct packets remain drop_packets.pop(other_type) forward_packets.pop(ip_type) - dynamic_acl_verify_packets(setup, ptfadapter, drop_packets, packets_dropped=True) - dynamic_acl_verify_packets(setup, ptfadapter, forward_packets, packets_dropped=False) + dynamic_acl_verify_packets(setup, ptfadapter, drop_packets, packets_dropped=True, + skip_traffic_test=skip_traffic_test) + dynamic_acl_verify_packets(setup, ptfadapter, forward_packets, packets_dropped=False, + skip_traffic_test=skip_traffic_test) def test_gcu_acl_scale_rules(rand_selected_dut, rand_unselected_dut, ptfadapter, setup, dynamic_acl_create_table, - toggle_all_simulator_ports_to_rand_selected_tor): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + skip_traffic_test): # noqa F811 """Perform a scale test, creating 150 forward rules with top priority, and then creating a drop rule for every single VLAN port on our device. Select any one of our blocked ports, as well as the ips for two of our forward rules, @@ -1259,23 +1280,27 @@ def test_gcu_acl_scale_rules(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfadapter, generate_packets(setup, v4_dest, v6_dest), packets_dropped=False, - src_port=blocked_scale_port) + src_port=blocked_scale_port, + skip_traffic_test=skip_traffic_test) dynamic_acl_verify_packets(setup, ptfadapter, generate_packets(setup, DST_IP_BLOCKED, DST_IPV6_BLOCKED), packets_dropped=True, - src_port=blocked_scale_port) + src_port=blocked_scale_port, + skip_traffic_test=skip_traffic_test) def test_gcu_acl_nonexistent_rule_replacement(rand_selected_dut, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 - setup): + setup, + skip_traffic_test): # noqa F811 """Confirm that replacing a nonexistent rule results in operation failure""" dynamic_acl_replace_nonexistent_rule(rand_selected_dut, setup) def test_gcu_acl_nonexistent_table_removal(rand_selected_dut, toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 - setup): + setup, + skip_traffic_test): # noqa F811 """Confirm that removing a nonexistent table results in operation failure""" dynamic_acl_remove_nonexistent_table(rand_selected_dut, setup) diff --git a/tests/generic_config_updater/test_ecn_config_update.py b/tests/generic_config_updater/test_ecn_config_update.py index 6b5a018d16..a345925398 100644 --- a/tests/generic_config_updater/test_ecn_config_update.py +++ b/tests/generic_config_updater/test_ecn_config_update.py @@ -5,10 +5,10 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until from tests.common.helpers.dut_utils import verify_orchagent_running_or_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload -from tests.generic_config_updater.gu_utils import is_valid_platform_and_version +from tests.common.gu_utils import apply_patch, expect_op_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import is_valid_platform_and_version pytestmark = [ pytest.mark.topology('any'), @@ -56,10 +56,10 @@ def ensure_application_of_updated_config(duthost, configdb_field, values): def _confirm_value_in_asic_db(): wred_objects = duthost.shell('sonic-db-cli ASIC_DB keys *WRED*')["stdout"] wred_objects = wred_objects.split("\n") - if(len(wred_objects) > 1): + if (len(wred_objects) > 1): for wred_object in wred_objects: wred_data = duthost.shell('sonic-db-cli ASIC_DB hgetall {}'.format(wred_object))["stdout"] - if('NULL' in wred_data): + if ('NULL' in wred_data): continue wred_data = ast.literal_eval(wred_data) for field, value in zip(configdb_field.split(','), values.split(',')): diff --git a/tests/generic_config_updater/test_eth_interface.py b/tests/generic_config_updater/test_eth_interface.py index 8462195649..7d63aaf8a9 100644 --- a/tests/generic_config_updater/test_eth_interface.py +++ b/tests/generic_config_updater/test_eth_interface.py @@ -5,9 +5,9 @@ from tests.common.config_reload import config_reload from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_op_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload from tests.common.utilities import wait_until pytestmark = [ diff --git a/tests/generic_config_updater/test_incremental_qos.py b/tests/generic_config_updater/test_incremental_qos.py index 53fc6a1dd4..7282f251b6 100644 --- a/tests/generic_config_updater/test_incremental_qos.py +++ b/tests/generic_config_updater/test_incremental_qos.py @@ -6,11 +6,11 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until from tests.common.helpers.dut_utils import verify_orchagent_running_or_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, \ - expect_res_success, expect_op_failure # noqa F401 -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload -from tests.generic_config_updater.gu_utils import is_valid_platform_and_version +from tests.common.gu_utils import apply_patch, expect_op_success, \ + expect_op_failure # noqa F401 +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import is_valid_platform_and_version from tests.common.mellanox_data import is_mellanox_device pytestmark = [ diff --git a/tests/generic_config_updater/test_ipv6.py b/tests/generic_config_updater/test_ipv6.py index 5b0c001375..249732feb9 100644 --- a/tests/generic_config_updater/test_ipv6.py +++ b/tests/generic_config_updater/test_ipv6.py @@ -4,9 +4,9 @@ import re from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_op_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload logger = logging.getLogger(__name__) diff --git a/tests/generic_config_updater/test_kubernetes_config.py b/tests/generic_config_updater/test_kubernetes_config.py index 42ea76e822..51d4234141 100644 --- a/tests/generic_config_updater/test_kubernetes_config.py +++ b/tests/generic_config_updater/test_kubernetes_config.py @@ -3,9 +3,9 @@ import re from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_op_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload pytestmark = [ diff --git a/tests/generic_config_updater/test_lo_interface.py b/tests/generic_config_updater/test_lo_interface.py index 2b401663cb..2b04831e87 100644 --- a/tests/generic_config_updater/test_lo_interface.py +++ b/tests/generic_config_updater/test_lo_interface.py @@ -3,10 +3,10 @@ import ipaddress from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload -from tests.generic_config_updater.gu_utils import create_path, check_show_ip_intf, check_vrf_route_for_intf +from tests.common.gu_utils import apply_patch, expect_op_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import create_path, check_show_ip_intf, check_vrf_route_for_intf # Test on t0 topo to verify functionality and to choose predefined variable # "LOOPBACK_INTERFACE": { diff --git a/tests/generic_config_updater/test_mgmt_interface.py b/tests/generic_config_updater/test_mgmt_interface.py new file mode 100644 index 0000000000..e5d9a220a5 --- /dev/null +++ b/tests/generic_config_updater/test_mgmt_interface.py @@ -0,0 +1,135 @@ +import ipaddress +import logging +import pytest + +from tests.common.helpers.assertions import pytest_assert +from tests.common.gu_utils import apply_patch, create_path +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.utilities import wait_for_file_changed, FORCED_MGMT_ROUTE_PRIORITY + +pytestmark = [ + pytest.mark.topology('any'), +] + +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="function") +def ensure_dut_readiness(duthost): + """ + Setup/teardown fixture for pg headroom update test + + Args: + duthost: DUT host object + """ + create_checkpoint(duthost) + + yield + + try: + logger.info("Rolled back to original checkpoint") + rollback_or_reload(duthost) + finally: + delete_checkpoint(duthost) + + +def update_forced_mgmt_route(duthost, interface_address, interface_key, routes): + # Escape '/' in interface key + json_patch = [ + { + "path": create_path(["MGMT_INTERFACE", + "eth0|{}".format(interface_address), + "forced_mgmt_routes"]) + } + ] + + if len(routes) == 0: + json_patch[0]["op"] = "remove" + else: + json_patch[0]["value"] = routes + # Replace if forced_mgmt_routes already exist + current_config = duthost.command("sonic-db-cli CONFIG_DB HGET '{}' forced_mgmt_routes@" + .format(interface_key))['stdout'] + if current_config != "": + json_patch[0]["op"] = "replace" + else: + json_patch[0]["op"] = "add" + + tmpfile = generate_tmpfile(duthost) + try: + output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) + logging.debug("json_patch: {}".format(json_patch)) + logging.debug("apply_patch result: {}".format(output)) + finally: + delete_tmpfile(duthost, tmpfile) + + +def update_and_check_forced_mgmt_routes(duthost, forced_mgmt_routes, interface_address, interface_key, + ip_type, test_route, expect_exist): + # Update forced mgmt routes with new route address + wait_for_file_changed( + duthost, + "/etc/network/interfaces", + update_forced_mgmt_route, + duthost, + interface_address, + interface_key, + forced_mgmt_routes) + + # Check /etc/network/interfaces generate correct + interfaces = duthost.command("cat /etc/network/interfaces")['stdout'] + logging.debug("interfaces: {}".format(interfaces)) + + pytest_assert(("up ip {} rule add pref {} to {} table default" + .format(ip_type, FORCED_MGMT_ROUTE_PRIORITY, test_route) in interfaces) == expect_exist) + pytest_assert(("pre-down ip {} rule delete pref {} to {} table default" + .format(ip_type, FORCED_MGMT_ROUTE_PRIORITY, test_route) in interfaces) == expect_exist) + + +def test_forced_mgmt_routes_update(duthost, ensure_dut_readiness): + # Get interface and check config generate correct + mgmt_interface_keys = duthost.command("sonic-db-cli CONFIG_DB keys 'MGMT_INTERFACE|eth0|*'")['stdout'] + logging.debug("mgmt_interface_keys: {}".format(mgmt_interface_keys)) + + for interface_key in mgmt_interface_keys.split('\n'): + logging.debug("interface_key: {}".format(interface_key)) + interface_address = interface_key.split('|')[2] + + # Get current forced mgmt routes + forced_mgmt_routes_config = duthost.command("sonic-db-cli CONFIG_DB HGET '{}' forced_mgmt_routes@" + .format(interface_key))['stdout'] + + original_forced_mgmt_routes = [] + if forced_mgmt_routes_config != "": + original_forced_mgmt_routes = forced_mgmt_routes_config.split(",") + + # Prepare new forced mgmt routes + test_route = "1::2:3:4/64" + ip_type = "-6" + if type(ipaddress.ip_network(interface_address, False)) == ipaddress.IPv4Network: + test_route = "1.2.3.4/24" + ip_type = "-4" + + updated_forced_mgmt_routes = original_forced_mgmt_routes.copy() + updated_forced_mgmt_routes.append(test_route) + logging.debug("interface address: {}, original_forced_mgmt_routes: {}, updated_forced_mgmt_routes: {}" + .format(interface_address, original_forced_mgmt_routes, updated_forced_mgmt_routes)) + + # Update forced mgmt routes with new route address + update_and_check_forced_mgmt_routes(duthost, + updated_forced_mgmt_routes, + interface_address, + interface_key, + ip_type, + test_route, + True) + + # Revert change and check again + update_and_check_forced_mgmt_routes(duthost, + original_forced_mgmt_routes, + interface_address, + interface_key, + ip_type, + test_route, + False) diff --git a/tests/generic_config_updater/test_mmu_dynamic_threshold_config_update.py b/tests/generic_config_updater/test_mmu_dynamic_threshold_config_update.py index 2438cf557e..0d80da1ed6 100644 --- a/tests/generic_config_updater/test_mmu_dynamic_threshold_config_update.py +++ b/tests/generic_config_updater/test_mmu_dynamic_threshold_config_update.py @@ -5,9 +5,9 @@ from tests.common.helpers.assertions import pytest_assert, pytest_require from tests.common.utilities import wait_until from tests.common.helpers.dut_utils import verify_orchagent_running_or_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_op_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload pytestmark = [ pytest.mark.topology('any'), diff --git a/tests/generic_config_updater/test_monitor_config.py b/tests/generic_config_updater/test_monitor_config.py index e15d3b4a98..860a567655 100644 --- a/tests/generic_config_updater/test_monitor_config.py +++ b/tests/generic_config_updater/test_monitor_config.py @@ -2,9 +2,9 @@ import pytest from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_res_success -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_op_success, expect_res_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback, rollback_or_reload pytestmark = [ pytest.mark.topology('any'), diff --git a/tests/generic_config_updater/test_ntp.py b/tests/generic_config_updater/test_ntp.py index 310c6dae18..c8cd298d2d 100644 --- a/tests/generic_config_updater/test_ntp.py +++ b/tests/generic_config_updater/test_ntp.py @@ -4,9 +4,9 @@ import re from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_failure, expect_op_success -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_op_failure, expect_op_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload from tests.common.utilities import wait_until logger = logging.getLogger(__name__) diff --git a/tests/generic_config_updater/test_pfcwd_interval.py b/tests/generic_config_updater/test_pfcwd_interval.py index 50bb489516..fd6d8d16ec 100644 --- a/tests/generic_config_updater/test_pfcwd_interval.py +++ b/tests/generic_config_updater/test_pfcwd_interval.py @@ -4,10 +4,10 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload -from tests.generic_config_updater.gu_utils import is_valid_platform_and_version +from tests.common.gu_utils import apply_patch, expect_op_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import is_valid_platform_and_version pytestmark = [ pytest.mark.asic('mellanox'), diff --git a/tests/generic_config_updater/test_pfcwd_status.py b/tests/generic_config_updater/test_pfcwd_status.py index 7af82555a9..76f5828d6b 100644 --- a/tests/generic_config_updater/test_pfcwd_status.py +++ b/tests/generic_config_updater/test_pfcwd_status.py @@ -7,20 +7,20 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until from tests.common.helpers.dut_utils import verify_orchagent_running_or_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload -from tests.generic_config_updater.gu_utils import is_valid_platform_and_version +from tests.common.gu_utils import apply_patch, expect_op_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import is_valid_platform_and_version pytestmark = [ - pytest.mark.topology('any'), - pytest.mark.device_type('physical') + pytest.mark.topology('any') ] logger = logging.getLogger(__name__) READ_FLEXDB_TIMEOUT = 20 READ_FLEXDB_INTERVAL = 5 +FLEXDB_COUNTERS_PER_PORT = 3 @pytest.fixture(autouse=True) @@ -37,6 +37,12 @@ def ignore_expected_loganalyzer_exceptions(duthosts, loganalyzer): '.*ERR syncd#syncd:.*SAI_API_SWITCH:sai_bulk_object_get_stats.* ', ] ) + if duthost.facts["asic_type"] == "vs": + loganalyzer[duthost.hostname].ignore_regex.extend( + [ + '.*ERR syncd#syncd: :- queryStatsCapability: failed to find switch oid:.* in switch state map' + ] + ) return @@ -67,7 +73,7 @@ def set_default_pfcwd_config(duthost): @pytest.fixture -def ensure_dut_readiness(duthost): +def ensure_dut_readiness(duthost, extract_pfcwd_config): """ Verify dut health/create and rollback checkpoint @@ -77,6 +83,10 @@ def ensure_dut_readiness(duthost): verify_orchagent_running_or_assert(duthost) create_checkpoint(duthost) + pfcwd_config = extract_pfcwd_config + number_of_ports = len(pfcwd_config) + check_config_update(duthost, number_of_ports * FLEXDB_COUNTERS_PER_PORT) + yield try: @@ -185,9 +195,10 @@ def test_stop_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, port): 4. Validates that orchagent is running fine pre and post test """ pfcwd_config = extract_pfcwd_config + initial_count = len(pfcwd_config) * FLEXDB_COUNTERS_PER_PORT if port == 'single': - expected_count = get_flex_db_count(duthost) - 3 + expected_count = initial_count - FLEXDB_COUNTERS_PER_PORT else: expected_count = 0 json_patch = list() @@ -227,9 +238,9 @@ def test_start_pfcwd(duthost, extract_pfcwd_config, ensure_dut_readiness, stop_p pfcwd_config = extract_pfcwd_config if port == 'single': - expected_count = 3 + expected_count = FLEXDB_COUNTERS_PER_PORT else: - expected_count = len(pfcwd_config) * 3 + expected_count = len(pfcwd_config) * FLEXDB_COUNTERS_PER_PORT json_patch = list() exp_str = 'Ethernet' op = 'add' diff --git a/tests/generic_config_updater/test_pg_headroom_update.py b/tests/generic_config_updater/test_pg_headroom_update.py index 036537c785..16a0b2e6f0 100644 --- a/tests/generic_config_updater/test_pg_headroom_update.py +++ b/tests/generic_config_updater/test_pg_headroom_update.py @@ -5,10 +5,10 @@ from tests.common.helpers.assertions import pytest_assert, pytest_require from tests.common.utilities import wait_until from tests.common.helpers.dut_utils import verify_orchagent_running_or_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload -from tests.generic_config_updater.gu_utils import is_valid_platform_and_version +from tests.common.gu_utils import apply_patch, expect_op_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import is_valid_platform_and_version, get_asic_name pytestmark = [ pytest.mark.topology('any'), @@ -79,7 +79,7 @@ def _confirm_value_in_app_and_asic_db(): @pytest.mark.parametrize("operation", ["replace"]) def test_pg_headroom_update(duthost, ensure_dut_readiness, operation, skip_when_buffer_is_dynamic_model): - asic_type = duthost.get_asic_name() + asic_type = get_asic_name(duthost) pytest_require("td2" not in asic_type, "PG headroom should be skipped on TD2") tmpfile = generate_tmpfile(duthost) diff --git a/tests/generic_config_updater/test_portchannel_interface.py b/tests/generic_config_updater/test_portchannel_interface.py index aa64a86773..f7f8e0b29c 100644 --- a/tests/generic_config_updater/test_portchannel_interface.py +++ b/tests/generic_config_updater/test_portchannel_interface.py @@ -4,10 +4,10 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.assertions import pytest_require -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload -from tests.generic_config_updater.gu_utils import create_path, check_show_ip_intf +from tests.common.gu_utils import apply_patch, expect_op_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import create_path, check_show_ip_intf # Test on t0 topo to verify functionality and to choose predefined variable # "PORTCHANNEL_INTERFACE": { diff --git a/tests/generic_config_updater/test_syslog.py b/tests/generic_config_updater/test_syslog.py index 6b10dd5dd9..47bbecb484 100644 --- a/tests/generic_config_updater/test_syslog.py +++ b/tests/generic_config_updater/test_syslog.py @@ -2,9 +2,9 @@ import pytest from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_res_success, expect_op_failure, expect_op_success -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import apply_patch, expect_res_success, expect_op_failure, expect_op_success +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload pytestmark = [ pytest.mark.topology('any'), diff --git a/tests/generic_config_updater/test_vlan_interface.py b/tests/generic_config_updater/test_vlan_interface.py index 3315e2432b..b0f697534b 100644 --- a/tests/generic_config_updater/test_vlan_interface.py +++ b/tests/generic_config_updater/test_vlan_interface.py @@ -5,10 +5,10 @@ import pytest from tests.common.helpers.assertions import pytest_assert -from tests.generic_config_updater.gu_utils import apply_patch, expect_op_success, expect_op_failure -from tests.generic_config_updater.gu_utils import generate_tmpfile, delete_tmpfile -from tests.generic_config_updater.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload -from tests.generic_config_updater.gu_utils import create_path, check_show_ip_intf +from tests.common.gu_utils import apply_patch, expect_op_success, expect_op_failure +from tests.common.gu_utils import generate_tmpfile, delete_tmpfile +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import create_path, check_show_ip_intf # Test on t0 topo to verify functionality and to choose predefined variable # "VLAN_INTERFACE": { diff --git a/tests/gnmi/conftest.py b/tests/gnmi/conftest.py index 812a13c8cb..f282cbe174 100644 --- a/tests/gnmi/conftest.py +++ b/tests/gnmi/conftest.py @@ -6,7 +6,7 @@ from tests.common.helpers.dut_utils import check_container_state from tests.gnmi.helper import gnmi_container, apply_cert_config, recover_cert_config, create_ext_conf from tests.gnmi.helper import GNMI_SERVER_START_WAIT_TIME -from tests.generic_config_updater.gu_utils import create_checkpoint, rollback +from tests.common.gu_utils import create_checkpoint, rollback logger = logging.getLogger(__name__) SETUP_ENV_CP = "test_setup_checkpoint" @@ -35,7 +35,7 @@ def download_gnmi_client(duthosts, rand_one_dut_hostname, localhost): @pytest.fixture(scope="module", autouse=True) -def setup_gnmi_server(duthosts, rand_one_dut_hostname, localhost): +def setup_gnmi_server(duthosts, rand_one_dut_hostname, localhost, ptfhost): ''' Create GNMI client certificates ''' @@ -112,10 +112,16 @@ def setup_gnmi_server(duthosts, rand_one_dut_hostname, localhost): -sha256" localhost.shell(local_command) - # Copy CA certificate and server certificate over to the DUT + # Copy CA certificate, server certificate and client certificate over to the DUT duthost.copy(src='gnmiCA.pem', dest='/etc/sonic/telemetry/') duthost.copy(src='gnmiserver.crt', dest='/etc/sonic/telemetry/') duthost.copy(src='gnmiserver.key', dest='/etc/sonic/telemetry/') + duthost.copy(src='gnmiclient.crt', dest='/etc/sonic/telemetry/') + duthost.copy(src='gnmiclient.key', dest='/etc/sonic/telemetry/') + # Copy CA certificate and client certificate over to the PTF + ptfhost.copy(src='gnmiCA.pem', dest='/root/') + ptfhost.copy(src='gnmiclient.crt', dest='/root/') + ptfhost.copy(src='gnmiclient.key', dest='/root/') create_checkpoint(duthost, SETUP_ENV_CP) apply_cert_config(duthost) diff --git a/tests/gnmi/helper.py b/tests/gnmi/helper.py index e7584b38f4..5c06ee8c3c 100644 --- a/tests/gnmi/helper.py +++ b/tests/gnmi/helper.py @@ -1,5 +1,4 @@ import time -import re import logging import pytest from tests.common.utilities import wait_until @@ -55,6 +54,15 @@ def verify_tcp_port(localhost, ip, port): logger.info("TCP: " + res['stdout'] + res['stderr']) +def add_gnmi_client_common_name(duthost, cname): + duthost.shell('sudo sonic-db-cli CONFIG_DB hset "GNMI_CLIENT_CERT|{}" "role" "role1"'.format(cname), + module_ignore_errors=True) + + +def del_gnmi_client_common_name(duthost, cname): + duthost.shell('sudo sonic-db-cli CONFIG_DB del "GNMI_CLIENT_CERT|{}"'.format(cname), module_ignore_errors=True) + + def apply_cert_config(duthost): env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) # Stop all running program @@ -74,8 +82,14 @@ def apply_cert_config(duthost): dut_command = "docker exec %s bash -c " % env.gnmi_container dut_command += "\"/usr/bin/nohup /usr/sbin/%s -logtostderr --port %s " % (env.gnmi_process, env.gnmi_port) dut_command += "--server_crt /etc/sonic/telemetry/gnmiserver.crt --server_key /etc/sonic/telemetry/gnmiserver.key " + dut_command += "--config_table_name GNMI_CLIENT_CERT " + dut_command += "--client_auth cert " dut_command += "--ca_crt /etc/sonic/telemetry/gnmiCA.pem -gnmi_native_write=true -v=10 >/root/gnmi.log 2>&1 &\"" duthost.shell(dut_command) + + # Setup gnmi client cert common name + add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic") + time.sleep(GNMI_SERVER_START_WAIT_TIME) dut_command = "sudo netstat -nap | grep %d" % env.gnmi_port output = duthost.shell(dut_command, module_ignore_errors=True) @@ -101,6 +115,9 @@ def recover_cert_config(duthost): 'systemctl restart %s' % (env.gnmi_container) ] duthost.shell_cmds(cmds=cmds) + + # Remove gnmi client cert common name + del_gnmi_client_common_name(duthost, "test.client.gnmi.sonic") assert wait_until(60, 3, 0, check_gnmi_status, duthost), "GNMI service failed to start" @@ -108,9 +125,13 @@ def gnmi_capabilities(duthost, localhost): env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) ip = duthost.mgmt_ip port = env.gnmi_port - cmd = "gnmi/gnmi_cli -client_types=gnmi -a %s:%s " % (ip, port) - cmd += "-logtostderr -client_crt ./gnmiclient.crt -client_key ./gnmiclient.key -ca_crt ./gnmiCA.pem -capabilities" - output = localhost.shell(cmd, module_ignore_errors=True) + # Run gnmi_cli in gnmi container as workaround + cmd = "docker exec %s gnmi_cli -client_types=gnmi -a %s:%s " % (env.gnmi_container, ip, port) + cmd += "-client_crt /etc/sonic/telemetry/gnmiclient.crt " + cmd += "-client_key /etc/sonic/telemetry/gnmiclient.key " + cmd += "-ca_crt /etc/sonic/telemetry/gnmiCA.pem " + cmd += "-logtostderr -capabilities" + output = duthost.shell(cmd, module_ignore_errors=True) if output['stderr']: dump_gnmi_log(duthost) dump_system_status(duthost) @@ -120,60 +141,268 @@ def gnmi_capabilities(duthost, localhost): return 0, output['stdout'] -def gnmi_set(duthost, localhost, delete_list, update_list, replace_list): +def gnmi_set(duthost, ptfhost, delete_list, update_list, replace_list): + """ + Send GNMI set request with GNMI client + + Args: + duthost: fixture for duthost + ptfhost: fixture for ptfhost + delete_list: list for delete operations + update_list: list for update operations + replace_list: list for replace operations + + Returns: + """ env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) ip = duthost.mgmt_ip port = env.gnmi_port - cmd = "gnmi/gnmi_set -target_addr %s:%s " % (ip, port) - cmd += "-alsologtostderr -cert ./gnmiclient.crt -key ./gnmiclient.key -ca ./gnmiCA.pem -time_out 60s" - for delete in delete_list: - cmd += " -delete " + delete + cmd = 'python2 /root/gnxi/gnmi_cli_py/py_gnmicli.py ' + cmd += '--timeout 30 ' + cmd += '-t %s -p %u ' % (ip, port) + cmd += '-xo sonic-db ' + cmd += '-rcert /root/gnmiCA.pem ' + cmd += '-pkey /root/gnmiclient.key ' + cmd += '-cchain /root/gnmiclient.crt ' + cmd += '-m set-update ' + xpath = '' + xvalue = '' + for path in delete_list: + path = path.replace('sonic-db:', '') + xpath += ' ' + path + xvalue += ' ""' for update in update_list: - cmd += " -update " + update + update = update.replace('sonic-db:', '') + result = update.rsplit(':', 1) + xpath += ' ' + result[0] + xvalue += ' ' + result[1] for replace in replace_list: - cmd += " -replace " + replace - output = localhost.shell(cmd, module_ignore_errors=True) + replace = replace.replace('sonic-db:', '') + result = replace.rsplit(':', 1) + xpath += ' ' + result[0] + if '#' in result[1]: + xvalue += ' ""' + else: + xvalue += ' ' + result[1] + cmd += '--xpath ' + xpath + cmd += ' ' + cmd += '--value ' + xvalue + output = ptfhost.shell(cmd, module_ignore_errors=True) + error = "GRPC error\n" + if error in output['stdout']: + dump_gnmi_log(duthost) + dump_system_status(duthost) + result = output['stdout'].split(error, 1) + raise Exception("GRPC error:" + result[1]) if output['stderr']: dump_gnmi_log(duthost) dump_system_status(duthost) - verify_tcp_port(localhost, ip, port) - return -1, output['stderr'] + raise Exception("error:" + output['stderr']) else: - return 0, output['stdout'] + return + +def gnmi_get(duthost, ptfhost, path_list): + """ + Send GNMI get request with GNMI client -def gnmi_get(duthost, localhost, path_list): + Args: + duthost: fixture for duthost + ptfhost: fixture for ptfhost + path_list: list for get path + + Returns: + msg_list: list for get result + """ env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) ip = duthost.mgmt_ip port = env.gnmi_port - cmd = "gnmi/gnmi_get -target_addr %s:%s " % (ip, port) - cmd += "-alsologtostderr -cert ./gnmiclient.crt -key ./gnmiclient.key -ca ./gnmiCA.pem" + cmd = 'python2 /root/gnxi/gnmi_cli_py/py_gnmicli.py ' + cmd += '--timeout 30 ' + cmd += '-t %s -p %u ' % (ip, port) + cmd += '-xo sonic-db ' + cmd += '-rcert /root/gnmiCA.pem ' + cmd += '-pkey /root/gnmiclient.key ' + cmd += '-cchain /root/gnmiclient.crt ' + cmd += '--encoding 4 ' + cmd += '-m get ' + cmd += '--xpath ' for path in path_list: - cmd += " -xpath " + path - output = localhost.shell(cmd, module_ignore_errors=True) + path = path.replace('sonic-db:', '') + cmd += " " + path + output = ptfhost.shell(cmd, module_ignore_errors=True) if output['stderr']: - dump_gnmi_log(duthost) - dump_system_status(duthost) - verify_tcp_port(localhost, ip, port) - return -1, [output['stderr']] + raise Exception("error:" + output['stderr']) else: msg = output['stdout'].replace('\\', '') - find_list = re.findall(r'json_ietf_val:\s*"(.*?)"\s*>', msg) - if find_list: - return 0, find_list + error = "GRPC error\n" + if error in msg: + dump_gnmi_log(duthost) + dump_system_status(duthost) + result = msg.split(error, 1) + raise Exception("GRPC error:" + result[1]) + mark = 'The GetResponse is below\n' + '-'*25 + '\n' + if mark in msg: + result = msg.split(mark, 1) + msg_list = result[1].split('-'*25)[0:-1] + return [msg.strip("\n") for msg in msg_list] else: - return -1, [msg] + dump_gnmi_log(duthost) + dump_system_status(duthost) + raise Exception("error:" + msg) + +# py_gnmicli does not fully support POLLING mode +# Use gnmi_cli instead +def gnmi_subscribe_polling(duthost, ptfhost, path_list, interval_ms, count): + """ + Send GNMI subscribe request with GNMI client -def gnoi_reboot(duthost, localhost, method, delay, message): + Args: + duthost: fixture for duthost + ptfhost: fixture for ptfhost + path_list: list for get path + interval_ms: interval, unit is ms + count: update count + + Returns: + msg: gnmi client output + """ + if path_list is None: + logger.error("path_list is None") + return "", "" env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) ip = duthost.mgmt_ip port = env.gnmi_port - cmd = "gnmi/gnoi_client -target %s:%s " % (ip, port) - cmd += "-logtostderr -cert ./gnmiclient.crt -key ./gnmiclient.key -ca ./gnmiCA.pem -rpc Reboot " + interval = interval_ms / 1000.0 + # Run gnmi_cli in gnmi container as workaround + cmd = "docker exec %s gnmi_cli -client_types=gnmi -a %s:%s " % (env.gnmi_container, ip, port) + cmd += "-client_crt /etc/sonic/telemetry/gnmiclient.crt " + cmd += "-client_key /etc/sonic/telemetry/gnmiclient.key " + cmd += "-ca_crt /etc/sonic/telemetry/gnmiCA.pem " + cmd += "-logtostderr " + # Use sonic-db as default origin + cmd += '-origin=sonic-db ' + cmd += '-query_type=polling ' + cmd += '-polling_interval %us -count %u ' % (int(interval), count) + for path in path_list: + path = path.replace('sonic-db:', '') + cmd += '-q %s ' % (path) + output = duthost.shell(cmd, module_ignore_errors=True) + return output['stdout'], output['stderr'] + + +def gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, interval_ms, count): + """ + Send GNMI subscribe request with GNMI client + + Args: + duthost: fixture for duthost + ptfhost: fixture for ptfhost + path_list: list for get path + interval_ms: interval, unit is ms + count: update count + + Returns: + msg: gnmi client output + """ + if path_list is None: + logger.error("path_list is None") + return "", "" + env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) + ip = duthost.mgmt_ip + port = env.gnmi_port + cmd = 'python2 /root/gnxi/gnmi_cli_py/py_gnmicli.py ' + cmd += '--timeout 30 ' + cmd += '-t %s -p %u ' % (ip, port) + cmd += '-xo sonic-db ' + cmd += '-rcert /root/gnmiCA.pem ' + cmd += '-pkey /root/gnmiclient.key ' + cmd += '-cchain /root/gnmiclient.crt ' + cmd += '--encoding 4 ' + cmd += '-m subscribe ' + cmd += '--subscribe_mode 0 --submode 2 --create_connections 1 ' + cmd += '--interval %u --update_count %u ' % (interval_ms, count) + cmd += '--xpath ' + for path in path_list: + path = path.replace('sonic-db:', '') + cmd += " " + path + output = ptfhost.shell(cmd, module_ignore_errors=True) + msg = output['stdout'].replace('\\', '') + return msg, output['stderr'] + + +def gnmi_subscribe_streaming_onchange(duthost, ptfhost, path_list, count): + """ + Send GNMI subscribe request with GNMI client + + Args: + duthost: fixture for duthost + ptfhost: fixture for ptfhost + path_list: list for get path + count: update count + + Returns: + msg: gnmi client output + """ + if path_list is None: + logger.error("path_list is None") + return "", "" + env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) + ip = duthost.mgmt_ip + port = env.gnmi_port + cmd = 'python2 /root/gnxi/gnmi_cli_py/py_gnmicli.py ' + cmd += '--timeout 30 ' + cmd += '-t %s -p %u ' % (ip, port) + cmd += '-xo sonic-db ' + cmd += '-rcert /root/gnmiCA.pem ' + cmd += '-pkey /root/gnmiclient.key ' + cmd += '-cchain /root/gnmiclient.crt ' + cmd += '--encoding 4 ' + cmd += '-m subscribe ' + cmd += '--subscribe_mode 0 --submode 1 --create_connections 1 ' + cmd += '--update_count %u ' % count + cmd += '--xpath ' + for path in path_list: + path = path.replace('sonic-db:', '') + cmd += " " + path + output = ptfhost.shell(cmd, module_ignore_errors=True) + msg = output['stdout'].replace('\\', '') + return msg, output['stderr'] + + +def gnoi_reboot(duthost, method, delay, message): + env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) + ip = duthost.mgmt_ip + port = env.gnmi_port + # Run gnoi_client in gnmi container as workaround + cmd = "docker exec %s gnoi_client -target %s:%s " % (env.gnmi_container, ip, port) + cmd += "-cert /etc/sonic/telemetry/gnmiclient.crt " + cmd += "-key /etc/sonic/telemetry/gnmiclient.key " + cmd += "-ca /etc/sonic/telemetry/gnmiCA.pem " + cmd += "-logtostderr -rpc Reboot " cmd += '-jsonin "{\\\"method\\\":%d, \\\"delay\\\":%d, \\\"message\\\":\\\"%s\\\"}"' % (method, delay, message) - output = localhost.shell(cmd, module_ignore_errors=True) + output = duthost.shell(cmd, module_ignore_errors=True) + if output['stderr']: + logger.error(output['stderr']) + return -1, output['stderr'] + else: + return 0, output['stdout'] + + +def gnoi_request(duthost, localhost, rpc, request_json_data): + env = GNMIEnvironment(duthost, GNMIEnvironment.GNMI_MODE) + ip = duthost.mgmt_ip + port = env.gnmi_port + cmd = "docker exec %s gnoi_client -target %s:%s " % (env.gnmi_container, ip, port) + cmd += "-cert /etc/sonic/telemetry/gnmiclient.crt " + cmd += "-key /etc/sonic/telemetry/gnmiclient.key " + cmd += "-ca /etc/sonic/telemetry/gnmiCA.pem " + cmd += "-logtostderr -rpc {} ".format(rpc) + cmd += f'-jsonin \'{request_json_data}\'' + output = duthost.shell(cmd, module_ignore_errors=True) if output['stderr']: + logger.error(output['stderr']) return -1, output['stderr'] else: return 0, output['stdout'] diff --git a/tests/gnmi/test_gnmi.py b/tests/gnmi/test_gnmi.py index 37380f1036..c2203ace33 100644 --- a/tests/gnmi/test_gnmi.py +++ b/tests/gnmi/test_gnmi.py @@ -1,7 +1,7 @@ import pytest import logging -from .helper import gnmi_capabilities +from .helper import gnmi_capabilities, gnmi_set, add_gnmi_client_common_name, del_gnmi_client_common_name logger = logging.getLogger(__name__) @@ -20,3 +20,45 @@ def test_gnmi_capabilities(duthosts, rand_one_dut_hostname, localhost): assert ret == 0, msg assert "sonic-db" in msg, msg assert "JSON_IETF" in msg, msg + + +@pytest.fixture(scope="function") +def setup_invalid_client_cert_cname(duthosts, rand_one_dut_hostname): + duthost = duthosts[rand_one_dut_hostname] + del_gnmi_client_common_name(duthost, "test.client.gnmi.sonic") + add_gnmi_client_common_name(duthost, "invalid.cname") + + keys = duthost.shell('sudo sonic-db-cli CONFIG_DB keys GNMI*')["stdout_lines"] + logger.debug("GNMI client cert keys: {}".format(keys)) + + yield + + del_gnmi_client_common_name(duthost, "invalid.cname") + add_gnmi_client_common_name(duthost, "test.client.gnmi.sonic") + + +def test_gnmi_authorize_failed_with_invalid_cname(duthosts, + rand_one_dut_hostname, + ptfhost, + setup_invalid_client_cert_cname): + ''' + Verify GNMI native write, incremental config for configDB + GNMI set request with invalid path + ''' + duthost = duthosts[rand_one_dut_hostname] + + file_name = "vnet.txt" + text = "{\"Vnet1\": {\"vni\": \"1000\", \"guid\": \"559c6ce8-26ab-4193-b946-ccc6e8f930b2\"}}" + with open(file_name, 'w') as file: + file.write(text) + ptfhost.copy(src=file_name, dest='/root') + # Add DASH_VNET_TABLE + update_list = ["/sonic-db:APPL_DB/localhost/DASH_VNET_TABLE:@/root/%s" % (file_name)] + msg = "" + try: + gnmi_set(duthost, ptfhost, [], update_list, []) + except Exception as e: + logger.info("Failed to set: " + str(e)) + msg = str(e) + + assert "Unauthenticated" in msg diff --git a/tests/gnmi/test_gnmi_appldb.py b/tests/gnmi/test_gnmi_appldb.py index 145ee9e7ec..9ebd18edfe 100644 --- a/tests/gnmi/test_gnmi_appldb.py +++ b/tests/gnmi/test_gnmi_appldb.py @@ -11,7 +11,7 @@ ] -def test_gnmi_appldb_01(duthosts, rand_one_dut_hostname, localhost): +def test_gnmi_appldb_01(duthosts, rand_one_dut_hostname, ptfhost): ''' Verify GNMI native write with ApplDB Update DASH_VNET_TABLE @@ -21,29 +21,42 @@ def test_gnmi_appldb_01(duthosts, rand_one_dut_hostname, localhost): text = "{\"Vnet1\": {\"vni\": \"1000\", \"guid\": \"559c6ce8-26ab-4193-b946-ccc6e8f930b2\"}}" with open(file_name, 'w') as file: file.write(text) + ptfhost.copy(src=file_name, dest='/root') # Add DASH_VNET_TABLE - update_list = ["/sonic-db:APPL_DB/localhost/DASH_VNET_TABLE:@./%s" % (file_name)] - ret, msg = gnmi_set(duthost, localhost, [], update_list, []) - assert ret == 0, msg + update_list = ["/sonic-db:APPL_DB/localhost/DASH_VNET_TABLE:@/root/%s" % (file_name)] + gnmi_set(duthost, ptfhost, [], update_list, []) # Check gnmi_get result path_list1 = ["/sonic-db:APPL_DB/localhost/DASH_VNET_TABLE/Vnet1/vni"] path_list2 = ["/sonic-db:APPL_DB/localhost/_DASH_VNET_TABLE/Vnet1/vni"] - ret1, msg_list1 = gnmi_get(duthost, localhost, path_list1) - ret2, msg_list2 = gnmi_get(duthost, localhost, path_list2) - output = "" - if ret1 == 0: + try: + msg_list1 = gnmi_get(duthost, ptfhost, path_list1) + except Exception as e: + logger.info("Failed to read path1: " + str(e)) + else: output = msg_list1[0] - if ret2 == 0: + try: + msg_list2 = gnmi_get(duthost, ptfhost, path_list2) + except Exception as e: + logger.info("Failed to read path2: " + str(e)) + else: output = msg_list2[0] assert output == "\"1000\"", output # Remove DASH_VNET_TABLE delete_list = ["/sonic-db:APPL_DB/localhost/DASH_VNET_TABLE/Vnet1"] - ret, msg = gnmi_set(duthost, localhost, delete_list, [], []) - assert ret == 0, msg + gnmi_set(duthost, ptfhost, delete_list, [], []) # Check gnmi_get result path_list1 = ["/sonic-db:APPL_DB/localhost/DASH_VNET_TABLE/Vnet1/vni"] path_list2 = ["/sonic-db:APPL_DB/localhost/_DASH_VNET_TABLE/Vnet1/vni"] - ret1, msg_list1 = gnmi_get(duthost, localhost, path_list1) - ret2, msg_list2 = gnmi_get(duthost, localhost, path_list2) - assert ret1 != 0 and ret2 != 0, msg_list1[0] + msg_list2[0] + try: + msg_list1 = gnmi_get(duthost, ptfhost, path_list1) + except Exception as e: + logger.info("Failed to read path1: " + str(e)) + else: + pytest.fail("Remove DASH_VNET_TABLE failed: " + msg_list1[0]) + try: + msg_list2 = gnmi_get(duthost, ptfhost, path_list2) + except Exception as e: + logger.info("Failed to read path2: " + str(e)) + else: + pytest.fail("Remove DASH_VNET_TABLE failed: " + msg_list2[0]) diff --git a/tests/gnmi/test_gnmi_configdb.py b/tests/gnmi/test_gnmi_configdb.py index e8efc6a24f..53e3b47ee0 100644 --- a/tests/gnmi/test_gnmi_configdb.py +++ b/tests/gnmi/test_gnmi_configdb.py @@ -1,8 +1,13 @@ import json import logging +import multiprocessing import pytest +import re +import time from .helper import gnmi_set, gnmi_get, gnoi_reboot +from .helper import gnmi_subscribe_polling +from .helper import gnmi_subscribe_streaming_sample, gnmi_subscribe_streaming_onchange from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until from tests.common.platform.processes_utils import wait_critical_processes @@ -46,7 +51,7 @@ def get_interface_status(duthost, field, interface='Ethernet0'): return output["stdout"] -def test_gnmi_configdb_incremental_01(duthosts, rand_one_dut_hostname, localhost): +def test_gnmi_configdb_incremental_01(duthosts, rand_one_dut_hostname, ptfhost): ''' Verify GNMI native write, incremental config for configDB Toggle interface admin status @@ -55,54 +60,165 @@ def test_gnmi_configdb_incremental_01(duthosts, rand_one_dut_hostname, localhost file_name = "port.txt" interface = get_first_interface(duthost) assert interface is not None, "Invalid interface" - update_list = ["/sonic-db:CONFIG_DB/localhost/PORT/%s/admin_status:@./%s" % (interface, file_name)] + update_list = ["/sonic-db:CONFIG_DB/localhost/PORT/%s/admin_status:@/root/%s" % (interface, file_name)] path_list = ["/sonic-db:CONFIG_DB/localhost/PORT/%s/admin_status" % (interface)] # Shutdown interface text = "\"down\"" with open(file_name, 'w') as file: file.write(text) - ret, msg = gnmi_set(duthost, localhost, [], update_list, []) - assert ret == 0, msg + ptfhost.copy(src=file_name, dest='/root') + gnmi_set(duthost, ptfhost, [], update_list, []) # Check interface status and gnmi_get result status = get_interface_status(duthost, "admin_status", interface) assert status == "down", "Incremental config failed to toggle interface %s status" % interface - ret, msg_list = gnmi_get(duthost, localhost, path_list) - assert ret == 0, msg_list[0] + msg_list = gnmi_get(duthost, ptfhost, path_list) assert msg_list[0] == "\"down\"", msg_list[0] # Startup interface text = "\"up\"" with open(file_name, 'w') as file: file.write(text) - ret, msg = gnmi_set(duthost, localhost, [], update_list, []) - assert ret == 0, msg + ptfhost.copy(src=file_name, dest='/root') + gnmi_set(duthost, ptfhost, [], update_list, []) # Check interface status and gnmi_get result status = get_interface_status(duthost, "admin_status", interface) assert status == "up", "Incremental config failed to toggle interface %s status" % interface - ret, msg_list = gnmi_get(duthost, localhost, path_list) - assert ret == 0, msg_list[0] + msg_list = gnmi_get(duthost, ptfhost, path_list) assert msg_list[0] == "\"up\"", msg_list[0] -def test_gnmi_configdb_incremental_02(duthosts, rand_one_dut_hostname, localhost): +def test_gnmi_configdb_incremental_02(duthosts, rand_one_dut_hostname, ptfhost): ''' Verify GNMI native write, incremental config for configDB GNMI set request with invalid path ''' duthost = duthosts[rand_one_dut_hostname] file_name = "port.txt" - update_list = ["/sonic-db:CONFIG_DB/localhost/PORTABC/Ethernet100/admin_status:@./%s" % (file_name)] + update_list = ["/sonic-db:CONFIG_DB/localhost/PORTABC/Ethernet100/admin_status:@/root/%s" % (file_name)] # GNMI set request with invalid path text = "\"down\"" with open(file_name, 'w') as file: file.write(text) - ret, msg = gnmi_set(duthost, localhost, [], update_list, []) - assert ret != 0, msg + ptfhost.copy(src=file_name, dest='/root') + try: + gnmi_set(duthost, ptfhost, [], update_list, []) + except Exception as e: + logger.info("Incremental config failed: " + str(e)) + else: + pytest.fail("Set request with invalid path") -def test_gnmi_configdb_full_01(duthosts, rand_one_dut_hostname, localhost): +test_data_metadata = [ + { + "name": "Subscribe table for DEVICE_METADATA", + "path": "/sonic-db:CONFIG_DB/localhost/DEVICE_METADATA" + }, + { + "name": "Subscribe table key for DEVICE_METADATA", + "path": "/sonic-db:CONFIG_DB/localhost/DEVICE_METADATA/localhost" + }, + { + "name": "Subscribe table field for DEVICE_METADATA", + "path": "/sonic-db:CONFIG_DB/localhost/DEVICE_METADATA/localhost/bgp_asn" + } +] + + +@pytest.mark.parametrize('test_data', test_data_metadata) +def test_gnmi_configdb_polling_01(duthosts, rand_one_dut_hostname, ptfhost, test_data): + ''' + Verify GNMI subscribe API, streaming onchange mode + Subscribe polling mode + ''' + duthost = duthosts[rand_one_dut_hostname] + exp_cnt = 3 + path_list = [test_data["path"]] + msg, _ = gnmi_subscribe_polling(duthost, ptfhost, path_list, 1000, exp_cnt) + assert msg.count("bgp_asn") >= exp_cnt, test_data["name"] + ": " + msg + + +@pytest.mark.parametrize('test_data', test_data_metadata) +def test_gnmi_configdb_streaming_sample_01(duthosts, rand_one_dut_hostname, ptfhost, test_data): + ''' + Verify GNMI subscribe API, streaming onchange mode + Subscribe streaming sample mode + ''' + duthost = duthosts[rand_one_dut_hostname] + exp_cnt = 5 + path_list = [test_data["path"]] + msg, _ = gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, 0, exp_cnt) + assert msg.count("bgp_asn") >= exp_cnt, test_data["name"] + ": " + msg + + +@pytest.mark.parametrize('test_data', test_data_metadata) +def test_gnmi_configdb_streaming_onchange_01(duthosts, rand_one_dut_hostname, ptfhost, test_data): + ''' + Verify GNMI subscribe API, streaming onchange mode + Subscribe streaming onchange mode + ''' + duthost = duthosts[rand_one_dut_hostname] + run_flag = multiprocessing.Value('I', True) + + # Update DEVICE_METADATA table to trigger onchange event + def worker(duthost, run_flag): + for i in range(100): + if not run_flag.value: + break + time.sleep(0.5) + cmd = "sonic-db-cli CONFIG_DB hdel \"DEVICE_METADATA|localhost\" bgp_asn " + duthost.shell(cmd, module_ignore_errors=True) + time.sleep(0.5) + cmd = "sonic-db-cli CONFIG_DB hset \"DEVICE_METADATA|localhost\" bgp_asn " + str(i+1000) + duthost.shell(cmd, module_ignore_errors=True) + + client_task = multiprocessing.Process(target=worker, args=(duthost, run_flag,)) + client_task.start() + exp_cnt = 5 + path_list = [test_data["path"]] + msg, _ = gnmi_subscribe_streaming_onchange(duthost, ptfhost, path_list, exp_cnt*2) + run_flag.value = False + client_task.join() + assert msg.count("bgp_asn") >= exp_cnt, test_data["name"] + ": " + msg + + +def test_gnmi_configdb_streaming_onchange_02(duthosts, rand_one_dut_hostname, ptfhost): + ''' + Verify GNMI subscribe API, streaming onchange mode + Subscribe table, and verify gnmi output has table key + ''' + duthost = duthosts[rand_one_dut_hostname] + run_flag = multiprocessing.Value('I', True) + + # Update DEVICE_METADATA table to trigger onchange event + def worker(duthost, run_flag): + for i in range(100): + if not run_flag.value: + break + time.sleep(0.5) + cmd = "sonic-db-cli CONFIG_DB hset \"DEVICE_METADATA|localhost\" bgp_asn " + str(i+1000) + duthost.shell(cmd, module_ignore_errors=True) + + client_task = multiprocessing.Process(target=worker, args=(duthost, run_flag,)) + client_task.start() + exp_cnt = 3 + path_list = ["/sonic-db:CONFIG_DB/localhost/DEVICE_METADATA"] + msg, _ = gnmi_subscribe_streaming_onchange(duthost, ptfhost, path_list, exp_cnt) + run_flag.value = False + client_task.join() + + match_list = re.findall("json_ietf_val: \"({.*?})\"", msg) + assert len(match_list) >= exp_cnt, "Missing json_ietf_val in gnmi response: " + msg + for match in match_list: + result = json.loads(match) + # Verify table key + assert "localhost" in result, "Invalid result: " + match + # Verify table field + assert "bgp_asn" in result["localhost"], "Invalid result: " + match + + +def test_gnmi_configdb_full_01(duthosts, rand_one_dut_hostname, ptfhost): ''' Verify GNMI native write, full config for configDB Toggle interface admin status @@ -122,15 +238,15 @@ def test_gnmi_configdb_full_01(duthosts, rand_one_dut_hostname, localhost): filename = "full.txt" with open(filename, 'w') as file: json.dump(dic, file) + ptfhost.copy(src=filename, dest='/root') delete_list = ["/sonic-db:CONFIG_DB/localhost/"] - update_list = ["/sonic-db:CONFIG_DB/localhost/:@%s" % filename] - ret, msg = gnmi_set(duthost, localhost, delete_list, update_list, []) - assert ret == 0, msg + update_list = ["/sonic-db:CONFIG_DB/localhost/:@/root/%s" % filename] + gnmi_set(duthost, ptfhost, delete_list, update_list, []) # Check interface status and gnmi_get result status = get_interface_status(duthost, "admin_status", interface) assert status == "up", "Port status is changed" # GNOI reboot - ret, msg = gnoi_reboot(duthost, localhost, 0, 0, "abc") + gnoi_reboot(duthost, 0, 0, "abc") pytest_assert( wait_until(600, 10, 0, duthost.critical_services_fully_started), "All critical services should be fully started!") diff --git a/tests/gnmi/test_gnmi_countersdb.py b/tests/gnmi/test_gnmi_countersdb.py new file mode 100644 index 0000000000..182737b20d --- /dev/null +++ b/tests/gnmi/test_gnmi_countersdb.py @@ -0,0 +1,164 @@ +import logging +import pytest +import re + +from .helper import gnmi_get, gnmi_subscribe_polling, gnmi_subscribe_streaming_sample +from tests.common.helpers.assertions import pytest_assert + + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('any'), + pytest.mark.disable_loganalyzer +] + + +def test_gnmi_queue_buffer_cnt(duthosts, rand_one_dut_hostname, ptfhost): + """ + Check number of queue counters + """ + duthost = duthosts[rand_one_dut_hostname] + if duthost.is_supervisor_node(): + pytest.skip("Skipping test as no Ethernet0 frontpanel port on supervisor") + logger.info('start gnmi output testing') + iface = "Ethernet0" + # Get UC for Ethernet0 + dut_command = "show queue counters %s" % iface + result = duthost.shell(dut_command, module_ignore_errors=True) + uc_list = re.findall(r"UC(\d+)", result["stdout"]) + for i in uc_list: + # Read UC + path_list = ["/sonic-db:COUNTERS_DB/localhost/COUNTERS_QUEUE_NAME_MAP/" + iface + ":" + str(i)] + msg_list = gnmi_get(duthost, ptfhost, path_list) + result = msg_list[0] + pytest_assert("oid" in result, result) + # Read invalid UC + path_list = ["/sonic-db:COUNTERS_DB/localhost/COUNTERS_QUEUE_NAME_MAP/" + iface + ":abc"] + try: + msg_list = gnmi_get(duthost, ptfhost, path_list) + except Exception as e: + assert "GRPC error" in str(e), str(e) + else: + pytest.fail("Should fail for invalid path: " + path_list[0]) + + +def test_gnmi_output(duthosts, rand_one_dut_hostname, ptfhost): + """ + Read COUNTERS table + Get table key from COUNTERS_PORT_NAME_MAP + """ + duthost = duthosts[rand_one_dut_hostname] + if duthost.is_supervisor_node(): + pytest.skip("Skipping test as no Ethernet0 frontpanel port on supervisor") + logger.info('start gnmi output testing') + # Get COUNTERS table key for Ethernet0 + dut_command = "sonic-db-cli COUNTERS_DB hget COUNTERS_PORT_NAME_MAP Ethernet0" + result = duthost.shell(dut_command, module_ignore_errors=True) + counter_key = result['stdout'].strip() + assert "oid" in counter_key, "Invalid oid: " + counter_key + path_list = ["/sonic-db:COUNTERS_DB/localhost/COUNTERS/" + counter_key] + msg_list = gnmi_get(duthost, ptfhost, path_list) + result = msg_list[0] + logger.info("GNMI Server output") + logger.info(result) + pytest_assert("SAI_PORT_STAT_IF_IN_ERRORS" in result, + "SAI_PORT_STAT_IF_IN_ERRORS not found in gnmi_output: " + result) + + +test_data_counters_port_name_map = [ + { + "name": "Subscribe table for COUNTERS_PORT_NAME_MAP", + "path": "/sonic-db:COUNTERS_DB/localhost/COUNTERS_PORT_NAME_MAP" + }, + { + "name": "Subscribe table field for COUNTERS_PORT_NAME_MAP", + "path": "/sonic-db:COUNTERS_DB/localhost/COUNTERS_PORT_NAME_MAP/Ethernet0" + } +] + + +@pytest.mark.parametrize('test_data', test_data_counters_port_name_map) +def test_gnmi_counterdb_polling_01(duthosts, rand_one_dut_hostname, ptfhost, test_data): + ''' + Verify GNMI subscribe API + Subscribe polling mode for COUNTERS_PORT_NAME_MAP + ''' + duthost = duthosts[rand_one_dut_hostname] + if duthost.is_supervisor_node(): + pytest.skip("Skipping test as no Ethernet0 frontpanel port on supervisor") + exp_cnt = 3 + path_list = [test_data["path"]] + msg, _ = gnmi_subscribe_polling(duthost, ptfhost, path_list, 1000, exp_cnt) + assert msg.count("oid") >= exp_cnt, test_data["name"] + ": " + msg + + +def test_gnmi_counterdb_polling_02(duthosts, rand_one_dut_hostname, ptfhost): + ''' + Verify GNMI subscribe API + Subscribe polling mode for COUNTERS + ''' + duthost = duthosts[rand_one_dut_hostname] + if duthost.is_supervisor_node(): + pytest.skip("Skipping test as no Ethernet0 frontpanel port on supervisor") + exp_cnt = 3 + # Get COUNTERS table key for Ethernet0 + dut_command = "sonic-db-cli COUNTERS_DB hget COUNTERS_PORT_NAME_MAP Ethernet0" + result = duthost.shell(dut_command, module_ignore_errors=True) + counter_key = result['stdout'].strip() + assert "oid" in counter_key, "Invalid oid: " + counter_key + # Subscribe table + path_list = ["/sonic-db:COUNTERS_DB/localhost/COUNTERS/"] + msg, _ = gnmi_subscribe_polling(duthost, ptfhost, path_list, 1000, exp_cnt) + assert msg.count("SAI_PORT_STAT_IF_IN_ERRORS") >= exp_cnt, msg + # Subscribe table key + path_list = ["/sonic-db:COUNTERS_DB/localhost/COUNTERS/" + counter_key] + msg, _ = gnmi_subscribe_polling(duthost, ptfhost, path_list, 1000, exp_cnt) + assert msg.count("SAI_PORT_STAT_IF_IN_ERRORS") >= exp_cnt, msg + # Subscribe table field + path_list = ["/sonic-db:COUNTERS_DB/localhost/COUNTERS/" + counter_key + "/SAI_PORT_STAT_IF_IN_ERRORS"] + msg, _ = gnmi_subscribe_polling(duthost, ptfhost, path_list, 1000, exp_cnt) + assert msg.count("SAI_PORT_STAT_IF_IN_ERRORS") >= exp_cnt, msg + + +@pytest.mark.parametrize('test_data', test_data_counters_port_name_map) +def test_gnmi_counterdb_streaming_sample_01(duthosts, rand_one_dut_hostname, ptfhost, test_data): + ''' + Verify GNMI subscribe API + Subscribe streaming sample mode for COUNTERS_PORT_NAME_MAP + ''' + duthost = duthosts[rand_one_dut_hostname] + if duthost.is_supervisor_node(): + pytest.skip("Skipping test as no Ethernet0 frontpanel port on supervisor") + exp_cnt = 3 + path_list = [test_data["path"]] + msg, _ = gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, 0, exp_cnt) + assert msg.count("oid") >= exp_cnt, test_data["name"] + ": " + msg + + +def test_gnmi_counterdb_streaming_sample_02(duthosts, rand_one_dut_hostname, ptfhost): + ''' + Verify GNMI subscribe API + Subscribe streaming sample mode for COUNTERS + ''' + duthost = duthosts[rand_one_dut_hostname] + if duthost.is_supervisor_node(): + pytest.skip("Skipping test as no Ethernet0 frontpanel port on supervisor") + exp_cnt = 3 + # Get COUNTERS table key for Ethernet0 + dut_command = "sonic-db-cli COUNTERS_DB hget COUNTERS_PORT_NAME_MAP Ethernet0" + result = duthost.shell(dut_command, module_ignore_errors=True) + counter_key = result['stdout'].strip() + assert "oid" in counter_key, "Invalid oid: " + counter_key + # Subscribe table + path_list = ["/sonic-db:COUNTERS_DB/localhost/COUNTERS/"] + msg, _ = gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, 0, exp_cnt) + assert msg.count("SAI_PORT_STAT_IF_IN_ERRORS") >= exp_cnt, msg + # Subscribe table key + path_list = ["/sonic-db:COUNTERS_DB/localhost/COUNTERS/" + counter_key] + msg, _ = gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, 0, exp_cnt) + assert msg.count("SAI_PORT_STAT_IF_IN_ERRORS") >= exp_cnt, msg + # Subscribe table field + path_list = ["/sonic-db:COUNTERS_DB/localhost/COUNTERS/" + counter_key + "/SAI_PORT_STAT_IF_IN_ERRORS"] + msg, _ = gnmi_subscribe_streaming_sample(duthost, ptfhost, path_list, 0, exp_cnt) + assert msg.count("SAI_PORT_STAT_IF_IN_ERRORS") >= exp_cnt, msg diff --git a/tests/gnmi/test_gnoi_killprocess.py b/tests/gnmi/test_gnoi_killprocess.py new file mode 100644 index 0000000000..fabdf28165 --- /dev/null +++ b/tests/gnmi/test_gnoi_killprocess.py @@ -0,0 +1,79 @@ +import pytest +from .helper import gnoi_request +from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.dut_utils import is_container_running + + +# This test ensures functionality of KillProcess API to kill and restart a process when a valid process name is passed +# When an invalid process name is passed, this test ensures that the expected error is returned +@pytest.mark.parametrize("process,is_valid, expected_msg", [ + ("gnmi", False, "Dbus does not support gnmi service management"), + ("nonexistent", False, "Dbus does not support nonexistent service management"), + ("", False, "Dbus stop_service called with no service specified"), + ("snmp", True, ""), + ("dhcp_relay", True, ""), + ("radv", True, ""), + ("restapi", True, ""), + ("lldp", True, ""), + ("sshd", True, ""), + ("swss", True, ""), + ("pmon", True, ""), + ("rsyslog", True, ""), + ("telemetry", True, "") +]) +def test_gnoi_killprocess_then_restart(duthosts, rand_one_dut_hostname, localhost, process, is_valid, expected_msg): + duthost = duthosts[rand_one_dut_hostname] + + if process and process != "nonexistent": + pytest_assert(duthost.is_host_service_running(process), + "{} should be running before KillProcess test attempts to kill this process".format(process)) + + request_kill_json_data = '{{"name": "{}", "signal": 1}}'.format(process) + ret, msg = gnoi_request(duthost, localhost, "KillProcess", request_kill_json_data) + if is_valid: + pytest_assert(ret == 0, "KillProcess API unexpectedly reported failure") + pytest_assert(not is_container_running(duthost, process), + "{} found running after KillProcess reported success".format(process)) + + request_restart_json_data = '{{"name": "{}", "restart": true, "signal": 1}}'.format(process) + ret, msg = gnoi_request(duthost, localhost, "KillProcess", request_restart_json_data) + pytest_assert(ret == 0, + "KillProcess API unexpectedly reported failure when attempting to restart {}".format(process)) + pytest_assert(duthost.is_host_service_running(process), + "{} not running after KillProcess reported successful restart".format(process)) + else: + pytest_assert(ret != 0, "KillProcess API unexpectedly succeeded with invalid request parameters") + pytest_assert(expected_msg in msg, "Unexpected error message in response to invalid gNOI request") + + pytest_assert(duthost.critical_services_fully_started, "System unhealthy after gNOI API request") + + +# This test performs additional verification of the restart request under KillProcess API +# This test focuses on edge conditions of restart value in the request, so we only test against one service: snmp +@pytest.mark.parametrize("request_restart_value, is_valid", [ + ("invalid", False), + ("", False) +]) +def test_gnoi_killprocess_restart(duthosts, rand_one_dut_hostname, localhost, request_restart_value, is_valid): + duthost = duthosts[rand_one_dut_hostname] + request_json_data = f'{{"name": "snmp", "restart": {request_restart_value}, "signal": 1}}' + ret, msg = gnoi_request(duthost, localhost, "KillProcess", request_json_data) + if is_valid: + pytest_assert(ret == 0, "KillProcess API unexpectedly reported failure") + pytest_assert(is_container_running(duthost, "snmp"), + "snmp not running after KillProcess API reported successful restart") + else: + pytest_assert(ret != 0, "KillProcess API unexpectedly succeeded with invalid request parameters") + pytest_assert("panic" in msg, "Unexpected error message in response to invalid gNOI request") + pytest_assert(duthost.critical_services_fully_started, "System unhealthy after gNOI API request") + + +def test_invalid_signal(duthosts, rand_one_dut_hostname, localhost): + duthost = duthosts[rand_one_dut_hostname] + request_json_data = '{"name": "snmp", "restart": true, "signal": 2}' + ret, msg = gnoi_request(duthost, localhost, "KillProcess", request_json_data) + + pytest_assert(ret != 0, "KillProcess API unexpectedly succeeded with invalid request parameters") + pytest_assert("KillProcess only supports SIGNAL_TERM (option 1)" in msg, + "Unexpected error message in response to invalid gNOI request") + pytest_assert(duthost.critical_services_fully_started, "System unhealthy after gNOI API request") diff --git a/tests/golden_config_infra/templates/sample_golden_config_db.j2 b/tests/golden_config_infra/templates/sample_golden_config_db.j2 new file mode 100644 index 0000000000..07a119dd2e --- /dev/null +++ b/tests/golden_config_infra/templates/sample_golden_config_db.j2 @@ -0,0 +1,16 @@ +{% set portchannels= [] %} +{% for pc, value in PORTCHANNEL.items() %} + {% set _ = portchannels.append(pc) %} +{% endfor %} + +{ + "NEW_FEATURE": { + "test_entry": { + "platform": "{{ DEVICE_METADATA.localhost.platform }}" + {% if portchannels %} + , + "ports": "{{ portchannels | join(',') }}" + {% endif %} + } + } +} diff --git a/tests/golden_config_infra/test_config_reload_with_rendered_golden_config.py b/tests/golden_config_infra/test_config_reload_with_rendered_golden_config.py new file mode 100644 index 0000000000..0cc7d7e0ca --- /dev/null +++ b/tests/golden_config_infra/test_config_reload_with_rendered_golden_config.py @@ -0,0 +1,123 @@ +import pytest +import logging +import json +import os + +from tests.common.helpers.assertions import pytest_assert +from tests.common.config_reload import config_reload, config_reload_minigraph_with_rendered_golden_config_override +from tests.common.utilities import backup_config, restore_config, get_running_config, \ + compare_dicts_ignore_list_order, NON_USER_CONFIG_TABLES +from tests.common.utilities import update_pfcwd_default_state + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('any'), + pytest.mark.disable_loganalyzer, +] + +GOLDEN_CONFIG = "/etc/sonic/golden_config_db.json" +GOLDEN_CONFIG_BACKUP = "/etc/sonic/golden_config_db.json_before_override" +CONFIG_DB = "/etc/sonic/config_db.json" +CONFIG_DB_BACKUP = "/etc/sonic/config_db.json_before_override" + + +def file_exists_on_dut(duthost, filename): + return duthost.stat(path=filename).get('stat', {}).get('exists', False) + + +@pytest.fixture(scope="module") +def setup_env(duthosts, rand_one_dut_hostname, tbinfo): + """ + Setup/teardown + Args: + duthost: DUT. + golden_config_exists_on_dut: Check if golden config exists on DUT. + """ + duthost = duthosts[rand_one_dut_hostname] + if duthost.is_multi_asic: + pytest.skip("Skip test on multi-asic platforms as it is designed for single asic.") + + topo_type = tbinfo["topo"]["type"] + if topo_type in ["m0", "mx"]: + original_pfcwd_value = update_pfcwd_default_state(duthost, "/etc/sonic/init_cfg.json", "disable") + + # Backup configDB + backup_config(duthost, CONFIG_DB, CONFIG_DB_BACKUP) + if file_exists_on_dut(duthost, GOLDEN_CONFIG): + backup_config(duthost, GOLDEN_CONFIG, GOLDEN_CONFIG_BACKUP) + + # Reload test env with minigraph + config_reload(duthost, safe_reload=True, check_intf_up_ports=True) + + yield + + if topo_type in ["m0", "mx"]: + update_pfcwd_default_state(duthost, "/etc/sonic/init_cfg.json", original_pfcwd_value) + + # Restore configDB after test. + restore_config(duthost, CONFIG_DB, CONFIG_DB_BACKUP) + if file_exists_on_dut(duthost, GOLDEN_CONFIG_BACKUP): + restore_config(duthost, GOLDEN_CONFIG, GOLDEN_CONFIG_BACKUP) + else: + duthost.file(path=GOLDEN_CONFIG, state='absent') + + # Restore config before test + config_reload(duthost) + + +def config_compare(golden_config, running_config): + for table in golden_config: + if table in NON_USER_CONFIG_TABLES: + continue + + if table == "ACL_TABLE": + pytest_assert( + compare_dicts_ignore_list_order(golden_config[table], running_config[table]), + "ACL_TABLE compare fail!" + ) + else: + pytest_assert( + golden_config[table] == running_config[table], + "Table compare fail! {}".format(table) + ) + + +def golden_config_override_with_general_template(duthost): + # This is to copy and parse the default template: tests/common/templates/golden_config_db.j2 + config_reload_minigraph_with_rendered_golden_config_override( + duthost, safe_reload=True, check_intf_up_ports=True + ) + overrided_config = get_running_config(duthost) + golden_config = json.loads( + duthost.shell("cat /etc/sonic/golden_config_db.json")['stdout'] + ) + + config_compare(golden_config, overrided_config) + + +def golden_config_override_with_specific_template(duthost): + # This is to copy and parse the template: tests/golden_config_infra/templates/sample_golden_config_db.j2 + base_dir = os.path.dirname(os.path.realpath(__file__)) + template_dir = os.path.join(base_dir, 'templates') + golden_config_j2 = os.path.join(template_dir, 'sample_golden_config_db.j2') + config_reload_minigraph_with_rendered_golden_config_override( + duthost, safe_reload=True, check_intf_up_ports=True, + local_golden_config_template=golden_config_j2 + ) + overrided_config = get_running_config(duthost) + golden_config = json.loads( + duthost.shell("cat /etc/sonic/golden_config_db.json")['stdout'] + ) + + config_compare(golden_config, overrided_config) + + +def test_rendered_golden_config_override(duthosts, rand_one_dut_hostname, setup_env): + duthost = duthosts[rand_one_dut_hostname] + if duthost.is_multi_asic: + pytest.skip("Skip this test on multi-asic platforms, \ + since golden config format here is not compatible with multi-asics") + + golden_config_override_with_general_template(duthost) + golden_config_override_with_specific_template(duthost) diff --git a/tests/hash/conftest.py b/tests/hash/conftest.py new file mode 100644 index 0000000000..c13cfdf16b --- /dev/null +++ b/tests/hash/conftest.py @@ -0,0 +1,23 @@ +""" + Pytest configuration used by the read generic hash tests. +""" + + +def pytest_addoption(parser): + parser.addoption("--algorithm", action="store", default="random", + help="The hash algorithm to test, can be 'all', 'random' or a designated one or algorithms " + "separated by comma, such as 'CRC,CRC_CCITT'") + parser.addoption("--hash_field", action="store", default="random", + help="The hash field to test, can be 'all', 'random' or a designated one or hash fields " + "separated by comma, such as 'SRC_IP,DST_IP,L4_SRC_PORT'") + parser.addoption("--ip_version", action="store", default="random", choices=('all', 'random', 'ipv4', 'ipv6'), + help="The outer ip version to test.") + parser.addoption("--inner_ip_version", action="store", default="random", choices=('all', 'random', 'ipv4', 'ipv6'), + help="The inner ip version to test, only needed when hash field is an inner field.") + parser.addoption("--encap_type", action="store", default="random", + choices=('random', 'all', 'ipinip', 'vxlan', 'nvgre'), + help="The encapsulation type for the inner fields, " + "only needed when hash field is an inner field.") + parser.addoption("--reboot", action="store", default="random", + choices=('random', 'all', 'cold', 'fast', 'warm', 'reload'), + help="The reboot type for the reboot test, only needed for the reboot test case.") diff --git a/tests/hash/generic_hash_helper.py b/tests/hash/generic_hash_helper.py new file mode 100644 index 0000000000..8e98e558f5 --- /dev/null +++ b/tests/hash/generic_hash_helper.py @@ -0,0 +1,723 @@ +import random +import json +import time +import logging +import pytest +import ipaddress + +from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import wait_until +from tests.common import config_reload +from tests.conftest import get_testbed_metadata +from tests.common.vxlan_ecmp_utils import Ecmp_Utils as VxLAN_Ecmp_Utils +from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports # noqa:F401 +from tests.common.dualtor.constants import UPPER_TOR + +SRC_IP_RANGE = ['8.0.0.0', '8.255.255.255'] +DST_IP_RANGE = ['9.0.0.0', '9.255.255.255'] +SRC_IPV6_RANGE = ['20D0:A800:0:00::', '20D0:FFFF:0:00::FFFF'] +DST_IPV6_RANGE = ['20D0:A800:0:01::', '20D0:FFFF:0:01::FFFF'] +IP_RANGE = {'ipv4': {'src': SRC_IP_RANGE, 'dst': DST_IP_RANGE}, + 'ipv6': {'src': SRC_IPV6_RANGE, 'dst': DST_IPV6_RANGE}, + 'None': {'src': [], 'dst': []}} +PTF_QLEN = 20000 +VLAN_RANGE = [1032, 1060] +ETHERTYPE_RANGE = [0x0800, 0x0900] +ENCAPSULATION = ['ipinip', 'vxlan', 'nvgre'] +MELLANOX_SUPPORTED_HASH_ALGORITHM = ['CRC', 'CRC_CCITT'] +DEFAULT_SUPPORTED_HASH_ALGORITHM = ['CRC', 'CRC_CCITT', 'RANDOM', 'XOR'] + +MELLANOX_ECMP_HASH_FIELDS = [ + 'IN_PORT', 'SRC_MAC', 'DST_MAC', 'ETHERTYPE', 'VLAN_ID', 'IP_PROTOCOL', 'SRC_IP', 'DST_IP', 'L4_SRC_PORT', + 'L4_DST_PORT', 'INNER_SRC_IP', 'INNER_DST_IP', 'INNER_IP_PROTOCOL', 'INNER_ETHERTYPE', 'INNER_L4_SRC_PORT', + 'INNER_L4_DST_PORT', 'INNER_SRC_MAC', 'INNER_DST_MAC' +] +MELLANOX_LAG_HASH_FIELDS = [ + 'IN_PORT', 'SRC_MAC', 'DST_MAC', 'ETHERTYPE', 'VLAN_ID', 'IP_PROTOCOL', 'SRC_IP', 'DST_IP', 'L4_SRC_PORT', + 'L4_DST_PORT', 'INNER_SRC_IP', 'INNER_DST_IP', 'INNER_IP_PROTOCOL', 'INNER_ETHERTYPE', 'INNER_L4_SRC_PORT', + 'INNER_L4_DST_PORT', 'INNER_SRC_MAC', 'INNER_DST_MAC' +] +DEFAULT_ECMP_HASH_FIELDS = [ + 'IN_PORT', 'SRC_MAC', 'DST_MAC', 'ETHERTYPE', 'VLAN_ID', 'IP_PROTOCOL', 'SRC_IP', 'DST_IP', 'L4_SRC_PORT', + 'L4_DST_PORT', 'INNER_SRC_IP', 'INNER_DST_IP', 'INNER_IP_PROTOCOL', 'INNER_ETHERTYPE', 'INNER_L4_SRC_PORT', + 'INNER_L4_DST_PORT', 'INNER_SRC_MAC', 'INNER_DST_MAC' +] +DEFAULT_LAG_HASH_FIELDS = [ + 'IN_PORT', 'SRC_MAC', 'DST_MAC', 'ETHERTYPE', 'VLAN_ID', 'IP_PROTOCOL', 'SRC_IP', 'DST_IP', 'L4_SRC_PORT', + 'L4_DST_PORT', 'INNER_SRC_IP', 'INNER_DST_IP', 'INNER_IP_PROTOCOL', 'INNER_ETHERTYPE', 'INNER_L4_SRC_PORT', + 'INNER_L4_DST_PORT', 'INNER_SRC_MAC', 'INNER_DST_MAC' +] +HASH_CAPABILITIES = {'mellanox': {'ecmp': MELLANOX_ECMP_HASH_FIELDS, + 'lag': MELLANOX_LAG_HASH_FIELDS}, + 'default': {'ecmp': DEFAULT_ECMP_HASH_FIELDS, + 'lag': DEFAULT_LAG_HASH_FIELDS}} + +logger = logging.getLogger(__name__) +vlan_member_to_restore = {} +ip_interface_to_restore = [] +l2_ports = set() +vlans_to_remove = [] +interfaces_to_startup = [] +balancing_test_times = 240 +balancing_range = 0.25 +vxlan_ecmp_utils = VxLAN_Ecmp_Utils() +vxlan_port_list = [13330, 4789] +restore_vxlan = False + + +@pytest.fixture(scope="module") +def get_supported_hash_algorithms(request): + asic_type = get_asic_type(request) + if asic_type in 'mellanox': + supported_hash_algorithm_list = MELLANOX_SUPPORTED_HASH_ALGORITHM[:] + else: + supported_hash_algorithm_list = DEFAULT_SUPPORTED_HASH_ALGORITHM[:] + return supported_hash_algorithm_list + + +@pytest.fixture(scope="module", autouse=True) +def skip_vs_setups(duthost): + """ Fixture to skip the test on vs setups. """ + if duthost.facts['asic_type'] in ["vs"]: + pytest.skip("Generic hash test only runs on physical setups.") + + +@pytest.fixture(scope="module") +def mg_facts(duthost, tbinfo): + """ Fixture to get the extended minigraph facts """ + mg_facts = duthost.get_extended_minigraph_facts(tbinfo) + return mg_facts + + +@pytest.fixture(scope='function', autouse=True) +def restore_init_hash_config(duthost): + """ Fixture to restore the initial generic hash configurations after the test. """ + logger.info("Store the initial generic hash configurations") + init_ecmp_hash_fields, init_ecmp_hash_algo, init_lag_hash_fields, init_lag_hash_algo = \ + get_global_hash_config(duthost) + yield + if init_ecmp_hash_fields: + duthost.set_switch_hash_global('ecmp', init_ecmp_hash_fields) + if init_lag_hash_fields: + duthost.set_switch_hash_global('lag', init_lag_hash_fields) + if init_ecmp_hash_algo and init_ecmp_hash_algo != 'N/A': + duthost.set_switch_hash_global_algorithm('ecmp', init_ecmp_hash_algo) + if init_lag_hash_algo and init_lag_hash_algo != 'N/A': + duthost.set_switch_hash_global_algorithm('lag', init_lag_hash_algo) + logger.info("The initial generic hash configurations have been restored.") + + +@pytest.fixture(scope='function') +def reload(duthost): + """ Fixture to do the config reload after the test. """ + yield + config_reload(duthost, safe_reload=True) + + +@pytest.fixture(scope='function') +def restore_configuration(duthost): + """ Fixture to restore the interface and vlan configurations after the L2 test. + The configurations are restored from the global variables. """ + + yield + try: + logger.info("Restore the interface and vlan configurations after the L2 test.") + # Remove vlans + for vlan in vlans_to_remove: + for interface in l2_ports: + duthost.shell(f'config vlan member del {vlan} {interface}') + duthost.shell(f'config vlan del {vlan}') + # Re-config ip interface + for ip_interface in ip_interface_to_restore: + duthost.shell(f"config interface ip add {ip_interface['attachto']} " + f"{ip_interface['addr']}/{ip_interface['mask']}") + # Re-config vlan interface + if vlan_member_to_restore: + duthost.shell(f"config vlan member add {vlan_member_to_restore['vlan_id']} " + f"{vlan_member_to_restore['interface']} --untagged") + except Exception as err: + config_reload(duthost, safe_reload=True) + logger.info("Exception occurred when restoring the configuration.") + raise err + finally: + del ip_interface_to_restore[:] + del vlans_to_remove[:] + vlan_member_to_restore.clear() + l2_ports.clear() + + +@pytest.fixture(scope='function') +def restore_interfaces(duthost): + """ Fixture to startup interfaces after the flap test in case the test fails and some + interfaces are shutdown during the test. The interfaces to start are from a global variable """ + + yield + logger.info("Startup the interfaces which were shutdown during the test") + if interfaces_to_startup: + duthost.no_shutdown_multiple(interfaces_to_startup) + try: + for interface in interfaces_to_startup: + pytest_assert(wait_until(30, 5, 0, duthost.check_intf_link_state, interface), + "Not all interfaces are restored to up after the flap test.") + finally: + del interfaces_to_startup[:] + + +@pytest.fixture(scope='function') +def restore_vxlan_port(duthost): + """ Fixture to restore the vxlan port to default 4789 """ + global restore_vxlan + yield + if restore_vxlan: + vxlan_ecmp_utils.Constants['DEBUG'] = False + vxlan_ecmp_utils.Constants['KEEP_TEMP_FILES'] = False + vxlan_ecmp_utils.configure_vxlan_switch(duthost, 4789, duthost.facts['router_mac']) + restore_vxlan = False + + +@pytest.fixture(scope='module') +def global_hash_capabilities(duthost): + """ + Get the generic hash capabilities. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + Returns: + ecmp_hash_fields: a list of supported ecmp hash fields + lag_hash_fields: a list of supported lag hash fields + """ + global_hash_capabilities = duthost.get_switch_hash_capabilities() + return {'ecmp': global_hash_capabilities['ecmp'], 'ecmp_algo': global_hash_capabilities['ecmp_algo'], + 'lag': global_hash_capabilities['lag'], 'lag_algo': global_hash_capabilities['lag_algo']} + + +@pytest.fixture() +def toggle_all_simulator_ports_to_upper_tor(toggle_all_simulator_ports): # noqa:F811 + """ Fixture to toggle all ports to upper tor """ + toggle_all_simulator_ports(UPPER_TOR) + + +def get_global_hash_config(duthost): + """ + Get the generic hash configurations. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + Returns: + ecmp_hash_fields: a list of currently configured ecmp hash fields + lag_hash_fields: a list of currently configured lag hash fields + """ + logger.info("Get current generic hash configurations.") + global_hash_config = duthost.get_switch_hash_configurations() + ecmp_hash_fields = global_hash_config['ecmp'] + lag_hash_fields = global_hash_config['lag'] + ecmp_hash_algo = global_hash_config['ecmp_algo'] + lag_hash_algo = global_hash_config['lag_algo'] + return ecmp_hash_fields, ecmp_hash_algo, lag_hash_fields, lag_hash_algo + + +def check_global_hash_config(duthost, ecmp_hash_fields, lag_hash_fields): + """ + Validate if the current generic hash configurations are as expected. Assert when validation fails. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ecmp_hash_fields: a list of expected ecmp hash fields + lag_hash_fields: a list of expected lag hash fields + """ + ecmp_hash_fields_fact, _, lag_hash_fields_fact, _ = get_global_hash_config(duthost) + ecmp_hash_matched = set(ecmp_hash_fields) == set(ecmp_hash_fields_fact) + lag_hash_matched = set(lag_hash_fields) == set(lag_hash_fields_fact) + pytest_assert(ecmp_hash_matched == lag_hash_matched is True, + 'The global hash configuration is not as expected:\n' + f'expected ecmp hash fields: {ecmp_hash_fields}\n' + f'actual ecmp hash fields: {ecmp_hash_fields_fact}\n' + f'expected lag hash fields: {lag_hash_fields}\n' + f'actual lag hash fields: {lag_hash_fields_fact}') + + +def check_global_hash_algorithm(duthost, ecmp_hash_algo=None, lag_hash_algo=None): + """ + Validate if the current generic hash algorithm configurations are as expected. Assert when validation fails. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ecmp_hash_algo: ecmp hash algorithm + lag_hash_algo: lag hash algorithm + """ + _, ecmp_hash_algorithm, _, lag_hash_algorithm = get_global_hash_config(duthost) + if ecmp_hash_algo: + pytest_assert(ecmp_hash_algo == ecmp_hash_algorithm, + 'The global hash algorithm configuration is not as expected:\n' + f'expected ecmp hash algorithm: {ecmp_hash_algo}\n' + f'actual ecmp hash algorithm: {ecmp_hash_algorithm}\n') + if lag_hash_algo: + pytest_assert(lag_hash_algo == lag_hash_algorithm, + 'The global hash algorithm configuration is not as expected:\n' + f'expected lag hash algorithm: {lag_hash_algo}\n' + f'actual lag hash algorithm: {lag_hash_algorithm}\n') + + +def get_ip_route_nexthops(duthost, destination): + """ + Get nexthop interfaces for a specific destination + Args: + duthost (AnsibleHost): Device Under Test (DUT) + destination: get the nexthops of this route + Returns: + The nexthop interfaces + """ + output = duthost.shell(f'show ip route {destination} json')['stdout'] + ip_route_json = json.loads(output) + nexthop_list = [] + for route in ip_route_json[destination]: + nexthop_list.extend(route["nexthops"]) + nexthops = [] + for nexthop in nexthop_list: + nexthops.append(nexthop["interfaceName"]) + return nexthops + + +def check_default_route(duthost, expected_nexthops): + """ + Check the default route exists and the nexthops interfaces are as expected. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + expected_nexthops: expected nexthop interfaces + Returns: + True if the nexthops are the same as the expected. + """ + logger.info("Check if the default route is available.") + nexthops = get_ip_route_nexthops(duthost, "0.0.0.0/0") + return set(nexthops) == set(expected_nexthops) + + +def get_ptf_port_indices(mg_facts, downlink_interfaces, uplink_interfaces): + """ + Get the ptf port indices for the interfaces under test. + Args: + mg_facts: minigraph facts + downlink_interfaces: a list of downlink interfaces on dut + uplink_interfaces: a dictionary of uplink(egress) interfaces on dut + Returns: + sending_ports: a list of the ptf port indices which will be used to send the test traffic example: [57] + expected_port_groups: a list of the ptf port indices which will be used to received the test traffic, + the indices in a group means the ports are in a same portchannel + example: [[0, 2], [8, 10], [21, 22], [40, 41]] + """ + sending_ports = [] + for interface in downlink_interfaces: + sending_ports.append(mg_facts['minigraph_ptf_indices'][interface]) + expected_port_groups = [] + for index, portchannel in enumerate(uplink_interfaces.keys()): + expected_port_groups.append([]) + for interface in uplink_interfaces[portchannel]: + expected_port_groups[index].append(mg_facts['minigraph_ptf_indices'][interface]) + expected_port_groups[index].sort() + return sending_ports, expected_port_groups + + +def flap_interfaces(duthost, interfaces, portchannels=[], times=3): + """ + Flap the specified interfaces. Assert when any of the interfaces is not up after the flapping. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + interfaces: a list of interfaces to be flapped + portchannels: a list of portchannels which need to check the status after the flapping + times: flap times, every interface will be shutdown/startup for the value number times + """ + logger.info(f"Flap the interfaces {interfaces} for {times} times.") + # Flap the interface + for _ in range(times): + for interface in interfaces: + shutdown_interface(duthost, interface) + startup_interface(duthost, interface) + # Check the interfaces status are up + for interface in interfaces: + pytest_assert(wait_until(30, 2, 0, duthost.is_interface_status_up, interface), + f"The interface {interface} is not up after the flapping.") + for portchannel in portchannels: + pytest_assert(wait_until(30, 2, 0, duthost.is_interface_status_up, portchannel), + f"The portchannel {portchannel} is not up after the flapping.") + + +def remove_add_portchannel_member(duthost, interface, portchannel): + """ + Remove and then add the specified members. Assert when any of the interfaces is not up after the remove/add. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + interface: the member to be removed/added + portchannel: the portchannel which the member belongs to + """ + logger.info(f"Remove the member {interface} from the portchannel {portchannel}.") + duthost.shell(f"config portchannel member del {portchannel} {interface}") + logger.info(f"Add back {interface} to {portchannel}.") + duthost.shell(f"config portchannel member add {portchannel} {interface}") + # Check the portchannel is up + pytest_assert(wait_until(30, 2, 0, duthost.is_interface_status_up, portchannel), + f"The portchannel {portchannel} is not up after the member remove/add.") + + +def get_ip_range(ipver, inner_ipver): + """ + Generate the ip address range according to the ip versions. + If the hash field is a inner field, generate both outer and inner versions. + Args: + ipver: outer frame ip version + inner_ipver: inner frame ip version + Returns: + src_ip_range: outer source ip address range + dst_ip_range: outer destination ip address range + inner_src_ip_range: inner source ip address range + inner_dst_ip_range: inner destination ip address range + """ + src_ip_range = IP_RANGE[ipver]['src'] + dst_ip_range = IP_RANGE[ipver]['dst'] + inner_src_ip_range = IP_RANGE[inner_ipver]['src'] + inner_dst_ip_range = IP_RANGE[inner_ipver]['dst'] + return src_ip_range, dst_ip_range, inner_src_ip_range, inner_dst_ip_range + + +def get_interfaces_for_test(duthost, mg_facts, hash_field): + """ + Get the interfaces used in the test according to the hash field. + On t0 and t1 topologies, all uplink interfaces are portchannel interfaces. + Down link interfaces could be ethernet interfaces or members of vlan interface or portchannel interfaces + which differs with the topologies. Here we need the name of the ethernet interface. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + mg_facts: minigraph facts + hash_field: the hash field under test + Returns: + uplink_interfaces: a dictionary of the uplink interfaces + example: {'PortChannel101': ['Ethernet0', 'Ethernet2'], + 'PortChannel102': ['Ethernet8', 'Ethernet10']} + downlink_interfaces: a list of the downlink interfaces on the dut. If the hash field is not IN_PORT, + only one interface is randomly selected, otherwise all the downlinks are used. + example: ['Ethernet48'] + ['Ethernet2', 'Ethernet4', ..., 'Ethernet48'] for IN_PORT test + """ + # Get uplink interfaces + uplink_interfaces = {} + # Find the uplink interfaces which are the nexthop interfaces of the default route + for interface in get_ip_route_nexthops(duthost, "0.0.0.0/0"): + uplink_interfaces[interface] = [] + # All uplink interfaces are portchannels, need to find the members + portchannel_members = mg_facts['minigraph_portchannels'][interface]['members'] + uplink_interfaces[interface].extend(portchannel_members) + # Randomly choose a downlink interface + downlink_interfaces = [] + if mg_facts['minigraph_vlan_interfaces']: + vlan_interface = mg_facts['minigraph_vlan_interfaces'][0]['attachto'] + downlink_interfaces = mg_facts['minigraph_vlans'][vlan_interface]['members'] + elif mg_facts['minigraph_interfaces']: + for interface in mg_facts['minigraph_interfaces']: + downlink_interfaces.append(interface['attachto']) + else: + portchannels = mg_facts['minigraph_portchannels'] + for portchannel in portchannels.keys(): + if portchannel not in uplink_interfaces.keys(): + downlink_interfaces.extend(portchannels[portchannel]['members']) + if hash_field != 'IN_PORT': + downlink_interfaces = [random.choice(downlink_interfaces)] + logger.info( + f"Interfaces are selected for the test: downlink: {downlink_interfaces}, uplink: {uplink_interfaces}") + + return uplink_interfaces, downlink_interfaces + + +def get_asic_type(request): + metadata = get_testbed_metadata(request) + if metadata is None: + logger.warning("Failed to get asic type, " + "need to run test_update_testbed_metadata in test_pretest.py to collect dut asic type .") + logger.warning("Using the default hash capabilities for asic type is unknown.") + asic_type = 'unknown' + else: + # Always get the asic type from the first dut + dut_info = metadata[list(metadata.keys())[0]] + asic_type = dut_info.get('asic_type', "") + return asic_type + + +def get_hash_fields_from_option(request, test_type, hash_field_option): + """ + Generate the hash fields to test based on the pytest option. + Args: + request: pytest request + test_type: indicates if it is a ecmp test or lag test. DST_MAC, ETHERTYPE, VLAN_ID are not suitable + for ecmp test because the traffic need to be L2 + hash_field_option: the value of pytest option "--hash_field" + Returns: + a list of the hash fields to test + """ + asic_type = get_asic_type(request) + if asic_type in HASH_CAPABILITIES: + hash_fields = HASH_CAPABILITIES[asic_type][test_type] + else: + hash_fields = HASH_CAPABILITIES['default'][test_type] + + if hash_field_option == "all": + return hash_fields + elif hash_field_option == "random": + return [random.choice(hash_fields)] + elif hash_field_option in hash_fields: + return [hash_field_option] + elif set(hash_field_option.split(',')).issubset(hash_fields): + return hash_field_option.split(',') + else: + pytest.fail("Invalid value of the '--hash_field' option.") + + +def get_hash_algorithm_from_option(request, hash_algorithm_identifier): + """ + Generate the hash algorithm to test based on the pytest option. + Args: + hash_algorithm_identifier: the pytest option value of the --algorithm + Returns: + a list of the hash algorithm to test + """ + asic_type = get_asic_type(request) + if asic_type in 'mellanox': + supported_hash_algorithm_list = MELLANOX_SUPPORTED_HASH_ALGORITHM[:] + else: + supported_hash_algorithm_list = DEFAULT_SUPPORTED_HASH_ALGORITHM[:] + if hash_algorithm_identifier == 'all': + return supported_hash_algorithm_list + elif hash_algorithm_identifier == 'random': + return [random.choice(supported_hash_algorithm_list)] + elif hash_algorithm_identifier in supported_hash_algorithm_list: + return [hash_algorithm_identifier] + elif set(hash_algorithm_identifier.split(',')).issubset(set(supported_hash_algorithm_list)): + return hash_algorithm_identifier.split(',') + else: + pytest.fail("Invalid value of the '--algorithm' option.") + + +def get_diff_hash_algorithm(supported_algorithm, get_supported_hash_algorithms): + """ + Get a different supported hash algorithm + :param supported_algorithm: current supported algorithm + :return: another supported algorithm + """ + supported_hash_algorithm_list = get_supported_hash_algorithms[:] + if supported_algorithm in supported_hash_algorithm_list: + temp_hash_algo_list = supported_hash_algorithm_list + temp_hash_algo_list.remove(supported_algorithm) + return random.choice(temp_hash_algo_list) + else: + return random.choice(supported_hash_algorithm_list) + + +def get_ip_version_from_option(ip_version_option): + """ + Generate the ip version to test based on the pytest option. + Args: + ip_version_option: the pytest option value of the --ip_version or --inner_ip_version + Returns: + a list of the ip versions to test + """ + if ip_version_option == 'all': + return ['ipv4', 'ipv6'] + elif ip_version_option == 'random': + return [random.choice(['ipv4', 'ipv6'])] + else: + return [ip_version_option] + + +def get_reboot_type_from_option(reboot_option): + """ + Generate the reboot type to test based on the pytest option. + Args: + reboot_option: the pytest option value of --reboot + Returns: + the list of reboot types + """ + if reboot_option == 'all': + return ['cold', 'warm', 'fast', 'reload'] + elif reboot_option == 'random': + return [random.choice(['cold', 'warm', 'fast', 'reload'])] + else: + return [reboot_option] + + +def get_encap_type_from_option(encap_type_option): + """ + Generate the encapsulation type to test based on the pytest option. + Args: + encap_type_option: the pytest option value of --encap_type + Returns: + the encap type + """ + if encap_type_option == 'random': + return [random.choice(['ipinip', 'vxlan', 'nvgre'])] + elif encap_type_option == 'all': + return ['ipinip', 'vxlan', 'nvgre'] + else: + return [encap_type_option] + + +def format_ip_mask(ip_network, strict=False): + """ + Format the full mask notation to the mask in dotted decimal notation + """ + ip_addr = ipaddress.ip_network(ip_network, strict=strict) + if ip_addr.version == 4: + return str(ip_addr) + else: + return ip_network + + +def remove_ip_interface_and_config_vlan(duthost, mg_facts, tbinfo, downlink_interface, uplink_interfaces, hash_field): + """ + Re-configure the interface and vlan on dut to enable switching of L2 traffic. + Only for testing DST_MAC, ETHERTYPE, VLAN_ID fields. + The changed configurations are stored in global variables for later restoration + Args: + duthost (AnsibleHost): Device Under Test (DUT) + mg_facts: minigraph facts + tbinfo: testbed info + downlink_interface: the downlink(ingress) interface under test + uplink_interfaces: the uplink(egress) interfaces under test + hash_field: the hash field to test + """ + logger.info("Modify the interface and vlan configurations for L2 test.") + # re-config the downlink interfaces + # if topology is t0, move the downlink interfaces out of the VLAN + if tbinfo['topo']['type'] == 't0': + for vlan in mg_facts['minigraph_vlans'].keys(): + if downlink_interface in mg_facts['minigraph_vlans'][vlan]['members']: + duthost.shell(f"config vlan member del {vlan.strip('Vlan')} {downlink_interface}") + vlan_member_to_restore['vlan_id'] = vlan.strip('Vlan') + vlan_member_to_restore['interface'] = downlink_interface + l2_ports.add(downlink_interface) + else: + # if topology is t1, remove the ip address on downlink interface + for ip_interface in mg_facts['minigraph_interfaces']: + if ip_interface['attachto'] == downlink_interface: + formatted_ip_addr = format_ip_mask(f"{ip_interface['addr']}/{ip_interface['mask']}") + duthost.shell(f"config interface ip remove {ip_interface['attachto']} {formatted_ip_addr}") + ip_interface_to_restore.append(ip_interface) + l2_ports.add(downlink_interface) + for portchannel in mg_facts['minigraph_portchannels'].values(): + if downlink_interface in portchannel['members']: + for portchannel_ip_interface in mg_facts['minigraph_portchannel_interfaces']: + if portchannel_ip_interface['attachto'] == portchannel['name']: + formatted_ip_addr = format_ip_mask( + f"{portchannel_ip_interface['addr']}/{portchannel_ip_interface['mask']}") + duthost.shell( + f"config interface ip remove {portchannel_ip_interface['attachto']} {formatted_ip_addr}") + ip_interface_to_restore.append(portchannel_ip_interface) + l2_ports.add(portchannel_ip_interface['attachto']) + # re-config the uplink interfaces, remove the ip address on the egress portchannel interfaces + for ip_interface in mg_facts['minigraph_portchannel_interfaces']: + if ip_interface['attachto'] in uplink_interfaces: + formatted_ip_addr = format_ip_mask(f"{ip_interface['addr']}/{ip_interface['mask']}") + duthost.shell(f"config interface ip remove {ip_interface['attachto']} {formatted_ip_addr}") + ip_interface_to_restore.append(ip_interface) + l2_ports.add(ip_interface['attachto']) + # Configure VLANs for VLAN_ID test + if hash_field == 'VLAN_ID': + for vlan in range(VLAN_RANGE[0], VLAN_RANGE[1]): + duthost.shell(f'config vlan add {vlan}') + for port in l2_ports: + duthost.shell(f'config vlan member add {vlan} {port}') + vlans_to_remove.extend(list(range(VLAN_RANGE[0], VLAN_RANGE[1]))) + else: + # Add the interfaces into one vlan for other hash fields + duthost.shell(f'config vlan add {VLAN_RANGE[0]}') + for port in l2_ports: + duthost.shell(f'config vlan member add {VLAN_RANGE[0]} {port} --untagged') + vlans_to_remove.append(VLAN_RANGE[0]) + # Wait 10 seconds for the configurations to take effect + time.sleep(10) + + +def shutdown_interface(duthost, interface): + """ + Shutdown interface and add it to the global variable. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + interface: interface to shutdown + """ + duthost.shutdown(interface) + interfaces_to_startup.append(interface) + + +def startup_interface(duthost, interface): + """ + Startup interface and remove it from the global variable. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + interface: interface to startup + """ + duthost.no_shutdown(interface) + if interface in interfaces_to_startup: + interfaces_to_startup.remove(interface) + + +def get_vlan_intf_mac(duthost): + config_facts = duthost.get_running_config_facts() + vlan_intfs = list(config_facts['VLAN_INTERFACE']) + vlan_intf_mac = config_facts['VLAN'][vlan_intfs[0]]['mac'] + return vlan_intf_mac + + +def generate_test_params(duthost, tbinfo, mg_facts, hash_field, ipver, inner_ipver, encap_type, uplink_interfaces, + downlink_interfaces, ecmp_hash, lag_hash, is_l2_test=False): + """ + Generate ptf test parameters. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + mg_facts: minigraph facts + hash_field: hash field to test + ipver: outer frame IP version + inner_ipver: inner frame ip version + uplink_interfaces: uplink interfaces of dut + downlink_interfaces: downlink interfaces used in the test + ecmp_hash: if ecmp hash is tested + lag_hash: if lag hash is tested + encap_type: the encapsulation type when testing inner fields + is_l2_test: if L2 traffic is should be used in test + """ + src_ip_range, dst_ip_range, inner_src_ip_range, inner_dst_ip_range = get_ip_range(ipver, inner_ipver) + # Get the ptf src and dst ports + ptf_sending_ports, ptf_expected_port_groups = get_ptf_port_indices( + mg_facts, downlink_interfaces=downlink_interfaces, uplink_interfaces=uplink_interfaces) + if 'dualtor' in tbinfo['topo']['name']: + dest_mac = get_vlan_intf_mac(duthost) + else: + dest_mac = duthost.facts['router_mac'] + ptf_params = {"router_mac": dest_mac, + "sending_ports": ptf_sending_ports, + "expected_port_groups": ptf_expected_port_groups, + "hash_field": hash_field, + "vlan_range": VLAN_RANGE, + 'ethertype_range': ETHERTYPE_RANGE, + "ipver": ipver, + "src_ip_range": ",".join(src_ip_range), + "dst_ip_range": ",".join(dst_ip_range), + "balancing_test_times": balancing_test_times, + "balancing_range": balancing_range, + "ecmp_hash": ecmp_hash, + "lag_hash": lag_hash, + "is_l2_test": is_l2_test} + if "INNER" in hash_field: + ptf_params['inner_ipver'] = inner_ipver + ptf_params['inner_src_ip_range'] = ",".join(inner_src_ip_range) + ptf_params['inner_dst_ip_range'] = ",".join(inner_dst_ip_range) + ptf_params['encap_type'] = encap_type + if encap_type == 'vxlan': + ptf_params['vxlan_port'] = random.choice(vxlan_port_list) + return ptf_params + + +def config_custom_vxlan_port(duthost, port): + """ + Configure the custom VxLAN udp dport + Args: + duthost (AnsibleHost): Device Under Test (DUT) + port: the custom port number + """ + global restore_vxlan + logger.info(f"Configure VxLAN port to {port}") + vxlan_ecmp_utils.Constants['DEBUG'] = False + vxlan_ecmp_utils.Constants['KEEP_TEMP_FILES'] = False + vxlan_ecmp_utils.configure_vxlan_switch(duthost, port, duthost.facts['router_mac']) + restore_vxlan = True diff --git a/tests/hash/test_generic_hash.py b/tests/hash/test_generic_hash.py new file mode 100644 index 0000000000..444f424a9a --- /dev/null +++ b/tests/hash/test_generic_hash.py @@ -0,0 +1,832 @@ +import pytest +import random +import time +import logging + +from tests.common.helpers.assertions import pytest_assert +from generic_hash_helper import get_hash_fields_from_option, get_ip_version_from_option, get_encap_type_from_option, \ + get_reboot_type_from_option, HASH_CAPABILITIES, check_global_hash_config, startup_interface, \ + get_interfaces_for_test, get_ptf_port_indices, check_default_route, generate_test_params, flap_interfaces, \ + PTF_QLEN, remove_ip_interface_and_config_vlan, config_custom_vxlan_port, shutdown_interface, \ + remove_add_portchannel_member, get_hash_algorithm_from_option, check_global_hash_algorithm, get_diff_hash_algorithm +from generic_hash_helper import restore_configuration, reload, global_hash_capabilities, restore_interfaces # noqa:F401 +from generic_hash_helper import mg_facts, restore_init_hash_config, restore_vxlan_port, \ + get_supported_hash_algorithms, toggle_all_simulator_ports_to_upper_tor # noqa:F401 +from tests.common.utilities import wait_until +from tests.ptf_runner import ptf_runner +from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 +from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer +from tests.common.reboot import reboot +from tests.common.config_reload import config_reload +from tests.common.plugins.allure_wrapper import allure_step_wrapper as allure + +DEFAULT_VXLAN_PORT = 4789 +PTF_LOG_PATH = "/tmp/generic_hash_test.GenericHashTest.log" + +pytestmark = [ + pytest.mark.topology('t0', 't1'), +] +logger = logging.getLogger(__name__) + + +def pytest_generate_tests(metafunc): + """ + Use the random hash field to generate the pytest test case, + this provides possibility to skip some hash field when there is some issue. + """ + params = [] + params_tuple = [] + if 'lag' in metafunc.function.__name__: + hash_fields = get_hash_fields_from_option(metafunc, 'lag', metafunc.config.getoption("--hash_field")) + else: + hash_fields = get_hash_fields_from_option(metafunc, 'ecmp', metafunc.config.getoption("--hash_field")) + hash_algorithms = get_hash_algorithm_from_option(metafunc, metafunc.config.getoption("--algorithm")) + outer_ip_versions = get_ip_version_from_option(metafunc.config.getoption("--ip_version")) + inner_ip_versions = get_ip_version_from_option(metafunc.config.getoption("--inner_ip_version")) + encap_types = get_encap_type_from_option(metafunc.config.getoption("--encap_type")) + for field in hash_fields: + if 'INNER' not in field: + params_tuple.extend([(algorithm, field, ip_version, 'None', 'None') + for algorithm in hash_algorithms + for ip_version in outer_ip_versions]) + elif 'INNER_ETHERTYPE' in field: + params_tuple.extend([(algorithm, field, ip_version, 'None', encap_type) + for algorithm in hash_algorithms + for ip_version in outer_ip_versions + for encap_type in encap_types]) + else: + params_tuple.extend([(algorithm, field, ip_version, inner_ip_version, encap_type) + for algorithm in hash_algorithms + for ip_version in outer_ip_versions + for inner_ip_version in inner_ip_versions + for encap_type in encap_types]) + for param in params_tuple: + params.append('-'.join(param)) + if 'params' in metafunc.fixturenames: + metafunc.parametrize("params", params) + + reboot_types = get_reboot_type_from_option(metafunc.config.getoption("--reboot")) + if 'reboot_type' in metafunc.fixturenames: + metafunc.parametrize("reboot_type", reboot_types) + + +@pytest.fixture(scope='function') +def fine_params(params, global_hash_capabilities): # noqa:F811 + hash_algorithm, _, _, _, _ = params.split('-') + all_supported_hash_algorithms = set(global_hash_capabilities['ecmp_algo']).\ + union(set(global_hash_capabilities['ecmp_algo'])) + if hash_algorithm not in all_supported_hash_algorithms: + pytest.skip(f"{hash_algorithm} is not supported on current platform, " + f"the supported algorithms: {all_supported_hash_algorithms}") + return params + + +def skip_unsupported_packet(hash_field, encap_type): + if hash_field in ['INNER_SRC_MAC', 'INNER_DST_MAC', 'INNER_ETHERTYPE'] and encap_type == 'ipinip': + pytest.skip(f"The field {hash_field} is not supported in ipinip encapsulation.") + + +def skip_unsupported_field_for_ecmp_test(field, encap_type): + if field in ['DST_MAC', 'ETHERTYPE', 'VLAN_ID']: + pytest.skip(f"The field {field} is not supported by the ecmp test case.") + skip_unsupported_packet(field, encap_type) + + +def skip_single_member_lag_topology(uplink_portchannels, field, encap_type): + lag_member_count = len(list(uplink_portchannels.values())[0]) + if lag_member_count < 2: + pytest.skip("Skip the test_lag_member_flap case on setups without multi-member uplink portchannels.") + skip_unsupported_packet(field, encap_type) + + +def config_validate_algorithm(duthost, algorithm_type, supported_algorithms): + for algorithm in supported_algorithms: + with allure.step(f"Configure algorithm: {algorithm} from supported algorithms: {supported_algorithms}"): + if 'ecmp' == algorithm_type: + duthost.set_switch_hash_global_algorithm('ecmp', algorithm) + check_global_hash_algorithm(duthost, ecmp_hash_algo=algorithm) + if 'lag' == algorithm_type: + duthost.set_switch_hash_global_algorithm('lag', algorithm) + check_global_hash_algorithm(duthost, lag_hash_algo=algorithm) + + +def test_hash_capability(duthost, global_hash_capabilities): # noqa:F811 + """ + Test case to verify the 'show switch-hash capabilities' command. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + global_hash_capabilities: module level fixture to get the dut hash capabilities + """ + with allure.step('Check the dut hash capabilities are as expected'): + ecmp_hash_capability, lag_hash_capability = global_hash_capabilities['ecmp'], global_hash_capabilities['lag'] + asic_type = duthost.facts["asic_type"] + expected_hash_capabilities = HASH_CAPABILITIES.get(asic_type, HASH_CAPABILITIES['default']) + expected_ecmp_hash_capability = expected_hash_capabilities['ecmp'] + expected_lag_hash_capability = expected_hash_capabilities['lag'] + pytest_assert(sorted(ecmp_hash_capability) == sorted(expected_ecmp_hash_capability), + 'The ecmp hash capability is not as expected.') + pytest_assert(sorted(lag_hash_capability) == sorted(expected_lag_hash_capability), + 'The lag hash capability is not as expected.') + + +def test_ecmp_hash(duthost, tbinfo, ptfhost, fine_params, mg_facts, global_hash_capabilities, # noqa:F811 + restore_vxlan_port, toggle_all_simulator_ports_to_upper_tor, skip_traffic_test): # noqa:F811 + """ + Test case to validate the ecmp hash. The hash field to test is randomly chosen from the supported hash fields. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ptfhost (AnsibleHost): Packet Test Framework (PTF) + mg_facts: minigraph facts + hash_algorithm: randomly generated hash algorithm + ecmp_test_hash_field: randomly generated ecmp hash field parameter + ipver: randomly generated outer frame ip version + inner_ipver: randomly generated inner frame ip version + encap_type: randomly generated encapsulation type + restore_vxlan_port: fixture to restore vxlan port to default + global_hash_capabilities: module level fixture to get the dut hash capabilities + """ + hash_algorithm, ecmp_test_hash_field, ipver, inner_ipver, encap_type = fine_params.split('-') + skip_unsupported_field_for_ecmp_test(ecmp_test_hash_field, encap_type) + with allure.step('Randomly select an ecmp hash field to test and configure the global ecmp and lag hash'): + lag_hash_fields = global_hash_capabilities['lag'] + lag_hash_fields = lag_hash_fields[:] + lag_hash_fields.remove(ecmp_test_hash_field) + # Config the hash fields + duthost.set_switch_hash_global('ecmp', [ecmp_test_hash_field]) + duthost.set_switch_hash_global('lag', lag_hash_fields) + with allure.step(f'Configure ecmp hash algorithm: {hash_algorithm}'): + duthost.set_switch_hash_global_algorithm('ecmp', hash_algorithm) + with allure.step("Check the config result"): + check_global_hash_config( + duthost, ecmp_hash_fields=[ecmp_test_hash_field], lag_hash_fields=lag_hash_fields) + check_global_hash_algorithm(duthost, hash_algorithm) + with allure.step('Prepare test parameters'): + # Get the interfaces for the test, downlink interface is selected randomly + uplink_interfaces, downlink_interfaces = get_interfaces_for_test(duthost, mg_facts, ecmp_test_hash_field) + ptf_params = generate_test_params( + duthost, tbinfo, mg_facts, ecmp_test_hash_field, ipver, inner_ipver, encap_type, uplink_interfaces, + downlink_interfaces, ecmp_hash=True, lag_hash=False) + if ptf_params.get('vxlan_port') and ptf_params['vxlan_port'] != DEFAULT_VXLAN_PORT: + config_custom_vxlan_port(duthost, ptf_params['vxlan_port']) + + with allure.step('Start the ptf test, send traffic and check the balancing'): + # Check the default route before the ptf test + pytest_assert(check_default_route(duthost, uplink_interfaces.keys()), + 'The default route is not available or some nexthops are missing.') + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + +def test_lag_hash(duthost, ptfhost, tbinfo, fine_params, mg_facts, restore_configuration, # noqa:F811 + restore_vxlan_port, global_hash_capabilities, # noqa F811 + toggle_all_simulator_ports_to_upper_tor, skip_traffic_test): # noqa:F811 + """ + Test case to validate the lag hash. The hash field to test is randomly chosen from the supported hash fields. + When hash field is in [DST_MAC, ETHERTYPE, VLAN_ID], need to re-configure the dut for L2 traffic. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ptfhost (AnsibleHost): Packet Test Framework (PTF) + mg_facts: minigraph facts + tbinfo: testbed info fixture + restore_configuration: fixture to restore the interface and vlan configurations after L2 test + hash_algorithm: randomly generated hash algorithm + lag_test_hash_field: randomly generated lag hash field parameter + ipver: randomly generated outer frame ip version + inner_ipver: randomly generated inner frame ip version + encap_type: randomly generated encapsulation type + restore_vxlan_port: fixture to restore vxlan port to default + global_hash_capabilities: module level fixture to get the dut hash capabilities + """ + hash_algorithm, lag_test_hash_field, ipver, inner_ipver, encap_type = fine_params.split('-') + with allure.step('Randomly select a lag hash field to test and configure the global ecmp and lag hash'): + ecmp_hash_fields = global_hash_capabilities['ecmp'] + ecmp_hash_fields = ecmp_hash_fields[:] + ecmp_hash_fields.remove(lag_test_hash_field) + # Get the interfaces for the test, downlink interface is selected randomly + uplink_interfaces, downlink_interfaces = get_interfaces_for_test(duthost, mg_facts, lag_test_hash_field) + # If the uplinks are not multi-member portchannels, skip the test + skip_single_member_lag_topology(uplink_interfaces, lag_test_hash_field, encap_type) + # Config the hash fields + duthost.set_switch_hash_global('ecmp', ecmp_hash_fields) + duthost.set_switch_hash_global('lag', [lag_test_hash_field]) + with allure.step(f'Configure lag hash algorithm: {hash_algorithm}'): + duthost.set_switch_hash_global_algorithm('lag', hash_algorithm) + with allure.step("Check the config result"): + check_global_hash_config( + duthost, ecmp_hash_fields=ecmp_hash_fields, lag_hash_fields=[lag_test_hash_field]) + check_global_hash_algorithm(duthost, lag_hash_algo=hash_algorithm) + with allure.step('Change topology for L2 test if hash field in DST_MAC, ETHERTYPE, VLAN_ID'): + # Need to send l2 traffic to validate SRC_MAC, DST_MAC, ETHERTYPE, VLAN_ID keys, changing topology is required + is_l2_test = False + if lag_test_hash_field in ['DST_MAC', 'ETHERTYPE', 'VLAN_ID']: + # For L2 test, only one uplink portchannel interface is needed + is_l2_test = True + for _ in range(len(uplink_interfaces) - 1): + uplink_interfaces.popitem() + remove_ip_interface_and_config_vlan( + duthost, mg_facts, tbinfo, downlink_interfaces[0], uplink_interfaces, lag_test_hash_field) + with allure.step('Prepare test parameters'): + ptf_params = generate_test_params( + duthost, tbinfo, mg_facts, lag_test_hash_field, ipver, inner_ipver, encap_type, uplink_interfaces, + downlink_interfaces, ecmp_hash=False, lag_hash=True, is_l2_test=is_l2_test) + if ptf_params.get('vxlan_port') and ptf_params['vxlan_port'] != DEFAULT_VXLAN_PORT: + config_custom_vxlan_port(duthost, ptf_params['vxlan_port']) + with allure.step('Start the ptf test, send traffic and check the balancing'): + # Check the default route before the ptf test + if not is_l2_test: + pytest_assert(check_default_route(duthost, uplink_interfaces.keys()), + 'The default route is not available or some nexthops are missing.') + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + +def config_all_hash_fields(duthost, global_hash_capabilities): # noqa:F811 + duthost.set_switch_hash_global('ecmp', global_hash_capabilities['ecmp']) + duthost.set_switch_hash_global('lag', global_hash_capabilities['lag']) + + +def config_all_hash_algorithm(duthost, ecmp_algorithm, lag_algorithm): # noqa:F811 + duthost.set_switch_hash_global_algorithm('ecmp', ecmp_algorithm) + duthost.set_switch_hash_global_algorithm('lag', lag_algorithm) + + +def test_ecmp_and_lag_hash(duthost, tbinfo, ptfhost, fine_params, mg_facts, global_hash_capabilities, # noqa:F811 + restore_vxlan_port, get_supported_hash_algorithms, # noqa:F811 + toggle_all_simulator_ports_to_upper_tor, skip_traffic_test): # noqa:F811 + """ + Test case to validate the hash behavior when both ecmp and lag hash are configured with a same field. + The hash field to test is randomly chosen from the supported hash fields. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ptfhost (AnsibleHost): Packet Test Framework (PTF) + mg_facts: minigraph facts + ecmp_algorithm: randomly generated ecmp hash algorithm + ecmp_test_hash_field: randomly generated ecmp hash field parameter + ipver: randomly generated outer frame ip version + inner_ipver: randomly generated inner frame ip version + encap_type: randomly generated encapsulation type + restore_vxlan_port: fixture to restore vxlan port to default + global_hash_capabilities: module level fixture to get the dut hash capabilities + """ + ecmp_algorithm, ecmp_test_hash_field, ipver, inner_ipver, encap_type = fine_params.split('-') + skip_unsupported_field_for_ecmp_test(ecmp_test_hash_field, encap_type) + with allure.step('Randomly select an ecmp hash field to test ' + 'and configure all supported fields to the global ecmp and lag hash'): + config_all_hash_fields(duthost, global_hash_capabilities) + lag_algorithm = get_diff_hash_algorithm(ecmp_algorithm, get_supported_hash_algorithms) + with allure.step(f'Configure ecmp hash algorithm: {ecmp_algorithm} - lag hash algorithm: {lag_algorithm}'): + config_all_hash_algorithm(duthost, ecmp_algorithm, lag_algorithm) + with allure.step("Check the config result"): + check_global_hash_config(duthost, global_hash_capabilities['ecmp'], global_hash_capabilities['lag']) + check_global_hash_algorithm(duthost, ecmp_algorithm, lag_algorithm) + with allure.step('Prepare test parameters'): + # Get the interfaces for the test, downlink interface is selected randomly + uplink_interfaces, downlink_interfaces = get_interfaces_for_test(duthost, mg_facts, ecmp_test_hash_field) + ptf_params = generate_test_params( + duthost, tbinfo, mg_facts, ecmp_test_hash_field, ipver, inner_ipver, encap_type, uplink_interfaces, + downlink_interfaces, ecmp_hash=True, lag_hash=True) + if ptf_params.get('vxlan_port') and ptf_params['vxlan_port'] != DEFAULT_VXLAN_PORT: + config_custom_vxlan_port(duthost, ptf_params['vxlan_port']) + with allure.step('Start the ptf test, send traffic and check the balancing'): + # Check the default route before the ptf test + pytest_assert(check_default_route(duthost, uplink_interfaces.keys()), + 'The default route is not available or some nexthops are missing.') + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + +def test_nexthop_flap(duthost, tbinfo, ptfhost, fine_params, mg_facts, restore_interfaces, # noqa:F811 + restore_vxlan_port, global_hash_capabilities, get_supported_hash_algorithms, # noqa:F811 + toggle_all_simulator_ports_to_upper_tor, skip_traffic_test): # noqa:F811 + """ + Test case to validate the ecmp hash when there is nexthop flapping. + The hash field to test is randomly chosen from the supported hash fields. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ptfhost (AnsibleHost): Packet Test Framework (PTF) + mg_facts: minigraph facts + restore_interfaces: fixture to restore the interfaces used in the test + ecmp_algorithm: randomly generated ecmp hash algorithm + ecmp_test_hash_field: randomly generated ecmp hash field parameter + ipver: randomly generated outer frame ip version + inner_ipver: randomly generated inner frame ip version + encap_type: randomly generated encapsulation type + restore_vxlan_port: fixture to restore vxlan port to default + global_hash_capabilities: module level fixture to get the dut hash capabilities + """ + ecmp_algorithm, ecmp_test_hash_field, ipver, inner_ipver, encap_type = fine_params.split('-') + skip_unsupported_field_for_ecmp_test(ecmp_test_hash_field, encap_type) + with allure.step('Randomly select an ecmp hash field to test ' + 'and configure all supported fields to the global ecmp and lag hash'): + config_all_hash_fields(duthost, global_hash_capabilities) + lag_algorithm = get_diff_hash_algorithm(ecmp_algorithm, get_supported_hash_algorithms) + with allure.step(f'Configure ecmp hash algorithm: {ecmp_algorithm} - lag hash algorithm: {lag_algorithm}'): + config_all_hash_algorithm(duthost, ecmp_algorithm, lag_algorithm) + with allure.step("Check the config result"): + check_global_hash_config(duthost, global_hash_capabilities['ecmp'], global_hash_capabilities['lag']) + check_global_hash_algorithm(duthost, ecmp_algorithm, lag_algorithm) + with allure.step('Prepare test parameters'): + # Get the interfaces for the test, downlink interface is selected randomly + uplink_interfaces, downlink_interfaces = get_interfaces_for_test(duthost, mg_facts, ecmp_test_hash_field) + ptf_params = generate_test_params( + duthost, tbinfo, mg_facts, ecmp_test_hash_field, ipver, inner_ipver, encap_type, uplink_interfaces, + downlink_interfaces, ecmp_hash=True, lag_hash=True) + if ptf_params.get('vxlan_port') and ptf_params['vxlan_port'] != DEFAULT_VXLAN_PORT: + config_custom_vxlan_port(duthost, ptf_params['vxlan_port']) + with allure.step('Start the ptf test, send traffic and check the balancing'): + # Check the default route before the ptf test + pytest_assert(check_default_route(duthost, uplink_interfaces.keys()), + 'The default route is not available or some nexthops are missing.') + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + with allure.step('Randomly shutdown 1 nexthop interface'): + interface = random.choice(list(uplink_interfaces.keys())) + remaining_uplink_interfaces = uplink_interfaces.copy() + remaining_uplink_interfaces.pop(interface) + origin_ptf_expected_port_groups = ptf_params['expected_port_groups'] + _, ptf_params['expected_port_groups'] = get_ptf_port_indices( + mg_facts, downlink_interfaces=[], uplink_interfaces=remaining_uplink_interfaces) + shutdown_interface(duthost, interface) + with allure.step('Start the ptf test, send traffic and check the balancing'): + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + with allure.step('Startup the interface, and then flap it 3 more times'): + startup_interface(duthost, interface) + flap_interfaces(duthost, [interface], times=3) + pytest_assert(wait_until(10, 2, 0, check_default_route, duthost, uplink_interfaces.keys()), + 'The default route is not restored after the flapping.') + ptf_params['expected_port_groups'] = origin_ptf_expected_port_groups + with allure.step('Start the ptf test, send traffic and check the balancing'): + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + +def test_lag_member_flap(duthost, tbinfo, ptfhost, fine_params, mg_facts, restore_configuration, # noqa F811 + restore_interfaces, global_hash_capabilities, restore_vxlan_port, # noqa F811 + get_supported_hash_algorithms, toggle_all_simulator_ports_to_upper_tor, # noqa F811 + skip_traffic_test): # noqa F811 + """ + Test case to validate the lag hash when there is lag member flapping. + The hash field to test is randomly chosen from the supported hash fields. + When hash field is in [DST_MAC, ETHERTYPE, VLAN_ID], need to re-configure the dut for L2 traffic. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ptfhost (AnsibleHost): Packet Test Framework (PTF) + tbinfo: testbed info fixture + mg_facts: minigraph facts + restore_configuration: fixture to restore the interface and vlan configurations after L2 test + restore_interfaces: fixture to restore the interfaces used in the test + ecmp_algorithm: randomly generated ecmp hash algorithm + lag_test_hash_fields: randomly generated lag hash field parameter + ipver: randomly generated outer frame ip version + inner_ipver: randomly generated inner frame ip version + encap_type: randomly generated encapsulation type + restore_vxlan_port: fixture to restore vxlan port to default + global_hash_capabilities: module level fixture to get the dut hash capabilities + """ + ecmp_algorithm, lag_test_hash_field, ipver, inner_ipver, encap_type = fine_params.split('-') + with allure.step('Randomly select an lag hash field to test ' + 'and configure all supported fields to the global ecmp and lag hash'): + # Get the interfaces for the test, downlink interface is selected randomly + uplink_interfaces, downlink_interfaces = get_interfaces_for_test(duthost, mg_facts, lag_test_hash_field) + # If the uplinks are not multi-member portchannels, skip the test + skip_single_member_lag_topology(uplink_interfaces, lag_test_hash_field, encap_type) + config_all_hash_fields(duthost, global_hash_capabilities) + lag_algorithm = get_diff_hash_algorithm(ecmp_algorithm, get_supported_hash_algorithms) + with allure.step(f'Configure ecmp hash algorithm: {ecmp_algorithm} - lag hash algorithm: {lag_algorithm}'): + config_all_hash_algorithm(duthost, ecmp_algorithm, lag_algorithm) + with allure.step("Check the config result"): + check_global_hash_config(duthost, global_hash_capabilities['ecmp'], global_hash_capabilities['lag']) + check_global_hash_algorithm(duthost, ecmp_algorithm, lag_algorithm) + with allure.step('Change topology for L2 test if hash field in DST_MAC, ETHERTYPE, VLAN_ID'): + # Need to send l2 traffic to validate SRC_MAC, DST_MAC, ETHERTYPE, VLAN_ID fields, changing topology is required + is_l2_test = False + if lag_test_hash_field in ['DST_MAC', 'ETHERTYPE', 'VLAN_ID']: + with allure.step('Change the topology for L2 test'): + # For l2 test, only one uplink portchannel interface is needed + is_l2_test = True + for _ in range(len(uplink_interfaces) - 1): + uplink_interfaces.popitem() + remove_ip_interface_and_config_vlan(duthost, mg_facts, tbinfo, downlink_interfaces[0], + uplink_interfaces, + lag_test_hash_field) + with allure.step('Prepare test parameters'): + ptf_params = generate_test_params( + duthost, tbinfo, mg_facts, lag_test_hash_field, ipver, inner_ipver, encap_type, uplink_interfaces, + downlink_interfaces, ecmp_hash=True, lag_hash=True, is_l2_test=is_l2_test) + if ptf_params.get('vxlan_port') and ptf_params['vxlan_port'] != DEFAULT_VXLAN_PORT: + config_custom_vxlan_port(duthost, ptf_params['vxlan_port']) + with allure.step('Start the ptf test, send traffic and check the balancing'): + # Check the default route before the ptf test + if not is_l2_test: + pytest_assert(check_default_route(duthost, uplink_interfaces.keys()), + 'The default route is not available or some nexthops are missing.') + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + with allure.step('Randomly select one member in each portchannel and flap them 3 times'): + # Randomly choose the members to flap + interfaces = [] + for portchannel in uplink_interfaces: + interface = random.choice(uplink_interfaces[portchannel]) + interfaces.append(interface) + # Flap the members 3 more times + flap_interfaces(duthost, interfaces, uplink_interfaces.keys(), times=3) + + if not is_l2_test: + with allure.step('Wait for the default route to recover'): + pytest_assert(wait_until(30, 5, 0, check_default_route, duthost, uplink_interfaces.keys()), + 'The default route is not available or some nexthops are missing.') + with allure.step('Start the ptf test, send traffic and check the balancing'): + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + +def test_lag_member_remove_add(duthost, tbinfo, ptfhost, fine_params, mg_facts, restore_configuration, # noqa F811 + restore_interfaces, global_hash_capabilities, restore_vxlan_port, # noqa F811 + get_supported_hash_algorithms, toggle_all_simulator_ports_to_upper_tor, # noqa F811 + skip_traffic_test): # noqa F811 + """ + Test case to validate the lag hash when a lag member is removed from the lag and added back for + a few times. + The hash field to test is randomly chosen from the supported hash fields. + When hash field is in [DST_MAC, ETHERTYPE, VLAN_ID], need to re-configure the dut for L2 traffic. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ptfhost (AnsibleHost): Packet Test Framework (PTF) + tbinfo: testbed info fixture + mg_facts: minigraph facts + restore_configuration: fixture to restore the interface and vlan configurations after L2 test + restore_interfaces: fixture to restore the interfaces used in the test + ecmp_algorithm: randomly generated ecmp hash algorithm + lag_test_hash_fields: randomly generated lag hash field parameter + ipver: randomly generated outer frame ip version + inner_ipver: randomly generated inner frame ip version + encap_type: randomly generated encapsulation type + restore_vxlan_port: fixture to restore vxlan port to default + global_hash_capabilities: module level fixture to get the dut hash capabilities + """ + ecmp_algorithm, lag_test_hash_field, ipver, inner_ipver, encap_type = fine_params.split('-') + with allure.step('Randomly select an lag hash field to test ' + 'and configure all supported fields to the global ecmp and lag hash'): + # Get the interfaces for the test, downlink interface is selected randomly + uplink_interfaces, downlink_interfaces = get_interfaces_for_test(duthost, mg_facts, lag_test_hash_field) + # If the uplinks are not multi-member portchannels, skip the test + skip_single_member_lag_topology(uplink_interfaces, lag_test_hash_field, encap_type) + config_all_hash_fields(duthost, global_hash_capabilities) + lag_algorithm = get_diff_hash_algorithm(ecmp_algorithm, get_supported_hash_algorithms) + with allure.step(f'Configure ecmp hash algorithm: {ecmp_algorithm} - lag hash algorithm: {lag_algorithm}'): + config_all_hash_algorithm(duthost, ecmp_algorithm, lag_algorithm) + with allure.step("Check the config result"): + check_global_hash_config(duthost, global_hash_capabilities['ecmp'], global_hash_capabilities['lag']) + check_global_hash_algorithm(duthost, ecmp_algorithm, lag_algorithm) + with allure.step('Change topology for L2 test if hash field in DST_MAC, ETHERTYPE, VLAN_ID'): + # Need to send l2 traffic to validate SRC_MAC, DST_MAC, ETHERTYPE, VLAN_ID fields, changing topology is required + is_l2_test = False + if lag_test_hash_field in ['DST_MAC', 'ETHERTYPE', 'VLAN_ID']: + with allure.step('Change the topology for L2 test'): + # For l2 test, only one uplink portchannel interface is needed + is_l2_test = True + for _ in range(len(uplink_interfaces) - 1): + uplink_interfaces.popitem() + remove_ip_interface_and_config_vlan(duthost, mg_facts, tbinfo, downlink_interfaces[0], + uplink_interfaces, + lag_test_hash_field) + with allure.step('Prepare test parameters'): + ptf_params = generate_test_params( + duthost, tbinfo, mg_facts, lag_test_hash_field, ipver, inner_ipver, encap_type, uplink_interfaces, + downlink_interfaces, ecmp_hash=True, lag_hash=True, is_l2_test=is_l2_test) + if ptf_params.get('vxlan_port') and ptf_params['vxlan_port'] != DEFAULT_VXLAN_PORT: + config_custom_vxlan_port(duthost, ptf_params['vxlan_port']) + with allure.step('Start the ptf test, send traffic and check the balancing'): + # Check the default route before the ptf test + if not is_l2_test: + pytest_assert(check_default_route(duthost, uplink_interfaces.keys()), + 'The default route is not available or some nexthops are missing.') + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + with allure.step('Randomly select one member in each portchannel and remove it from the lag and add it back'): + # Randomly choose the members to remove/add + for portchannel in uplink_interfaces: + interface = random.choice(uplink_interfaces[portchannel]) + remove_add_portchannel_member(duthost, interface, portchannel) + + if not is_l2_test: + with allure.step('Wait for the default route to recover'): + pytest_assert(wait_until(30, 5, 0, check_default_route, duthost, uplink_interfaces.keys()), + 'The default route is not available or some nexthops are missing.') + + with allure.step('Start the ptf test, send traffic and check the balancing'): + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + +def test_reboot(duthost, tbinfo, ptfhost, localhost, fine_params, mg_facts, restore_vxlan_port, # noqa F811 + global_hash_capabilities, reboot_type, get_supported_hash_algorithms, # noqa F811 + toggle_all_simulator_ports_to_upper_tor, skip_traffic_test): # noqa F811 + """ + Test case to validate the hash behavior after fast/warm/cold reboot. + The hash field to test is randomly chosen from the supported hash fields. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + ptfhost (AnsibleHost): Packet Test Framework (PTF) + mg_facts: minigraph facts + localhost: local host object + ecmp_algorithm: randomly generated ecmp hash algorithm + ecmp_test_hash_field: randomly generated ecmp hash field parameter + ipver: randomly generated outer frame ip version + inner_ipver: randomly generated inner frame ip version + encap_type: randomly generated encapsulation type + restore_vxlan_port: fixture to restore vxlan port to default + global_hash_capabilities: module level fixture to get the dut hash capabilities + """ + ecmp_algorithm, ecmp_test_hash_field, ipver, inner_ipver, encap_type = fine_params.split('-') + skip_unsupported_field_for_ecmp_test(ecmp_test_hash_field, encap_type) + with allure.step('Randomly select an ecmp hash field to test ' + 'and configure all supported fields to the global ecmp and lag hash'): + config_all_hash_fields(duthost, global_hash_capabilities) + lag_algorithm = get_diff_hash_algorithm(ecmp_algorithm, get_supported_hash_algorithms) + with allure.step(f'Configure ecmp hash algorithm: {ecmp_algorithm} - lag hash algorithm: {lag_algorithm}'): + config_all_hash_algorithm(duthost, ecmp_algorithm, lag_algorithm) + with allure.step("Check the config result"): + check_global_hash_config(duthost, global_hash_capabilities['ecmp'], global_hash_capabilities['lag']) + check_global_hash_algorithm(duthost, ecmp_algorithm, lag_algorithm) + with allure.step('Prepare test parameters'): + # Get the interfaces for the test, downlink interface is selected randomly + uplink_interfaces, downlink_interfaces = get_interfaces_for_test(duthost, mg_facts, ecmp_test_hash_field) + ptf_params = generate_test_params( + duthost, tbinfo, mg_facts, ecmp_test_hash_field, ipver, inner_ipver, encap_type, uplink_interfaces, + downlink_interfaces, ecmp_hash=True, lag_hash=True) + if ptf_params.get('vxlan_port') and ptf_params['vxlan_port'] != DEFAULT_VXLAN_PORT: + config_custom_vxlan_port(duthost, ptf_params['vxlan_port']) + with allure.step('Start the ptf test, send traffic and check the balancing'): + # Check the default route before the ptf test + pytest_assert(check_default_route(duthost, uplink_interfaces.keys()), + 'The default route is not available or some nexthops are missing.') + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + with allure.step(f'Randomly choose a reboot type: {reboot_type}, and reboot'): + # Save config if reboot type is config reload or cold reboot + if reboot_type in ['cold', 'reload']: + duthost.shell('config save -y') + # Reload/Reboot the dut + if reboot_type == 'reload': + config_reload(duthost, safe_reload=True, check_intf_up_ports=True) + else: + reboot(duthost, localhost, reboot_type=reboot_type) + # Wait for the dut to recover + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), + "Not all critical services are fully started.") + with allure.step('Check the generic hash config after the reboot'): + check_global_hash_config(duthost, global_hash_capabilities['ecmp'], global_hash_capabilities['lag']) + with allure.step('Check the route is established'): + pytest_assert(wait_until(60, 10, 0, check_default_route, duthost, uplink_interfaces.keys()), + "The default route is not established after the cold reboot.") + with allure.step('Start the ptf test, send traffic and check the balancing'): + if not skip_traffic_test: + ptf_runner( + ptfhost, + "ptftests", + "generic_hash_test.GenericHashTest", + platform_dir="ptftests", + params=ptf_params, + log_file=PTF_LOG_PATH, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + +@pytest.mark.disable_loganalyzer +def test_backend_error_messages(duthost, reload, global_hash_capabilities): # noqa:F811 + """ + Test case to validate there are backend errors printed in the syslog when + the hash config is removed or updated with invalid values via redis cli. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + reload: fixture to reload the configuration after the test + global_hash_capabilities: module level fixture to get the dut hash capabilities + """ + test_data = [ + {'info': 'Remove the ecmp_hash entry via redis cli and check if there is an error in the log', + 'command': "redis-cli -n 4 HDEL 'SWITCH_HASH|GLOBAL' 'ecmp_hash@'", + 'expected_regex': [ + 'ERR swss#orchagent:.*setSwitchHash: Failed to remove switch ECMP hash configuration: ' + 'operation is not supported.*', + # noqa:E501 + 'ERR swss#orchagent:.*doCfgSwitchHashTableTask: Failed to set switch hash: ASIC and CONFIG DB are ' + 'diverged.*']}, + # noqa:E501 + {'info': 'Remove the lag_hash entry via redis cli and check if there is an error in the log', + 'command': "redis-cli -n 4 HDEL 'SWITCH_HASH|GLOBAL' 'lag_hash@'", + 'expected_regex': [ + 'ERR swss#orchagent:.*setSwitchHash: Failed to remove switch LAG hash configuration: ' + 'operation is not supported.*', + # noqa:E501 + 'ERR swss#orchagent:.*doCfgSwitchHashTableTask: Failed to set switch hash: ASIC and CONFIG DB are ' + 'diverged.*']}, + # noqa:E501 + {'info': 'Remove the ecmp_hash_algorithm entry via redis cli and check if there is an error in the log', + 'command': "redis-cli -n 4 HDEL 'SWITCH_HASH|GLOBAL' 'ecmp_hash_algorithm'", + 'expected_regex': [ + 'ERR swss#orchagent:.*setSwitchHash: Failed to remove switch ECMP hash algorithm configuration: ' + 'operation is not supported.*', + # noqa:E501 + 'ERR swss#orchagent:.*doCfgSwitchHashTableTask: Failed to set switch hash: ASIC and CONFIG DB are ' + 'diverged.*']}, + # noqa:E501 + {'info': 'Remove the lag_hash_algorithm entry via redis cli and check if there is an error in the log', + 'command': "redis-cli -n 4 HDEL 'SWITCH_HASH|GLOBAL' 'lag_hash_algorithm'", + 'expected_regex': [ + 'ERR swss#orchagent:.*setSwitchHash: Failed to remove switch LAG hash algorithm configuration: ' + 'operation is not supported.*', + # noqa:E501 + 'ERR swss#orchagent:.*doCfgSwitchHashTableTask: Failed to set switch hash: ASIC and CONFIG DB are ' + 'diverged.*']}, + # noqa:E501 + {'info': 'Update the ecmp hash fields with an invalid value via redis cli and check if there ' + 'is an error in the log.', + 'command': "redis-cli -n 4 HSET 'SWITCH_HASH|GLOBAL' 'ecmp_hash@' 'INVALID_FIELD'", + 'expected_regex': [ + 'ERR swss#orchagent:.*parseSwHashFieldList: Failed to parse field\\(ecmp_hash\\): ' + 'invalid value\\(INVALID_FIELD\\).*']}, + # noqa:E501 + {'info': 'Update the lag hash fields with an invalid value via redis cli and check if there ' + 'is an error in the log.', + 'command': "redis-cli -n 4 HSET 'SWITCH_HASH|GLOBAL' 'lag_hash@' 'INVALID_FIELD'", + 'expected_regex': [ + 'ERR swss#orchagent:.*parseSwHashFieldList: Failed to parse field\\(lag_hash\\): ' + 'invalid value\\(INVALID_FIELD\\).*'] + # noqa:E501 + }, + {'info': 'Update the ecmp hash algorithm with an invalid value via redis cli and check if there ' + 'is an error in the log.', + 'command': "redis-cli -n 4 HSET 'SWITCH_HASH|GLOBAL' 'ecmp_hash_algorithm' 'INVALID_FIELD'", + 'expected_regex': [ + 'ERR swss#orchagent:.*parseSwHashAlgorithm: Failed to parse field\\(ecmp_hash_algorithm\\): ' + 'invalid value\\(INVALID_FIELD\\).*'] + # noqa:E501 + }, + {'info': 'Update the lag hash algorithm with an invalid value via redis cli and check if there ' + 'is an error in the log.', + 'command': "redis-cli -n 4 HSET 'SWITCH_HASH|GLOBAL' 'lag_hash_algorithm' 'INVALID_FIELD'", + 'expected_regex': [ + 'ERR swss#orchagent:.*parseSwHashAlgorithm: Failed to parse field\\(lag_hash_algorithm\\): ' + 'invalid value\\(INVALID_FIELD\\).*'] + # noqa:E501 + }, + {'info': 'Remove the SWITCH_HASH|GLOBAL key via redis cli and check if there is an error in the log.', + 'command': "redis-cli -n 4 DEL 'SWITCH_HASH|GLOBAL'", + 'expected_regex': [ + 'ERR swss#orchagent:.*doCfgSwitchHashTableTask: Failed to remove switch hash: ' + 'operation is not supported: ASIC and CONFIG DB are diverged.*'] + # noqa:E501 + } + ] + loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="test_backend_error_msgs:") + for item in test_data: + random_ecmp_algo = random.choice(global_hash_capabilities['ecmp_algo']) + random_lag_algo = random.choice(global_hash_capabilities['lag_algo']) + with allure.step('Configure all supported fields to the global ecmp and lag hash'): + config_all_hash_fields(duthost, global_hash_capabilities) + + with allure.step(f"Random chose algorithm: {random_ecmp_algo} from supported ecmp hash " + f"algorithms: {global_hash_capabilities['ecmp_algo']}"): + duthost.set_switch_hash_global_algorithm('ecmp', random_ecmp_algo) + + with allure.step(f"Random chose algorithm: {random_lag_algo} from supported lag hash " + f"algorithms: {global_hash_capabilities['lag_algo']}"): + duthost.set_switch_hash_global_algorithm('lag', random_lag_algo) + + with allure.step(item['info']): + loganalyzer.expect_regex = item['expected_regex'] + marker = loganalyzer.init() + duthost.shell(item['command']) + time.sleep(1) + loganalyzer.analyze(marker) + + +def test_algorithm_config(duthost, global_hash_capabilities): # noqa:F811 + """ + Test case to validate the hash algorithm configuration. + Args: + duthost (AnsibleHost): Device Under Test (DUT) + global_hash_capabilities: module level fixture to get the dut hash capabilities + """ + with allure.step('Test ECMP hash algorithm configuration'): + config_validate_algorithm(duthost, 'ecmp', global_hash_capabilities['ecmp_algo']) + with allure.step('Test LAG hash algorithm configuration'): + config_validate_algorithm(duthost, 'lag', global_hash_capabilities['lag_algo']) diff --git a/tests/iface_loopback_action/conftest.py b/tests/iface_loopback_action/conftest.py index 53a07410ba..8bd27216d9 100644 --- a/tests/iface_loopback_action/conftest.py +++ b/tests/iface_loopback_action/conftest.py @@ -183,7 +183,7 @@ def generate_ip_list(): @pytest.fixture(scope="module", autouse=True) def setup(duthost, ptfhost, orig_ports_configuration, ports_configuration, - backup_and_restore_config_db_package, nbrhosts, tbinfo): # noqa: F811 + backup_and_restore_config_db_package, nbrhosts, tbinfo, is_sonic_mlnx_leaf_fanout): # noqa: F811 """ Config: Cleanup the original port configuration and add new configurations before test Cleanup: restore the config on the VMs @@ -195,6 +195,9 @@ def setup(duthost, ptfhost, orig_ports_configuration, ports_configuration, :param nbrhosts: nbrhosts fixture. :param tbinfo: Testbed object """ + if is_sonic_mlnx_leaf_fanout: + pytest.skip("Not supporteds on Mellanox leaf-fanout running SONiC") + return peer_shutdown_ports = get_portchannel_peer_port_map(duthost, orig_ports_configuration, tbinfo, nbrhosts) remove_orig_dut_port_config(duthost, orig_ports_configuration) for vm_host, peer_ports in list(peer_shutdown_ports.items()): @@ -209,12 +212,28 @@ def setup(duthost, ptfhost, orig_ports_configuration, ports_configuration, @pytest.fixture(scope="module", autouse=True) -def recover(duthost, ptfhost, ports_configuration): +def recover(duthost, ptfhost, ports_configuration, is_sonic_mlnx_leaf_fanout): """ restore the original configurations :param duthost: DUT host object :param ptfhost: PTF host object :param ports_configuration: ports configuration parameters """ + if is_sonic_mlnx_leaf_fanout: + yield + return yield recover_config(duthost, ptfhost, ports_configuration) + + +@pytest.fixture(scope='module') +def is_sonic_mlnx_leaf_fanout(fanouthosts): + """ + The test sends QinQ packet for testing purpose. However, the QinQ packet will be dropped on leaf fanout + if it's running SONiC and Mellanox ASIC. + More info https://github.com/sonic-net/SONiC/blob/master/doc/tpid/SonicTPIDSettingHLD1.md + """ + for fanouthost in list(fanouthosts.values()): + if fanouthost.get_fanout_os() == 'sonic' and fanouthost.facts['asic_type'] == "mellanox": + return True + return False diff --git a/tests/ip/test_ip_packet.py b/tests/ip/test_ip_packet.py index 56b840a62d..ac0ad5ef30 100644 --- a/tests/ip/test_ip_packet.py +++ b/tests/ip/test_ip_packet.py @@ -1,6 +1,7 @@ import re import time import logging +import random import ipaddress import ptf.testutils as testutils @@ -11,7 +12,8 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.portstat_utilities import parse_column_positions from tests.common.portstat_utilities import parse_portstat -from tests.drop_packets.drop_packets import is_mellanox_fanout +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 +from tests.common.helpers.dut_utils import is_mellanox_fanout pytestmark = [ @@ -106,6 +108,12 @@ def parse_rif_counters(output_lines): return results + @staticmethod + def random_mac(): + return "02:00:00:%02x:%02x:%02x" % (random.randint(0, 255), + random.randint(0, 255), + random.randint(0, 255)) + @pytest.fixture(scope="class") def common_param(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo): duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] @@ -114,6 +122,7 @@ def common_param(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbin # generate peer_ip and port channel pair, be like:[("10.0.0.57", "PortChannel0001")] peer_ip_pc_pair = [(pc["peer_addr"], pc["attachto"]) for pc in mg_facts["minigraph_portchannel_interfaces"] if ipaddress.ip_address(pc['peer_addr']).version == 4] + # generate port channel and member ports pair, be like:{"PortChannel0001": ["Ethernet48", "Ethernet56"]} pc_ports_map = {pair[1]: mg_facts["minigraph_portchannels"][pair[1]]["members"] for pair in peer_ip_pc_pair} @@ -186,7 +195,7 @@ def common_param(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbin .format(prefix, selected_peer_ip_ifaces_pairs[1][0]), ptf_port_idx_namespace)) def test_forward_ip_packet_with_0x0000_chksum(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - ptfadapter, common_param): + ptfadapter, common_param, skip_traffic_test): # noqa F811 # GIVEN a ip packet with checksum 0x0000(compute from scratch) # WHEN send the packet to DUT # THEN DUT should forward it as normal ip packet @@ -242,6 +251,8 @@ def test_forward_ip_packet_with_0x0000_chksum(self, duthosts, enum_rand_one_per_ tx_drp = TestIPPacket.sum_ifaces_counts(portstat_out, out_ifaces, "tx_drp") tx_err = TestIPPacket.sum_ifaces_counts(rif_counter_out, out_rif_ifaces, "tx_err") if rif_support else 0 + if skip_traffic_test is True: + return pytest_assert(rx_ok >= self.PKT_NUM_MIN, "Received {} packets in rx, not in expected range".format(rx_ok)) pytest_assert(tx_ok >= self.PKT_NUM_MIN, @@ -255,7 +266,7 @@ def test_forward_ip_packet_with_0x0000_chksum(self, duthosts, enum_rand_one_per_ .format(tx_ok, match_cnt)) def test_forward_ip_packet_with_0xffff_chksum_tolerant(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - ptfadapter, common_param): + ptfadapter, common_param, skip_traffic_test): # noqa F811 # GIVEN a ip packet with checksum 0x0000(compute from scratch) # WHEN manually set checksum as 0xffff and send the packet to DUT # THEN DUT should tolerant packet with 0xffff, forward it as normal packet @@ -311,6 +322,8 @@ def test_forward_ip_packet_with_0xffff_chksum_tolerant(self, duthosts, enum_rand tx_drp = TestIPPacket.sum_ifaces_counts(portstat_out, out_ifaces, "tx_drp") tx_err = TestIPPacket.sum_ifaces_counts(rif_counter_out, out_rif_ifaces, "tx_err") if rif_support else 0 + if skip_traffic_test is True: + return pytest_assert(rx_ok >= self.PKT_NUM_MIN, "Received {} packets in rx, not in expected range".format(rx_ok)) pytest_assert(tx_ok >= self.PKT_NUM_MIN, @@ -325,7 +338,7 @@ def test_forward_ip_packet_with_0xffff_chksum_tolerant(self, duthosts, enum_rand def test_forward_ip_packet_with_0xffff_chksum_drop(self, duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, ptfadapter, - common_param, tbinfo): + common_param, tbinfo, skip_traffic_test): # noqa F811 # GIVEN a ip packet with checksum 0x0000(compute from scratch) # WHEN manually set checksum as 0xffff and send the packet to DUT @@ -391,6 +404,8 @@ def test_forward_ip_packet_with_0xffff_chksum_drop(self, duthosts, localhost, logger.info("Setting PKT_NUM_ZERO for t2 max topology with 0.2 tolerance") self.PKT_NUM_ZERO = self.PKT_NUM * 0.2 + if skip_traffic_test is True: + return pytest_assert(rx_ok >= self.PKT_NUM_MIN, "Received {} packets in rx, not in expected range".format(rx_ok)) pytest_assert(max(rx_drp, rx_err) >= self.PKT_NUM_MIN, @@ -404,7 +419,7 @@ def test_forward_ip_packet_with_0xffff_chksum_drop(self, duthosts, localhost, .format(match_cnt)) def test_forward_ip_packet_recomputed_0xffff_chksum(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - ptfadapter, common_param): + ptfadapter, common_param, skip_traffic_test): # noqa F811 # GIVEN a ip packet, after forwarded(ttl-1) by DUT, # it's checksum will be 0xffff after wrongly incrementally recomputed # ref to https://datatracker.ietf.org/doc/html/rfc1624 @@ -462,6 +477,8 @@ def test_forward_ip_packet_recomputed_0xffff_chksum(self, duthosts, enum_rand_on tx_drp = TestIPPacket.sum_ifaces_counts(portstat_out, out_ifaces, "tx_drp") tx_err = TestIPPacket.sum_ifaces_counts(rif_counter_out, out_rif_ifaces, "tx_err") if rif_support else 0 + if skip_traffic_test is True: + return pytest_assert(rx_ok >= self.PKT_NUM_MIN, "Received {} packets in rx, not in expected range".format(rx_ok)) pytest_assert(tx_ok >= self.PKT_NUM_MIN, @@ -475,7 +492,7 @@ def test_forward_ip_packet_recomputed_0xffff_chksum(self, duthosts, enum_rand_on .format(tx_ok, match_cnt)) def test_forward_ip_packet_recomputed_0x0000_chksum(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - ptfadapter, common_param): + ptfadapter, common_param, skip_traffic_test): # noqa F811 # GIVEN a ip packet, after forwarded(ttl-1) by DUT, it's checksum will be 0x0000 after recompute from scratch # WHEN send the packet to DUT # THEN DUT recompute new checksum as 0x0000 and forward packet as expected. @@ -530,6 +547,8 @@ def test_forward_ip_packet_recomputed_0x0000_chksum(self, duthosts, enum_rand_on tx_drp = TestIPPacket.sum_ifaces_counts(portstat_out, out_ifaces, "tx_drp") tx_err = TestIPPacket.sum_ifaces_counts(rif_counter_out, out_rif_ifaces, "tx_err") if rif_support else 0 + if skip_traffic_test is True: + return pytest_assert(rx_ok >= self.PKT_NUM_MIN, "Received {} packets in rx, not in expected range".format(rx_ok)) pytest_assert(tx_ok >= self.PKT_NUM_MIN, @@ -543,7 +562,7 @@ def test_forward_ip_packet_recomputed_0x0000_chksum(self, duthosts, enum_rand_on .format(tx_ok, match_cnt)) def test_forward_normal_ip_packet(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - ptfadapter, common_param): + ptfadapter, common_param, skip_traffic_test): # noqa F811 # GIVEN a random normal ip packet # WHEN send the packet to DUT # THEN DUT should forward it as normal ip packet, nothing change but ttl-1 @@ -591,6 +610,8 @@ def test_forward_normal_ip_packet(self, duthosts, enum_rand_one_per_hwsku_fronte tx_drp = TestIPPacket.sum_ifaces_counts(portstat_out, out_ifaces, "tx_drp") tx_err = TestIPPacket.sum_ifaces_counts(rif_counter_out, out_rif_ifaces, "tx_err") if rif_support else 0 + if skip_traffic_test is True: + return pytest_assert(rx_ok >= self.PKT_NUM_MIN, "Received {} packets in rx, not in expected range".format(rx_ok)) pytest_assert(tx_ok >= self.PKT_NUM_MIN, @@ -604,7 +625,7 @@ def test_forward_normal_ip_packet(self, duthosts, enum_rand_one_per_hwsku_fronte .format(tx_ok, match_cnt)) def test_drop_ip_packet_with_wrong_0xffff_chksum(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, - ptfadapter, common_param): + ptfadapter, common_param, skip_traffic_test): # noqa F811 # GIVEN a random normal ip packet, and manually modify checksum to 0xffff # WHEN send the packet to DUT # THEN DUT should drop it and add drop count @@ -645,6 +666,8 @@ def test_drop_ip_packet_with_wrong_0xffff_chksum(self, duthosts, enum_rand_one_p tx_err = TestIPPacket.sum_ifaces_counts(rif_counter_out, out_rif_ifaces, "tx_err") if rif_support else 0 asic_type = duthost.facts['asic_type'] + if skip_traffic_test is True: + return pytest_assert(rx_ok >= self.PKT_NUM_MIN, "Received {} packets in rx, not in expected range".format(rx_ok)) pytest_assert(max(rx_drp, rx_err) >= self.PKT_NUM_MIN if asic_type not in ["marvell"] else True, @@ -653,3 +676,58 @@ def test_drop_ip_packet_with_wrong_0xffff_chksum(self, duthosts, enum_rand_one_p "Forwarded {} packets in tx, not in expected range".format(tx_ok)) pytest_assert(max(tx_drp, tx_err) <= self.PKT_NUM_ZERO, "Dropped {} packets in tx, not in expected range".format(tx_err)) + + def test_drop_l3_ip_packet_non_dut_mac(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, + ptfadapter, common_param, skip_traffic_test): # noqa F811 + # GIVEN a random normal ip packet, and random dest mac address + # WHEN send the packet to DUT with dst_mac != ingress_router_mac to a layer 3 interface + # THEN DUT should drop it and add drop count + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + (peer_ip_ifaces_pair, rif_rx_ifaces, rif_support, ptf_port_idx, + pc_ports_map, _, ingress_router_mac) = common_param + + dst_mac = TestIPPacket.random_mac() + while dst_mac == ingress_router_mac: + dst_mac = TestIPPacket.random_mac() + + pkt = testutils.simple_ip_packet( + eth_dst=dst_mac, + eth_src=ptfadapter.dataplane.get_mac(0, ptf_port_idx), + ip_src=peer_ip_ifaces_pair[0][0], + ip_dst=peer_ip_ifaces_pair[1][0]) + + out_rif_ifaces, out_ifaces = TestIPPacket.parse_interfaces( + duthost.command("show ip route %s" % peer_ip_ifaces_pair[1][0])["stdout_lines"], pc_ports_map) + + duthost.command("portstat -c") + if rif_support: + duthost.command("sonic-clear rifcounters") + ptfadapter.dataplane.flush() + + testutils.send(ptfadapter, ptf_port_idx, pkt, self.PKT_NUM) + time.sleep(5) + + portstat_out = parse_portstat(duthost.command("portstat")["stdout_lines"]) + if rif_support: + rif_counter_out = TestIPPacket.parse_rif_counters( + duthost.command("show interfaces counters rif")["stdout_lines"]) + + # rx_ok counter to increase to show packets are being received correctly at layer 2 + # rx_drp counter to increase to show packets are being dropped + # tx_ok, tx_drop, tx_err counter to zero to show no packets are being forwarded + rx_ok = int(portstat_out[peer_ip_ifaces_pair[0][1][0]]["rx_ok"].replace(",", "")) + rx_drp = int(portstat_out[peer_ip_ifaces_pair[0][1][0]]["rx_drp"].replace(",", "")) + tx_ok = TestIPPacket.sum_ifaces_counts(portstat_out, out_ifaces, "tx_ok") + tx_drp = TestIPPacket.sum_ifaces_counts(portstat_out, out_ifaces, "tx_drp") + tx_rif_err = TestIPPacket.sum_ifaces_counts(rif_counter_out, out_rif_ifaces, "tx_err") if rif_support else 0 + + if skip_traffic_test is True: + return + pytest_assert(rx_ok >= self.PKT_NUM_MIN, + "Received {} packets in rx, not in expected range".format(rx_ok)) + pytest_assert(rx_drp >= self.PKT_NUM_MIN, + "Dropped {} packets in rx, not in expected range".format(rx_drp)) + pytest_assert(tx_ok <= self.PKT_NUM_ZERO, + "Forwarded {} packets in tx, not in expected range".format(tx_ok)) + pytest_assert(max(tx_drp, tx_rif_err) <= self.PKT_NUM_ZERO, + "Dropped {} packets in tx, tx_rif_err {}, not in expected range".format(tx_drp, tx_rif_err)) diff --git a/tests/ip/test_mgmt_ipv6_only.py b/tests/ip/test_mgmt_ipv6_only.py index db94ec7e58..86e2b4dced 100644 --- a/tests/ip/test_mgmt_ipv6_only.py +++ b/tests/ip/test_mgmt_ipv6_only.py @@ -3,19 +3,15 @@ import pytest import re -from tests.common.utilities import get_mgmt_ipv6 -from tests.common.helpers.assertions import pytest_assert -from tests.tacacs.utils import check_output -from tests.bgp.test_bgp_fact import run_bgp_facts -from tests.test_features import run_show_features -from tests.tacacs.test_ro_user import ssh_remote_run_retry -from tests.ntp.test_ntp import run_ntp, setup_ntp_func # noqa F401 -from tests.common.helpers.assertions import pytest_require -from tests.tacacs.conftest import tacacs_creds, check_tacacs_v6_func # noqa F401 -from tests.syslog.test_syslog import run_syslog, check_default_route # noqa F401 -from tests.common.fixtures.duthost_utils import convert_and_restore_config_db_to_ipv6_only # noqa F401 +from tests.common.utilities import get_mgmt_ipv6, check_output, run_show_features +from tests.common.helpers.assertions import pytest_assert, pytest_require +from tests.common.helpers.bgp import run_bgp_facts +from tests.common.helpers.tacacs.tacacs_helper import ssh_remote_run_retry +from tests.common.helpers.ntp_helper import run_ntp +from tests.common.helpers.syslog_helpers import run_syslog, check_default_route # noqa F401 from tests.common.helpers.gnmi_utils import GNMIEnvironment -from tests.telemetry.conftest import gnxi_path, setup_streaming_telemetry_func # noqa F401 +from tests.common.fixtures.duthost_utils import convert_and_restore_config_db_to_ipv6_only # noqa F401 + pytestmark = [ pytest.mark.disable_loganalyzer, @@ -62,6 +58,28 @@ def log_eth0_interface_info(duthosts): logging.debug(f"Checking host[{duthost.hostname}] ifconfig eth0:[{duthost_interface}] after fixture") +def log_tacacs(duthosts, ptfhost): + for duthost in duthosts: + # Print debug info for ipv6 pingability + ptfhost_vars = ptfhost.host.options['inventory_manager'].get_host(ptfhost.hostname).vars + if 'ansible_hostv6' in ptfhost_vars: + tacacs_server_ip = ptfhost_vars['ansible_hostv6'] + ping_result = duthost.shell(f"ping {tacacs_server_ip} -c 1 -W 3", module_ignore_errors=True)["stdout"] + logging.debug(f"Checking ping_result [{ping_result}]") + + # Print debug info for mgmt interfaces and forced mgmt routes + mgmt_interface_keys = duthost.command("sonic-db-cli CONFIG_DB keys 'MGMT_INTERFACE|*'")['stdout'] + logging.debug(f"mgmt_interface_keys: {mgmt_interface_keys}") + for intf_key in mgmt_interface_keys.split('\n'): + logging.debug(f"interface key: {intf_key}") + intf_values = intf_key.split('|') + if len(intf_values) != 3: + logging.debug(f"Unexpected interface key: {intf_key}") + continue + forced_mgmt_rte = duthost.command(f"sonic-db-cli CONFIG_DB HGET '{intf_key}' forced_mgmt_routes@")['stdout'] + logging.debug(f"forced_mgmt_routes: {forced_mgmt_rte}, interface address: {intf_values[2]}") + + def test_bgp_facts_ipv6_only(duthosts, enum_frontend_dut_hostname, enum_asic_index, convert_and_restore_config_db_to_ipv6_only): # noqa F811 # Add a temporary debug log to see if DUTs are reachable via IPv6 mgmt-ip. Will remove later @@ -134,6 +152,7 @@ def test_ro_user_ipv6_only(localhost, ptfhost, duthosts, enum_rand_one_per_hwsku log_eth0_interface_info(duthosts) duthost = duthosts[enum_rand_one_per_hwsku_hostname] dutipv6 = get_mgmt_ipv6(duthost) + log_tacacs(duthosts, ptfhost) res = ssh_remote_run_retry(localhost, dutipv6, ptfhost, tacacs_creds['tacacs_ro_user'], tacacs_creds['tacacs_ro_user_passwd'], 'cat /etc/passwd') @@ -148,6 +167,7 @@ def test_rw_user_ipv6_only(localhost, ptfhost, duthosts, enum_rand_one_per_hwsku log_eth0_interface_info(duthosts) duthost = duthosts[enum_rand_one_per_hwsku_hostname] dutipv6 = get_mgmt_ipv6(duthost) + log_tacacs(duthosts, ptfhost) res = ssh_remote_run_retry(localhost, dutipv6, ptfhost, tacacs_creds['tacacs_rw_user'], tacacs_creds['tacacs_rw_user_passwd'], "cat /etc/passwd") diff --git a/tests/ipfwd/test_dir_bcast.py b/tests/ipfwd/test_dir_bcast.py index 1c462271bb..0e07ce1811 100644 --- a/tests/ipfwd/test_dir_bcast.py +++ b/tests/ipfwd/test_dir_bcast.py @@ -2,7 +2,7 @@ import json import logging -from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory, skip_traffic_test # noqa F401 from tests.ptf_runner import ptf_runner from datetime import datetime from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F401 @@ -65,7 +65,7 @@ def ptf_test_port_map(duthost, ptfhost, mg_facts, testbed_type, tbinfo): def test_dir_bcast(duthosts, rand_one_dut_hostname, ptfhost, tbinfo, - toggle_all_simulator_ports_to_rand_selected_tor_m): # noqa F811 + toggle_all_simulator_ports_to_rand_selected_tor_m, skip_traffic_test): # noqa F811 duthost = duthosts[rand_one_dut_hostname] testbed_type = tbinfo['topo']['name'] @@ -81,6 +81,8 @@ def test_dir_bcast(duthosts, rand_one_dut_hostname, ptfhost, tbinfo, 'ptf_test_port_map': PTF_TEST_PORT_MAP } log_file = "/tmp/dir_bcast.BcastTest.{}.log".format(datetime.now().strftime("%Y-%m-%d-%H:%M:%S")) + if skip_traffic_test is True: + return ptf_runner( ptfhost, 'ptftests', diff --git a/tests/ipfwd/test_nhop_group.py b/tests/ipfwd/test_nhop_group.py index b7261c5d5c..01a6162a06 100644 --- a/tests/ipfwd/test_nhop_group.py +++ b/tests/ipfwd/test_nhop_group.py @@ -16,8 +16,7 @@ from tests.common.mellanox_data import is_mellanox_device, get_chip_type from tests.common.innovium_data import is_innovium_device from tests.common.utilities import wait_until -from tests.platform_tests.link_flap.link_flap_utils import toggle_one_link -from tests.common.platform.device_utils import fanout_switch_port_lookup +from tests.common.platform.device_utils import fanout_switch_port_lookup, toggle_one_link CISCO_NHOP_GROUP_FILL_PERCENTAGE = 0.92 @@ -595,27 +594,27 @@ def built_and_send_tcp_ip_packet(): 45: 'c0:ff:ee:00:00:0c', 46: 'c0:ff:ee:00:00:0d', 47: 'c0:ff:ee:00:00:0b', 48: 'c0:ff:ee:00:00:11', 49: 'c0:ff:ee:00:00:0f'} - gb_asic_flow_map = {0: 'c0:ff:ee:00:00:0f', 1: 'c0:ff:ee:00:00:10', - 2: 'c0:ff:ee:00:00:0e', 3: 'c0:ff:ee:00:00:0f', 4: 'c0:ff:ee:00:00:11', - 5: 'c0:ff:ee:00:00:0f', 6: 'c0:ff:ee:00:00:12', - 7: 'c0:ff:ee:00:00:0c', 8: 'c0:ff:ee:00:00:0e', 9: 'c0:ff:ee:00:00:10', - 10: 'c0:ff:ee:00:00:11', 11: 'c0:ff:ee:00:00:0f', - 12: 'c0:ff:ee:00:00:0c', 13: 'c0:ff:ee:00:00:0f', - 14: 'c0:ff:ee:00:00:11', - 15: 'c0:ff:ee:00:00:0c', 16: 'c0:ff:ee:00:00:0e', - 17: 'c0:ff:ee:00:00:11', 18: 'c0:ff:ee:00:00:11', 19: 'c0:ff:ee:00:00:0c', - 20: 'c0:ff:ee:00:00:10', 21: 'c0:ff:ee:00:00:0b', - 22: 'c0:ff:ee:00:00:0d', 23: 'c0:ff:ee:00:00:10', 24: 'c0:ff:ee:00:00:12', - 25: 'c0:ff:ee:00:00:11', 26: 'c0:ff:ee:00:00:11', - 27: 'c0:ff:ee:00:00:0c', 28: 'c0:ff:ee:00:00:11', 29: 'c0:ff:ee:00:00:0c', - 30: 'c0:ff:ee:00:00:12', 31: 'c0:ff:ee:00:00:10', - 32: 'c0:ff:ee:00:00:11', 33: 'c0:ff:ee:00:00:0c', 34: 'c0:ff:ee:00:00:0c', - 35: 'c0:ff:ee:00:00:0b', 36: 'c0:ff:ee:00:00:0d', - 37: 'c0:ff:ee:00:00:10', 38: 'c0:ff:ee:00:00:0e', 39: 'c0:ff:ee:00:00:0d', - 40: 'c0:ff:ee:00:00:0e', 41: 'c0:ff:ee:00:00:11', - 42: 'c0:ff:ee:00:00:11', 43: 'c0:ff:ee:00:00:0c', 44: 'c0:ff:ee:00:00:0e', - 45: 'c0:ff:ee:00:00:0f', 46: 'c0:ff:ee:00:00:0f', - 47: 'c0:ff:ee:00:00:0c', 48: 'c0:ff:ee:00:00:0e', 49: 'c0:ff:ee:00:00:10'} + gb_asic_flow_map = {0: 'c0:ff:ee:00:00:0b', 1: 'c0:ff:ee:00:00:11', + 2: 'c0:ff:ee:00:00:0c', 3: 'c0:ff:ee:00:00:0d', 4: 'c0:ff:ee:00:00:10', + 5: 'c0:ff:ee:00:00:0b', 6: 'c0:ff:ee:00:00:0c', + 7: 'c0:ff:ee:00:00:0d', 8: 'c0:ff:ee:00:00:0f', 9: 'c0:ff:ee:00:00:12', + 10: 'c0:ff:ee:00:00:0b', 11: 'c0:ff:ee:00:00:10', + 12: 'c0:ff:ee:00:00:0d', 13: 'c0:ff:ee:00:00:0c', + 14: 'c0:ff:ee:00:00:12', + 15: 'c0:ff:ee:00:00:0c', 16: 'c0:ff:ee:00:00:11', + 17: 'c0:ff:ee:00:00:12', 18: 'c0:ff:ee:00:00:0f', 19: 'c0:ff:ee:00:00:12', + 20: 'c0:ff:ee:00:00:0f', 21: 'c0:ff:ee:00:00:0d', + 22: 'c0:ff:ee:00:00:0f', 23: 'c0:ff:ee:00:00:12', 24: 'c0:ff:ee:00:00:0e', + 25: 'c0:ff:ee:00:00:0b', 26: 'c0:ff:ee:00:00:0d', + 27: 'c0:ff:ee:00:00:0f', 28: 'c0:ff:ee:00:00:12', 29: 'c0:ff:ee:00:00:10', + 30: 'c0:ff:ee:00:00:11', 31: 'c0:ff:ee:00:00:12', + 32: 'c0:ff:ee:00:00:0b', 33: 'c0:ff:ee:00:00:0c', 34: 'c0:ff:ee:00:00:0b', + 35: 'c0:ff:ee:00:00:0e', 36: 'c0:ff:ee:00:00:0d', + 37: 'c0:ff:ee:00:00:0e', 38: 'c0:ff:ee:00:00:0b', 39: 'c0:ff:ee:00:00:0d', + 40: 'c0:ff:ee:00:00:10', 41: 'c0:ff:ee:00:00:12', + 42: 'c0:ff:ee:00:00:12', 43: 'c0:ff:ee:00:00:11', 44: 'c0:ff:ee:00:00:0e', + 45: 'c0:ff:ee:00:00:0b', 46: 'c0:ff:ee:00:00:12', + 47: 'c0:ff:ee:00:00:0d', 48: 'c0:ff:ee:00:00:0c', 49: 'c0:ff:ee:00:00:0f'} td2_asic_flow_map = {0: 'c0:ff:ee:00:00:12', 1: 'c0:ff:ee:00:00:10', 2: 'c0:ff:ee:00:00:11', @@ -697,35 +696,35 @@ def built_and_send_tcp_ip_packet(): 45: 'c0:ff:ee:00:00:0c', 46: 'c0:ff:ee:00:00:0d', 47: 'c0:ff:ee:00:00:0b', 48: 'c0:ff:ee:00:00:11', 49: 'c0:ff:ee:00:00:0f'} - gr_asic_flow_map = {0: 'c0:ff:ee:00:00:12', 1: 'c0:ff:ee:00:00:10', - 2: 'c0:ff:ee:00:00:0c', - 3: 'c0:ff:ee:00:00:0b', 4: 'c0:ff:ee:00:00:0b', - 5: 'c0:ff:ee:00:00:0b', 6: 'c0:ff:ee:00:00:11', - 7: 'c0:ff:ee:00:00:12', 8: 'c0:ff:ee:00:00:0d', - 9: 'c0:ff:ee:00:00:0c', - 10: 'c0:ff:ee:00:00:0f', 11: 'c0:ff:ee:00:00:0e', - 12: 'c0:ff:ee:00:00:11', 13: 'c0:ff:ee:00:00:10', - 14: 'c0:ff:ee:00:00:0b', - 15: 'c0:ff:ee:00:00:12', 16: 'c0:ff:ee:00:00:0b', - 17: 'c0:ff:ee:00:00:12', 18: 'c0:ff:ee:00:00:11', - 19: 'c0:ff:ee:00:00:10', 20: 'c0:ff:ee:00:00:10', - 21: 'c0:ff:ee:00:00:11', 22: 'c0:ff:ee:00:00:12', - 23: 'c0:ff:ee:00:00:0b', 24: 'c0:ff:ee:00:00:0c', - 25: 'c0:ff:ee:00:00:0d', - 26: 'c0:ff:ee:00:00:0e', 27: 'c0:ff:ee:00:00:0f', - 28: 'c0:ff:ee:00:00:10', 29: 'c0:ff:ee:00:00:11', - 30: 'c0:ff:ee:00:00:12', 31: 'c0:ff:ee:00:00:0b', - 32: 'c0:ff:ee:00:00:12', 33: 'c0:ff:ee:00:00:0b', - 34: 'c0:ff:ee:00:00:10', - 35: 'c0:ff:ee:00:00:11', 36: 'c0:ff:ee:00:00:11', - 37: 'c0:ff:ee:00:00:10', 38: 'c0:ff:ee:00:00:0b', - 39: 'c0:ff:ee:00:00:12', - 40: 'c0:ff:ee:00:00:0e', 41: 'c0:ff:ee:00:00:10', - 42: 'c0:ff:ee:00:00:0d', 43: 'c0:ff:ee:00:00:0e', - 44: 'c0:ff:ee:00:00:0b', 45: 'c0:ff:ee:00:00:0c', - 46: 'c0:ff:ee:00:00:11', - 47: 'c0:ff:ee:00:00:11', 48: 'c0:ff:ee:00:00:11', - 49: 'c0:ff:ee:00:00:11'} + gr_asic_flow_map = {0: 'c0:ff:ee:00:00:0b', 1: 'c0:ff:ee:00:00:0c', + 2: 'c0:ff:ee:00:00:0d', + 3: 'c0:ff:ee:00:00:0b', 4: 'c0:ff:ee:00:00:12', + 5: 'c0:ff:ee:00:00:0e', 6: 'c0:ff:ee:00:00:0f', + 7: 'c0:ff:ee:00:00:10', 8: 'c0:ff:ee:00:00:0b', + 9: 'c0:ff:ee:00:00:0d', + 10: 'c0:ff:ee:00:00:0c', 11: 'c0:ff:ee:00:00:0b', + 12: 'c0:ff:ee:00:00:10', 13: 'c0:ff:ee:00:00:11', + 14: 'c0:ff:ee:00:00:11', + 15: 'c0:ff:ee:00:00:0e', 16: 'c0:ff:ee:00:00:0f', + 17: 'c0:ff:ee:00:00:10', 18: 'c0:ff:ee:00:00:12', + 19: 'c0:ff:ee:00:00:0e', 20: 'c0:ff:ee:00:00:0d', + 21: 'c0:ff:ee:00:00:0b', 22: 'c0:ff:ee:00:00:0b', + 23: 'c0:ff:ee:00:00:0c', 24: 'c0:ff:ee:00:00:11', + 25: 'c0:ff:ee:00:00:0e', + 26: 'c0:ff:ee:00:00:10', 27: 'c0:ff:ee:00:00:11', + 28: 'c0:ff:ee:00:00:11', 29: 'c0:ff:ee:00:00:0f', + 30: 'c0:ff:ee:00:00:0e', 31: 'c0:ff:ee:00:00:11', + 32: 'c0:ff:ee:00:00:10', 33: 'c0:ff:ee:00:00:0f', + 34: 'c0:ff:ee:00:00:0f', + 35: 'c0:ff:ee:00:00:12', 36: 'c0:ff:ee:00:00:0b', + 37: 'c0:ff:ee:00:00:0e', 38: 'c0:ff:ee:00:00:0c', + 39: 'c0:ff:ee:00:00:0b', + 40: 'c0:ff:ee:00:00:12', 41: 'c0:ff:ee:00:00:0e', + 42: 'c0:ff:ee:00:00:0f', 43: 'c0:ff:ee:00:00:10', + 44: 'c0:ff:ee:00:00:0c', 45: 'c0:ff:ee:00:00:0c', + 46: 'c0:ff:ee:00:00:12', + 47: 'c0:ff:ee:00:00:0d', 48: 'c0:ff:ee:00:00:0c', + 49: 'c0:ff:ee:00:00:0b'} spc_asic_flow_map = {0: 'c0:ff:ee:00:00:0b', 1: 'c0:ff:ee:00:00:12', 2: 'c0:ff:ee:00:00:0e', 3: 'c0:ff:ee:00:00:0f', 4: 'c0:ff:ee:00:00:10', 5: 'c0:ff:ee:00:00:0d', @@ -752,9 +751,10 @@ def built_and_send_tcp_ip_packet(): # Fill this array after first run of test case which will give neighbor selected SUPPORTED_ASIC_TO_NEXTHOP_SELECTED_MAP = {"th": th_asic_flow_map, "gb": gb_asic_flow_map, "gblc": gb_asic_flow_map, "td2": td2_asic_flow_map, "th2": th2_asic_flow_map, - "td3": td3_asic_flow_map, "gr": gr_asic_flow_map, - "spc1": spc_asic_flow_map, "spc2": spc_asic_flow_map, - "spc3": spc_asic_flow_map, "spc4": spc_asic_flow_map} + "th4": th_asic_flow_map, "td3": td3_asic_flow_map, + "gr": gr_asic_flow_map, "spc1": spc_asic_flow_map, + "spc2": spc_asic_flow_map, "spc3": spc_asic_flow_map, + "spc4": spc_asic_flow_map} vendor = duthost.facts["asic_type"] hostvars = duthost.host.options['variable_manager']._hostvars[duthost.hostname] @@ -771,12 +771,14 @@ def built_and_send_tcp_ip_packet(): pytest_assert(dutAsic, "Please add ASIC in the SUPPORTED_ASIC_TO_NEXTHOP_SELECTED_MAP \ list and update the asic to nexthop mapping") for flow_count, nexthop_selected in recvd_pkt_result.items(): + logger.info("dutAsic: {} Flow count {}, received nexthop {}, expected nexthop {}".format( + dutAsic, flow_count, nexthop_selected, nexthop_map[flow_count])) pytest_assert(nexthop_map[flow_count] in nexthop_selected, "Flow {} is not picking expected Neighbor".format(flow_count)) -def test_nhop_group_interface_flap(duthost, tbinfo, ptfadapter, gather_facts, - enum_rand_one_frontend_asic_index, fanouthosts): +def test_nhop_group_interface_flap(duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo, ptfadapter, + gather_facts, enum_rand_one_frontend_asic_index, fanouthosts): """ Test for packet drop when route is added with ECMP and all ECMP member's interfaces are down. Use kernel flag 'arp_evict_nocarrier' to disable ARP @@ -785,6 +787,7 @@ def test_nhop_group_interface_flap(duthost, tbinfo, ptfadapter, gather_facts, Nexthop members. Without this kernel flag, static route addition fails when Nexthop ARP entries are not resolved. """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] asic = duthost.asic_instance(enum_rand_one_frontend_asic_index) # Check Gather facts IP Interface is active one @@ -844,7 +847,7 @@ def test_nhop_group_interface_flap(duthost, tbinfo, ptfadapter, gather_facts, gather_facts['src_port'][i]) logger.debug("No Shut fanout sw: %s, port: %s", fanout, fanout_port) fanout.no_shutdown(fanout_port) - time.sleep(10) + time.sleep(20) duthost.shell("portstat -c") ptfadapter.dataplane.flush() testutils.send(ptfadapter, gather_facts['dst_port_ids'][0], pkt, pkt_count) diff --git a/tests/ixia/ecn/test_red_accuracy.py b/tests/ixia/ecn/test_red_accuracy.py index 670772ce71..68b80b58b0 100644 --- a/tests/ixia/ecn/test_red_accuracy.py +++ b/tests/ixia/ecn/test_red_accuracy.py @@ -12,6 +12,7 @@ pytestmark = [pytest.mark.topology('tgen')] + def test_red_accuracy(request, ixia_api, ixia_testbed_config, conn_graph_facts, # noqa F811 fanout_graph_facts, duthosts, localhost, # noqa F811 rand_one_dut_hostname, rand_one_dut_portname_oper_up, diff --git a/tests/ixia/ixanvl/test_bgp_conformance.py b/tests/ixia/ixanvl/test_bgp_conformance.py index 4b6aa510e2..1c8f36a245 100644 --- a/tests/ixia/ixanvl/test_bgp_conformance.py +++ b/tests/ixia/ixanvl/test_bgp_conformance.py @@ -5,7 +5,10 @@ from tests.common.fixtures.conn_graph_facts import conn_graph_facts # noqa F401 from scp import SCPClient -pytestmark = [pytest.mark.disable_loganalyzer] +pytestmark = [ + pytest.mark.topology('tgen'), + pytest.mark.disable_loganalyzer +] @pytest.fixture() diff --git a/tests/ixia/pfc/test_global_pause.py b/tests/ixia/pfc/test_global_pause.py index b9e9503017..e5c79954d1 100644 --- a/tests/ixia/pfc/test_global_pause.py +++ b/tests/ixia/pfc/test_global_pause.py @@ -11,6 +11,7 @@ pytestmark = [pytest.mark.topology('tgen')] + def test_global_pause(ixia_api, ixia_testbed_config, conn_graph_facts, fanout_graph_facts, # noqa F811 duthosts, rand_one_dut_hostname, rand_one_dut_portname_oper_up, lossless_prio_list, lossy_prio_list, prio_dscp_map): # noqa F811 diff --git a/tests/ixia/test_ixia_traffic.py b/tests/ixia/test_ixia_traffic.py index 0230245227..bab8ea7a87 100644 --- a/tests/ixia/test_ixia_traffic.py +++ b/tests/ixia/test_ixia_traffic.py @@ -26,8 +26,12 @@ get_peer_ixia_chassis -@pytest.mark.disable_loganalyzer -@pytest.mark.topology("tgen") +pytestmark = [ + pytest.mark.topology('tgen'), + pytest.mark.disable_loganalyzer +] + + def test_testbed(conn_graph_facts, duthosts, rand_one_dut_hostname, fanout_graph_facts, # noqa F811 ixia_api_server_session, fanouthosts): # noqa F811 duthost = duthosts[rand_one_dut_hostname] diff --git a/tests/ixia/test_tgen.py b/tests/ixia/test_tgen.py index d64c6faafd..ac48a2da1e 100644 --- a/tests/ixia/test_tgen.py +++ b/tests/ixia/test_tgen.py @@ -19,8 +19,12 @@ from abstract_open_traffic_generator.result import FlowRequest -@pytest.mark.topology("tgen") -@pytest.mark.disable_loganalyzer +pytestmark = [ + pytest.mark.topology('tgen'), + pytest.mark.disable_loganalyzer +] + + def __gen_all_to_all_traffic(testbed_config, port_config_list, dut_hostname, diff --git a/tests/lldp/test_lldp.py b/tests/lldp/test_lldp.py index 0306763a16..2f9fa91b21 100644 --- a/tests/lldp/test_lldp.py +++ b/tests/lldp/test_lldp.py @@ -1,5 +1,9 @@ import logging import pytest +from tests.common.platform.interface_utils import get_dpu_npu_ports_from_hwsku +from tests.common.helpers.dut_utils import get_program_info, kill_process_by_pid, is_container_running + +from tests.common.helpers.assertions import pytest_assert logger = logging.getLogger(__name__) @@ -17,6 +21,24 @@ def lldp_setup(duthosts, enum_rand_one_per_hwsku_frontend_hostname, patch_lldpct unpatch_lldpctl(localhost, duthost) +@pytest.fixture(scope="function") +def restart_orchagent(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_frontend_asic_index): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + asic = duthost.asic_instance(enum_frontend_asic_index) + container_name = asic.get_docker_name("swss") + program_name = "orchagent" + + logger.info("Restarting program '{}' in container '{}'".format(program_name, container_name)) + + duthost.shell("sudo config feature autorestart {} disabled".format(container_name)) + _, program_pid = get_program_info(duthost, container_name, program_name) + kill_process_by_pid(duthost, container_name, program_name, program_pid) + is_running = is_container_running(duthost, container_name) + pytest_assert(is_running, "Container '{}' is not running. Exiting...".format(container_name)) + duthost.shell("docker exec {} supervisorctl start {}".format(container_name, program_name)) + yield + + def test_lldp(duthosts, enum_rand_one_per_hwsku_frontend_hostname, localhost, collect_techsupport_all_duts, enum_frontend_asic_index, request): """ verify the LLDP message on DUT """ @@ -24,9 +46,10 @@ def test_lldp(duthosts, enum_rand_one_per_hwsku_frontend_hostname, localhost, config_facts = duthost.asic_instance( enum_frontend_asic_index).config_facts(host=duthost.hostname, source="running")['ansible_facts'] + internal_port_list = get_dpu_npu_ports_from_hwsku(duthost) lldpctl_facts = duthost.lldpctl_facts( asic_instance_id=enum_frontend_asic_index, - skip_interface_pattern_list=["eth0", "Ethernet-BP", "Ethernet-IB"])['ansible_facts'] + skip_interface_pattern_list=["eth0", "Ethernet-BP", "Ethernet-IB"] + internal_port_list)['ansible_facts'] if not list(lldpctl_facts['lldpctl'].items()): pytest.fail("No LLDP neighbors received (lldpctl_facts are empty)") for k, v in list(lldpctl_facts['lldpctl'].items()): @@ -40,26 +63,17 @@ def test_lldp(duthosts, enum_rand_one_per_hwsku_frontend_hostname, localhost, assert v['port']['descr'] == config_facts['DEVICE_NEIGHBOR'][k]['port'] -def test_lldp_neighbor(duthosts, enum_rand_one_per_hwsku_frontend_hostname, localhost, eos, sonic, - collect_techsupport_all_duts, loganalyzer, enum_frontend_asic_index, tbinfo, request): +def check_lldp_neighbor(duthost, localhost, eos, sonic, collect_techsupport_all_duts, + enum_frontend_asic_index, tbinfo, request): """ verify LLDP information on neighbors """ - duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - - if loganalyzer: - loganalyzer[enum_rand_one_per_hwsku_frontend_hostname].ignore_regex.extend([ - ".*ERR syncd#syncd: :- check_fdb_event_notification_data.*", - ".*ERR syncd#syncd: :- process_on_fdb_event: invalid OIDs in fdb \ - notifications, NOT translating and NOT storing in ASIC DB.*", - ".*ERR syncd#syncd: :- process_on_fdb_event: FDB notification was \ - not sent since it contain invalid OIDs, bug.*", - ]) res = duthost.shell( "docker exec -i lldp lldpcli show chassis | grep \"SysDescr:\" | sed -e 's/^\\s*SysDescr:\\s*//g'") dut_system_description = res['stdout'] + internal_port_list = get_dpu_npu_ports_from_hwsku(duthost) lldpctl_facts = duthost.lldpctl_facts( asic_instance_id=enum_frontend_asic_index, - skip_interface_pattern_list=["eth0", "Ethernet-BP", "Ethernet-IB"])['ansible_facts'] + skip_interface_pattern_list=["eth0", "Ethernet-BP", "Ethernet-IB"] + internal_port_list)['ansible_facts'] config_facts = duthost.asic_instance(enum_frontend_asic_index).config_facts(host=duthost.hostname, source="running")['ansible_facts'] if not list(lldpctl_facts['lldpctl'].items()): @@ -109,3 +123,28 @@ def test_lldp_neighbor(duthosts, enum_rand_one_per_hwsku_frontend_hostname, loca # Verify the published DUT port description field is correct assert nei_lldp_facts['ansible_lldp_facts'][neighbor_interface]['neighbor_port_desc'] == \ config_facts['PORT'][k]['description'] + + +def test_lldp_neighbor(duthosts, enum_rand_one_per_hwsku_frontend_hostname, localhost, eos, sonic, + collect_techsupport_all_duts, loganalyzer, enum_frontend_asic_index, tbinfo, request): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + if loganalyzer: + loganalyzer[enum_rand_one_per_hwsku_frontend_hostname].ignore_regex.extend([ + ".*ERR syncd#syncd: :- check_fdb_event_notification_data.*", + ".*ERR syncd#syncd: :- process_on_fdb_event: invalid OIDs in fdb \ + notifications, NOT translating and NOT storing in ASIC DB.*", + ".*ERR syncd#syncd: :- process_on_fdb_event: FDB notification was \ + not sent since it contain invalid OIDs, bug.*", + ]) + check_lldp_neighbor(duthost, localhost, eos, sonic, collect_techsupport_all_duts, + enum_frontend_asic_index, tbinfo, request) + + +@pytest.mark.disable_loganalyzer +def test_lldp_neighbor_post_orchagent_reboot(duthosts, enum_rand_one_per_hwsku_frontend_hostname, localhost, eos, + sonic, collect_techsupport_all_duts, + enum_frontend_asic_index, tbinfo, request, restart_orchagent): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + check_lldp_neighbor(duthost, localhost, eos, sonic, collect_techsupport_all_duts, + enum_frontend_asic_index, tbinfo, request) diff --git a/tests/lldp/test_lldp_syncd.py b/tests/lldp/test_lldp_syncd.py new file mode 100644 index 0000000000..2dd72fde8f --- /dev/null +++ b/tests/lldp/test_lldp_syncd.py @@ -0,0 +1,300 @@ +# Test plan in docs/testplan/LLDP-syncd-test-plan.md +import pytest +import json +from tests.common.helpers.sonic_db import SonicDbCli +import logging +from tests.common.reboot import reboot, REBOOT_TYPE_COLD +from tests.common.utilities import wait_until +from tests.common.helpers.assertions import pytest_assert + + +logger = logging.getLogger(__name__) + +APPL_DB = "APPL_DB" + +pytestmark = [ + pytest.mark.topology("any"), +] + + +@pytest.fixture(autouse="True") +def db_instance(duthosts, enum_rand_one_per_hwsku_frontend_hostname): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + appl_db = SonicDbCli(duthost, APPL_DB) + # Cleanup code here + return appl_db + + +# Helper function to get the LLDP_ENTRY_TABLE keys +def get_lldp_entry_keys(db): + items = db.get_keys("LLDP_ENTRY_TABLE*") + return [key.split(":")[1] for key in items] + + +# Helper function to get LLDP_ENTRY_TABLE content +def get_lldp_entry_content(db, interface): + return db.hget_all("LLDP_ENTRY_TABLE:{}".format(interface)) + + +# Helper function to get lldptcl output +def get_lldpctl_facts_output(duthost, enum_frontend_asic_index): + lldpctl_facts = duthost.lldpctl_facts( + asic_instance_id=enum_frontend_asic_index, + skip_interface_pattern_list=["Ethernet-BP", "Ethernet-IB"], + )["ansible_facts"] + return lldpctl_facts + + +def get_lldpctl_output(duthost): + result = duthost.shell("docker exec lldp /usr/sbin/lldpctl -f json")["stdout"] + return json.loads(result) + + +# Helper function to get show lldp table output +def get_show_lldp_table_output(duthost): + lines = duthost.shell("show lldp table")["stdout"].split("\n")[3:-2] + interface_list = [line.split()[0] for line in lines] + return interface_list + + +def assert_lldp_interfaces( + lldp_entry_keys, show_lldp_table_int_list, lldpctl_interface +): + # Verify LLDP_ENTRY_TABLE keys match show lldp table output + pytest_assert( + sorted(lldp_entry_keys) == sorted(show_lldp_table_int_list), + "LLDP_ENTRY_TABLE keys do not match 'show lldp table' output", + ) + + # Verify LLDP_ENTRY_TABLE keys match lldpctl interface indexes + # Handle cases where lldpctl_output["lldp"]["interface"] might be a list or dict + if isinstance(lldpctl_interface, dict): + lldpctl_interfaces = [interface for interface in lldpctl_interface.keys()] + elif isinstance(lldpctl_interface, list): + lldpctl_interfaces = [ + list(interface.keys())[0] for interface in lldpctl_interface + ] + else: + raise TypeError( + "Unexpected type for lldpctl interfaces: {}".format(type(lldpctl_interface)) + ) + + pytest_assert( + sorted(lldp_entry_keys) == sorted(lldpctl_interfaces), + "LLDP_ENTRY_TABLE keys do not match lldpctl interface indexes", + ) + + +def assert_lldp_entry_content(interface, entry_content, lldpctl_interface): + pytest_assert( + lldpctl_interface, + "No LLDP data found for {} in lldpctl output".format(interface), + ) + + chassis_info = lldpctl_interface["chassis"][entry_content["lldp_rem_sys_name"]] + port_info = lldpctl_interface["port"] + + # Compare relevant fields between LLDP_ENTRY_TABLE and lldpctl output + pytest_assert( + entry_content["lldp_rem_chassis_id"] == chassis_info["id"]["value"], + "lldp_rem_chassis_id does not match for {}".format(interface), + ) + pytest_assert( + entry_content["lldp_rem_port_id"] == port_info["id"]["value"], + "lldp_rem_port_id does not match for {}".format(interface), + ) + pytest_assert( + entry_content["lldp_rem_sys_name"] + == list(lldpctl_interface["chassis"].keys())[0], + "lldp_rem_sys_name does not match for {}".format(interface), + ) + pytest_assert( + entry_content["lldp_rem_sys_desc"] == chassis_info["descr"], + "lldp_rem_sys_desc does not match for {}".format(interface), + ) + pytest_assert( + entry_content["lldp_rem_port_desc"] == port_info.get("descr", ""), + "lldp_rem_port_desc does not match for {}".format(interface), + ) + pytest_assert( + entry_content["lldp_rem_man_addr"] == chassis_info.get("mgmt-ip", ""), + "lldp_rem_man_addr does not match for {}".format(interface), + ) + pytest_assert( + entry_content["lldp_rem_sys_cap_supported"] == "28 00", + "lldp_rem_sys_cap_supported does not match for {}".format(interface), + ) + pytest_assert( + entry_content["lldp_rem_sys_cap_enabled"] == "28 00", + "lldp_rem_sys_cap_enabled does not match for {}".format(interface), + ) + + +def verify_lldp_entry(db_instance, interface): + entry_content = get_lldp_entry_content(db_instance, interface) + if entry_content: + return True + else: + return False + + +def verify_lldp_table(duthost): + # based on experience, eth0 shows up at last + output = duthost.shell("show lldp table")["stdout"] + if "eth0" in output: + return True + else: + return False + + +# Test case 1: Verify LLDP_ENTRY_TABLE keys match show lldp table output and lldpctl output +def test_lldp_entry_table_keys( + duthosts, enum_rand_one_per_hwsku_frontend_hostname, db_instance +): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + lldp_entry_keys = get_lldp_entry_keys(db_instance) + show_lldp_table_int_list = get_show_lldp_table_output(duthost) + lldpctl_output = get_lldpctl_output(duthost) + assert_lldp_interfaces( + lldp_entry_keys, show_lldp_table_int_list, lldpctl_output["lldp"]["interface"] + ) + + +# Test case 2: Verify LLDP_ENTRY_TABLE content against lldpctl output +def test_lldp_entry_table_content( + duthosts, enum_rand_one_per_hwsku_frontend_hostname, db_instance +): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + lldpctl_output = get_lldpctl_output(duthost) + lldpctl_interfaces = lldpctl_output["lldp"]["interface"] + + for interface in get_lldp_entry_keys(db_instance): + entry_content = get_lldp_entry_content(db_instance, interface) + if isinstance(lldpctl_interfaces, dict): + lldpctl_interface = lldpctl_interfaces.get(interface) + elif isinstance(lldpctl_interfaces, list): + for iface in lldpctl_interfaces: + if list(iface.keys())[0].lower() == interface.lower(): + lldpctl_interface = iface.get(list(iface.keys())[0]) + logger.info("lldpctl_interface: {}".format(lldpctl_interface)) + break + assert_lldp_entry_content(interface, entry_content, lldpctl_interface) + + # Add assertions to compare specific fields between LLDP_ENTRY_TABLE and lldpctl output + + +# Test case 3: Verify LLDP_ENTRY_TABLE after interface flap +def test_lldp_entry_table_after_flap( + duthosts, enum_rand_one_per_hwsku_frontend_hostname, db_instance +): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + # Fetch interfaces from LLDP_ENTRY_TABLE + lldp_entry_keys = get_lldp_entry_keys(db_instance) + show_lldp_table_int_list = get_show_lldp_table_output(duthost) + lldpctl_output = get_lldpctl_output(duthost) + + for interface in lldp_entry_keys: + if interface == "eth0": + pytest.skip("Skipping test for eth0 interface") + # Shutdown and startup the interface + duthost.shell("sudo config interface shutdown {}".format(interface)) + duthost.shell("sudo config interface startup {}".format(interface)) + result = wait_until(30, 2, 5, verify_lldp_entry, db_instance, interface) + pytest_assert( + result, + "After interface {} flap, no LLDP_ENTRY_TABLE entry for it.".format( + interface + ), + ) + lldpctl_interfaces = lldpctl_output["lldp"]["interface"] + assert_lldp_interfaces( + lldp_entry_keys, show_lldp_table_int_list, lldpctl_interfaces + ) + entry_content = get_lldp_entry_content(db_instance, interface) + logger.info("entry_content={}".format(entry_content)) + if isinstance(lldpctl_interfaces, dict): + lldpctl_interface = lldpctl_interfaces.get(interface) + logger.info( + "lldpctl_interfaces type dict, lldpctl_interface: {}".format( + lldpctl_interface + ) + ) + elif isinstance(lldpctl_interfaces, list): + for iface in lldpctl_interfaces: + if list(iface.keys())[0].lower() == interface.lower(): + lldpctl_interface = iface.get(list(iface.keys())[0]) + logger.info( + "lldpctl_interfaces type list, lldpctl_interface: {}".format( + lldpctl_interface + ) + ) + break + assert_lldp_entry_content(interface, entry_content, lldpctl_interface) + + +# Test case 4: Verify LLDP_ENTRY_TABLE after system reboot +def test_lldp_entry_table_after_lldp_restart( + duthosts, enum_rand_one_per_hwsku_frontend_hostname, db_instance +): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + lldp_entry_keys = get_lldp_entry_keys(db_instance) + show_lldp_table_int_list = get_show_lldp_table_output(duthost) + lldpctl_output = get_lldpctl_output(duthost) + + # Restart the LLDP service + duthost.shell("sudo systemctl restart lldp") + result = wait_until( + 60, 2, 5, verify_lldp_table, duthost + ) # Adjust based on LLDP service restart time + pytest_assert(result, "eth0 is still not in output of show lldp table") + lldpctl_interfaces = lldpctl_output["lldp"]["interface"] + assert_lldp_interfaces( + lldp_entry_keys, show_lldp_table_int_list, lldpctl_interfaces + ) + for interface in lldp_entry_keys: + entry_content = get_lldp_entry_content(db_instance, interface) + + if isinstance(lldpctl_interfaces, dict): + lldpctl_interface = lldpctl_interfaces.get(interface) + elif isinstance(lldpctl_interfaces, list): + for iface in lldpctl_interfaces: + if list(iface.keys())[0].lower() == interface.lower(): + lldpctl_interface = iface.get(list(iface.keys())[0]) + break + assert_lldp_entry_content(interface, entry_content, lldpctl_interface) + + +# Test case 5: Verify LLDP_ENTRY_TABLE after reboot +def test_lldp_entry_table_after_reboot( + localhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, db_instance +): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + lldp_entry_keys = get_lldp_entry_keys(db_instance) + show_lldp_table_int_list = get_show_lldp_table_output(duthost) + lldpctl_output = get_lldpctl_output(duthost) + + # reboot + logging.info("Run cold reboot on DUT") + reboot( + duthost, + localhost, + reboot_type=REBOOT_TYPE_COLD, + reboot_helper=None, + reboot_kwargs=None, + safe_reboot=True, + ) + lldpctl_interfaces = lldpctl_output["lldp"]["interface"] + assert_lldp_interfaces( + lldp_entry_keys, show_lldp_table_int_list, lldpctl_interfaces + ) + for interface in get_lldp_entry_keys(db_instance): + entry_content = get_lldp_entry_content(db_instance, interface) + + if isinstance(lldpctl_interfaces, dict): + lldpctl_interface = lldpctl_interfaces.get(interface) + elif isinstance(lldpctl_interfaces, list): + for iface in lldpctl_interfaces: + if list(iface.keys())[0].lower() == interface.lower(): + lldpctl_interface = iface.get(list(iface.keys())[0]) + break + assert_lldp_entry_content(interface, entry_content, lldpctl_interface) diff --git a/tests/log_fidelity/test_bgp_shutdown.py b/tests/log_fidelity/test_bgp_shutdown.py index 55c4e7a7d4..e69e5b7780 100644 --- a/tests/log_fidelity/test_bgp_shutdown.py +++ b/tests/log_fidelity/test_bgp_shutdown.py @@ -10,6 +10,23 @@ ] +@pytest.fixture(autouse=True) +def ignore_expected_loganalyzer_exception(loganalyzer, duthosts): + + ignore_errors = [ + r".* ERR syncd#syncd: .*SAI_API_TUNNEL:_brcm_sai_mptnl_tnl_route_event_add:\d+ ecmp table entry lookup " + "failed with error.*", + r".* ERR syncd#syncd: .*SAI_API_TUNNEL:_brcm_sai_mptnl_process_route_add_mode_default_and_host:\d+ " + "_brcm_sai_mptnl_tnl_route_event_add failed with error.*" + ] + + if loganalyzer: + for duthost in duthosts: + loganalyzer[duthost.hostname].ignore_regex.extend(ignore_errors) + + return None + + def check_syslog(duthost, prefix, trigger_action, expected_log, restore_action): loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=prefix) loganalyzer.expect_regex = [expected_log] diff --git a/tests/monit/test_monit_status.py b/tests/monit/test_monit_status.py index 06b6553cdb..2f78801079 100644 --- a/tests/monit/test_monit_status.py +++ b/tests/monit/test_monit_status.py @@ -4,6 +4,7 @@ import logging import pytest +import random from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert @@ -31,33 +32,24 @@ def stop_and_start_lldpmgrd(duthosts, enum_rand_one_per_hwsku_frontend_hostname) """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - logger.info("Stopping 'lldpmgrd' process in 'lldp' container ...") - stop_command_result = duthost.command("docker exec lldp supervisorctl stop lldpmgrd") - exit_code = stop_command_result["rc"] - pytest_assert(exit_code == 0, "Failed to stop 'lldpmgrd' process in 'lldp' container!") - logger.info("'lldpmgrd' process in 'lldp' container is stopped.") - if duthost.is_multi_asic: - logger.info("Stopping 'lldpmgrd' process in 'lldp0' container ...") - stop_command_result = duthost.command("docker exec lldp0 supervisorctl stop lldpmgrd") - exit_code = stop_command_result["rc"] - pytest_assert(exit_code == 0, "Failed to stop 'lldpmgrd' process in 'lldp0' container!") - logger.info("'lldpmgrd' process in 'lldp0' container is stopped.") + process = random.choice(["lldp0", "lldp1"]) + else: + process = "lldp" + + logger.info("Stopping 'lldpmgrd' process in {} container ...".format(process)) + stop_command_result = duthost.command("docker exec {} supervisorctl stop lldpmgrd".format(process)) + exit_code = stop_command_result["rc"] + pytest_assert(exit_code == 0, "Failed to stop 'lldpmgrd' process in {} container!".format(process)) + logger.info("'lldpmgrd' process in {} container is stopped.".format(process)) yield - logger.info("Starting 'lldpmgrd' process in 'lldp' container ...") - start_command_result = duthost.command("docker exec lldp supervisorctl start lldpmgrd") + logger.info("Starting 'lldpmgrd' process in {} container ...".format(process)) + start_command_result = duthost.command("docker exec {} supervisorctl start lldpmgrd".format(process)) exit_code = start_command_result["rc"] - pytest_assert(exit_code == 0, "Failed to start 'lldpmgrd' process in 'lldp' container!") - logger.info("'lldpmgrd' process in 'lldp' container is started.") - - if duthost.is_multi_asic: - logger.info("Starting 'lldpmgrd' process in 'lldp0' container ...") - start_command_result = duthost.command("docker exec lldp0 supervisorctl start lldpmgrd") - exit_code = start_command_result["rc"] - pytest_assert(exit_code == 0, "Failed to start 'lldpmgrd' process in 'lldp0' container!") - logger.info("'lldpmgrd' process in 'lldp0' container is started.") + pytest_assert(exit_code == 0, "Failed to start 'lldpmgrd' process in {} container!".format(process)) + logger.info("'lldpmgrd' process in {} container is started.".format(process)) def check_monit_last_output(duthost): diff --git a/tests/mvrf/test_mgmtvrf.py b/tests/mvrf/test_mgmtvrf.py index ff639647d0..8a184579fc 100644 --- a/tests/mvrf/test_mgmtvrf.py +++ b/tests/mvrf/test_mgmtvrf.py @@ -228,7 +228,15 @@ def test_ntp(self, duthosts, rand_one_dut_hostname, ptfhost, check_ntp_sync, ntp # Check if ntp was not in sync with ntp server before enabling mvrf, if yes then setup ntp server on ptf if check_ntp_sync: setup_ntp(ptfhost, duthost, ntp_servers) - ntp_uid = ":".join(duthost.command("getent passwd ntp")['stdout'].split(':')[2:4]) + + # There is no entry ntp in `/etc/passwd` on kvm testbed. + cmd = "getent passwd ntp" + ntp_uid_output = duthost.command(cmd, module_ignore_errors=True) + if duthost.facts["asic_type"] == "vs" and ntp_uid_output['rc'] == 2: + return + assert ntp_uid_output['rc'] == 0, "Run command '{}' failed".format(cmd) + ntp_uid = ":".join(ntp_uid_output['stdout'].split(':')[2:4]) + force_ntp = "timeout 20 ntpd -gq -u {}".format(ntp_uid) duthost.service(name="ntp", state="stopped") logger.info("Ntp restart in mgmt vrf") @@ -276,8 +284,21 @@ def test_warmboot(self, duthosts, rand_one_dut_hostname, localhost, ptfhost, cre reboot(duthost, localhost, reboot_type="warm") pytest_assert(wait_until(120, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") + # Change default critical services to check services that starts with bootOn timer - duthost.reset_critical_services_tracking_list(['snmp', 'telemetry', 'mgmt-framework']) + # In some images, we have gnmi container only + # In some images, we have telemetry container only + # And in some images, we have both gnmi and telemetry container + critical_services = ['snmp', 'mgmt-framework'] + cmd = "docker ps | grep -w gnmi" + if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0: + critical_services.append('gnmi') + + cmd = "docker ps | grep -w telemetry" + if duthost.shell(cmd, module_ignore_errors=True)['rc'] == 0: + critical_services.append('telemetry') + duthost.reset_critical_services_tracking_list(critical_services) + pytest_assert(wait_until(180, 20, 0, duthost.critical_services_fully_started), "Not all services which start with bootOn timer are fully started") self.basic_check_after_reboot(duthost, localhost, ptfhost, creds) diff --git a/tests/ntp/test_ntp.py b/tests/ntp/test_ntp.py index 9236f66519..fc015e1d90 100644 --- a/tests/ntp/test_ntp.py +++ b/tests/ntp/test_ntp.py @@ -1,9 +1,10 @@ from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.ntp_helper import check_ntp_status, run_ntp, _context_for_setup_ntp import logging import time import pytest -from contextlib import contextmanager + logger = logging.getLogger(__name__) @@ -39,61 +40,20 @@ def config_long_jump(duthost, enable=False): regex = "s/NTPD_OPTS='-g'/NTPD_OPTS='-x'/" if using_ntpsec: - duthost.command("sed -i '%s' /etc/default/ntpsec" % regex) + duthost.command("sudo sed -i '%s' /etc/default/ntpsec" % regex) else: - duthost.command("sed -i %s /etc/default/ntp" % regex) + duthost.command("sudo sed -i %s /etc/default/ntp" % regex) duthost.service(name='ntp', state='restarted') @pytest.fixture(scope="module") def setup_ntp(ptfhost, duthosts, rand_one_dut_hostname, ptf_use_ipv6): + if ptf_use_ipv6 and not ptfhost.mgmt_ipv6: + pytest.skip("No IPv6 address on PTF host") with _context_for_setup_ntp(ptfhost, duthosts, rand_one_dut_hostname, ptf_use_ipv6) as result: yield result -@pytest.fixture(scope="function") -def setup_ntp_func(ptfhost, duthosts, rand_one_dut_hostname, ptf_use_ipv6): - with _context_for_setup_ntp(ptfhost, duthosts, rand_one_dut_hostname, ptf_use_ipv6) as result: - yield result - - -@contextmanager -def _context_for_setup_ntp(ptfhost, duthosts, rand_one_dut_hostname, ptf_use_ipv6): - """setup ntp client and server""" - duthost = duthosts[rand_one_dut_hostname] - - ptfhost.lineinfile(path="/etc/ntp.conf", line="server 127.127.1.0 prefer") - - # restart ntp server - ntp_en_res = ptfhost.service(name="ntp", state="restarted") - - pytest_assert(wait_until(120, 5, 0, check_ntp_status, ptfhost), - "NTP server was not started in PTF container {}; NTP service start result {}" - .format(ptfhost.hostname, ntp_en_res)) - - # setup ntp on dut to sync with ntp server - config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] - ntp_servers = config_facts.get('NTP_SERVER', {}) - for ntp_server in ntp_servers: - duthost.command("config ntp del %s" % ntp_server) - - duthost.command("config ntp add %s" % (ptfhost.mgmt_ipv6 if ptf_use_ipv6 else ptfhost.mgmt_ip)) - - yield - - # stop ntp server - ptfhost.service(name="ntp", state="stopped") - # reset ntp client configuration - duthost.command("config ntp del %s" % (ptfhost.mgmt_ipv6 if ptf_use_ipv6 else ptfhost.mgmt_ip)) - for ntp_server in ntp_servers: - duthost.command("config ntp add %s" % ntp_server) - # The time jump leads to exception in lldp_syncd. The exception has been handled by lldp_syncd, - # but it will leave error messages in syslog, which will cause subsequent test cases to fail. - # So we need to wait for a while to make sure the error messages are flushed. - # The default update interval of lldp_syncd is 10 seconds, so we wait for 20 seconds here. - time.sleep(20) - - @pytest.fixture def setup_long_jump_config(duthosts, rand_one_dut_hostname): """set long jump config and set DUT's time forward""" @@ -125,13 +85,6 @@ def setup_long_jump_config(duthosts, rand_one_dut_hostname): config_long_jump(duthost, long_jump_enable) -def check_ntp_status(host): - res = host.command("ntpstat", module_ignore_errors=True) - if res['rc'] != 0: - return False - return True - - def test_ntp_long_jump_enabled(duthosts, rand_one_dut_hostname, setup_ntp, setup_long_jump_config): duthost = duthosts[rand_one_dut_hostname] @@ -146,26 +99,8 @@ def test_ntp_long_jump_disabled(duthosts, rand_one_dut_hostname, setup_ntp, setu config_long_jump(duthost, enable=False) - if wait_until(720, 10, 0, check_ntp_status, duthost): - pytest.fail("NTP long jump disable failed") - - -def run_ntp(duthosts, rand_one_dut_hostname, setup_ntp): - """ Verify that DUT is synchronized with configured NTP server """ - duthost = duthosts[rand_one_dut_hostname] - - ntpsec_conf_stat = duthost.stat(path="/etc/ntpsec/ntp.conf") - using_ntpsec = ntpsec_conf_stat["stat"]["exists"] - - duthost.service(name='ntp', state='stopped') - if using_ntpsec: - duthost.command("timeout 20 ntpd -gq -u ntpsec:ntpsec") - else: - ntp_uid = ":".join(duthost.command("getent passwd ntp")['stdout'].split(':')[2:4]) - duthost.command("timeout 20 ntpd -gq -u {}".format(ntp_uid)) - duthost.service(name='ntp', state='restarted') pytest_assert(wait_until(720, 10, 0, check_ntp_status, duthost), - "NTP not in sync") + "NTP long jump disable failed") def test_ntp(duthosts, rand_one_dut_hostname, setup_ntp): diff --git a/tests/ospf/conftest.py b/tests/ospf/conftest.py index 4f72ac4493..92341709a7 100644 --- a/tests/ospf/conftest.py +++ b/tests/ospf/conftest.py @@ -1,3 +1,7 @@ +''' +Conftest file for OSPF tests +''' + import pytest import re from tests.common.config_reload import config_reload @@ -73,3 +77,63 @@ def get_ospf_neighbor_interface(host): ':')[1] # Return the interface name # Return None if interface name is not found or not PortChannels. return nbr_int_info + + +@pytest.fixture(scope='module') +def ospf_setup(duthosts, rand_one_dut_hostname, nbrhosts, tbinfo, request): + + # verify neighbors are type sonic + if request.config.getoption("neighbor_type") != "sonic": + pytest.skip("Neighbor type must be sonic") + + duthost = duthosts[rand_one_dut_hostname] + + setup_info = {'nbr_addr': {}, 'bgp_routes': []} + + mg_facts = duthost.get_extended_minigraph_facts(tbinfo) + + for bgp_nbr in mg_facts['minigraph_bgp']: + setup_info['nbr_addr'][bgp_nbr['name']] = bgp_nbr['addr'] + + # gather original BGP routes + cmd = "show ip route bgp" + bgp_routes = duthost.shell(cmd)['stdout'] + bgp_routes_pattern = re.compile(r'B>\*(\d+\.\d+\.\d+\.\d+/\d+)') + original_prefixes = bgp_routes_pattern.findall(bgp_routes).sort() + setup_info['bgp_routes'] = original_prefixes + + for neigh_name in list(nbrhosts.keys()): + ip_addr = None + asn = None + neigh_mg_facts = nbrhosts[neigh_name]["host"].minigraph_facts(host=nbrhosts[neigh_name]["host"].hostname) + for neigh_bgp_nbr in neigh_mg_facts['minigraph_bgp']: + if neigh_bgp_nbr['name'] == duthost.hostname: + ip_addr = neigh_bgp_nbr['addr'] + asn = neigh_bgp_nbr['asn'] + break + cmd_list = [ + 'docker exec -it bgp bash', + 'cd /usr/lib/frr', + './ospfd &', + 'exit', + 'vtysh', + 'config t', + 'router bgp', + 'no neighbor {} remote-as {}'.format(str(ip_addr), str(asn)), + 'exit', + 'router ospf', + 'network {}/31 area 0'.format(str(ip_addr)), + 'redistribute bgp', + 'do write', + 'end', + 'exit' + ] + nbrhosts[neigh_name]["host"].shell_cmds(cmd_list) + + yield setup_info + + # restore config to original state on both DUT and neighbor + config_reload(duthost, safe_reload=True) + time.sleep(10) + for neigh_name in list(nbrhosts.keys()): + config_reload(nbrhosts[neigh_name]["host"], is_dut=False) diff --git a/tests/ospf/test_ospf.py b/tests/ospf/test_ospf.py new file mode 100644 index 0000000000..d40e1578ee --- /dev/null +++ b/tests/ospf/test_ospf.py @@ -0,0 +1,144 @@ +import pytest +import logging +import time +import re + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('t0') +] + + +def test_ospf_neighborship(ospf_setup, duthosts, rand_one_dut_hostname): + setup_info_nbr_addr = ospf_setup['nbr_addr'] + neigh_ip_addrs = list(setup_info_nbr_addr.values()) + duthost = duthosts[rand_one_dut_hostname] + + # Check get existing bgp routes on the DUT + original_prefixes = ospf_setup['bgp_routes'] + + # Configure OSPF neighbors in DUT if not already configured + ospf_configured = False + cmd = 'vtysh -c "show ip ospf neighbor"' + ospf_neighbors = duthost.shell(cmd)['stdout'].split("\n") + for neighbor in ospf_neighbors: + if ("ospfd is not running" not in neighbor) and (neighbor != "") and ("Neighbor ID" not in neighbor): + ospf_configured = True + + if not ospf_configured: + cmd_list = [ + 'docker exec -it bgp bash', + 'cd /usr/lib/frr', + './ospfd &', + 'exit', + 'vtysh', + 'config t', + 'no router bgp', + 'router ospf' + ] + + for ip_addr in neigh_ip_addrs: + cmd_list.append('network {}/31 area 0'.format(str(ip_addr))) + + cmd_list.extend([ + 'do write', + 'end', + 'exit' + ]) + + duthost.shell_cmds(cmd_list) + time.sleep(5) + + # Verify old BGP routes are available as OSPF routes in the DUT + cmd = 'vtysh -c "show ip ospf neighbor"' + ospf_neighbors = duthost.shell(cmd)['stdout'].split("\n") + for neighbor in ospf_neighbors: + if (neighbor != "") and ("Neighbor ID" not in neighbor): + assert "Full" in neighbor + + # Compare new OSPF prefixes with old BGP prefixes + cmd = "show ip route ospf" + ospf_routes = duthost.shell(cmd)['stdout'] + ospf_routes_pattern = re.compile(r'O>\*(\d+\.\d+\.\d+\.\d+/\d+)') + new_prefixes = ospf_routes_pattern.findall(ospf_routes).sort() + + assert original_prefixes == new_prefixes + + +def test_ospf_dynamic_routing(ospf_setup, duthosts, rand_one_dut_hostname, nbrhosts): + setup_info_nbr_addr = ospf_setup['nbr_addr'] + neigh_ip_addrs = list(setup_info_nbr_addr.values()) + duthost = duthosts[rand_one_dut_hostname] + + # Add loopback interface in the first neighboring device + first_nbr = list(nbrhosts.keys())[0] + loopback_cmd = "config interface ip add Loopback10 192.168.10.1/32" + nbrhosts[first_nbr]["host"].shell(loopback_cmd) + + # Advertise newly created loopback network to the DUT via OSPF + advertise_network_cmd = "vtysh -c 'config terminal' -c 'router ospf' -c 'network 192.168.10.1/32 area 0'" + nbrhosts[first_nbr]["host"].shell(advertise_network_cmd) + + # Check OSPF already configured in DUT + ospf_configured = False + cmd = 'vtysh -c "show ip ospf neighbor"' + ospf_neighbors = duthost.shell(cmd)['stdout'].split("\n") + for neighbor in ospf_neighbors: + if ("ospfd is not running" not in neighbor) and (neighbor != "") and ("Neighbor ID" not in neighbor): + ospf_configured = True + + # Configure OSPF neighbors in DUT if not already configured + if not ospf_configured: + cmd_list = [ + 'docker exec -it bgp bash', + 'cd /usr/lib/frr', + './ospfd &', + 'exit', + 'vtysh', + 'config t', + 'no router bgp', + 'router ospf' + ] + + for ip_addr in neigh_ip_addrs: + cmd_list.append('network {}/31 area 0'.format(str(ip_addr))) + + cmd_list.extend([ + 'do write', + 'end', + 'exit' + ]) + + duthost.shell_cmds(cmd_list) + time.sleep(5) + + # Verify OSPF neighborship successfully established and loopback route shared to the DUT + cmd = 'vtysh -c "show ip ospf neighbor"' + ospf_neighbors = duthost.shell(cmd)['stdout'].split("\n") + for neighbor in ospf_neighbors: + if (neighbor != "") and ("Neighbor ID" not in neighbor): + assert "Full" in neighbor + route_found = False + cmd = 'show ip route ospf' + ospf_routes = duthost.shell(cmd)['stdout'].split("\n") + for route in ospf_routes: + if '192.168.10.1/32' in route: + route_found = True + break + assert route_found is True + + # Simulate link down by removing loopback interface from neighbor + rem_loopback_cmd = "config interface ip remove Loopback10 192.168.10.1/32" + nbrhosts[first_nbr]["host"].shell(rem_loopback_cmd) + time.sleep(5) + + # Verify that loopback route is not present in DUT + route_found = False + cmd = 'show ip route ospf' + ospf_routes = duthost.shell(cmd)['stdout'].split("\n") + for route in ospf_routes: + if '192.168.10.1/32' in route: + route_found = True + break + assert route_found is False diff --git a/tests/override_config_table/test_override_config_table.py b/tests/override_config_table/test_override_config_table.py index be233a787e..a470d63fcc 100644 --- a/tests/override_config_table/test_override_config_table.py +++ b/tests/override_config_table/test_override_config_table.py @@ -4,14 +4,15 @@ from tests.common.utilities import skip_release from tests.common.utilities import update_pfcwd_default_state from tests.common.config_reload import config_reload -from utilities import backup_config, restore_config, get_running_config,\ - reload_minigraph_with_golden_config, file_exists_on_dut +from tests.common.utilities import backup_config, restore_config, get_running_config,\ + reload_minigraph_with_golden_config, file_exists_on_dut, compare_dicts_ignore_list_order, \ + NON_USER_CONFIG_TABLES + GOLDEN_CONFIG = "/etc/sonic/golden_config_db.json" GOLDEN_CONFIG_BACKUP = "/etc/sonic/golden_config_db.json_before_override" CONFIG_DB = "/etc/sonic/config_db.json" CONFIG_DB_BACKUP = "/etc/sonic/config_db.json_before_override" -NON_USER_CONFIG_TABLES = ["FLEX_COUNTER_TABLE", "ASIC_SENSORS"] pytestmark = [ pytest.mark.topology('t0', 't1', 'any'), @@ -75,21 +76,6 @@ def setup_env(duthosts, golden_config_exists_on_dut, tbinfo, enum_rand_one_per_h config_reload(duthost) -def compare_dicts_ignore_list_order(dict1, dict2): - def normalize(data): - if isinstance(data, list): - return set(data) - elif isinstance(data, dict): - return {k: normalize(v) for k, v in data.items()} - else: - return data - - dict1_normalized = normalize(dict1) - dict2_normalized = normalize(dict2) - - return dict1_normalized == dict2_normalized - - def load_minigraph_with_golden_empty_input(duthost): """Test Golden Config with empty input """ diff --git a/tests/override_config_table/test_override_config_table_masic.py b/tests/override_config_table/test_override_config_table_masic.py index 846e2d98d5..204511cfd8 100644 --- a/tests/override_config_table/test_override_config_table_masic.py +++ b/tests/override_config_table/test_override_config_table_masic.py @@ -5,10 +5,8 @@ from tests.common.utilities import skip_release from tests.common.utilities import update_pfcwd_default_state from tests.common.config_reload import config_reload -from utilities import backup_config, restore_config, get_running_config,\ - reload_minigraph_with_golden_config, file_exists_on_dut - -NON_USER_CONFIG_TABLES = ["FLEX_COUNTER_TABLE"] +from tests.common.utilities import backup_config, restore_config, get_running_config,\ + reload_minigraph_with_golden_config, file_exists_on_dut, NON_USER_CONFIG_TABLES GOLDEN_CONFIG = "/etc/sonic/golden_config_db.json" GOLDEN_CONFIG_BACKUP = "/etc/sonic/golden_config_db.json_before_override" diff --git a/tests/override_config_table/utilities.py b/tests/override_config_table/utilities.py deleted file mode 100644 index a989983ba7..0000000000 --- a/tests/override_config_table/utilities.py +++ /dev/null @@ -1,36 +0,0 @@ -import json -import logging - -from tests.common.config_reload import config_reload - -logger = logging.getLogger(__name__) - - -def backup_config(duthost, config, config_backup): - logger.info("Backup {} to {} on {}".format( - config, config_backup, duthost.hostname)) - duthost.shell("cp {} {}".format(config, config_backup)) - - -def restore_config(duthost, config, config_backup): - logger.info("Restore {} with {} on {}".format( - config, config_backup, duthost.hostname)) - duthost.shell("mv {} {}".format(config_backup, config)) - - -def get_running_config(duthost, asic=None): - ns = "-n " + asic if asic else "" - return json.loads(duthost.shell("sonic-cfggen {} -d --print-data".format(ns))['stdout']) - - -def reload_minigraph_with_golden_config(duthost, json_data): - """ - for multi-asic/single-asic devices, we only have 1 golden_config_db.json - """ - golden_config = "/etc/sonic/golden_config_db.json" - duthost.copy(content=json.dumps(json_data, indent=4), dest=golden_config) - config_reload(duthost, config_source="minigraph", safe_reload=True, override_config=True) - - -def file_exists_on_dut(duthost, filename): - return duthost.stat(path=filename).get('stat', {}).get('exists', False) diff --git a/tests/pc/test_lag_2.py b/tests/pc/test_lag_2.py index ef9bf96542..cff700c924 100644 --- a/tests/pc/test_lag_2.py +++ b/tests/pc/test_lag_2.py @@ -3,6 +3,7 @@ import time import logging +from tests.common.fixtures.ptfhost_utils import copy_acstests_directory # noqa F401 from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 from tests.ptf_runner import ptf_runner from tests.common.fixtures.conn_graph_facts import conn_graph_facts # noqa F401 @@ -21,39 +22,14 @@ pytest.mark.topology('any'), ] -# The dir will be deleted from host, so be sure not to use system dir -TEST_DIR = "/tmp/acstests/" - - -@pytest.fixture(autouse=True) -def ignore_expected_loganalyzer_exceptions(duthosts, loganalyzer): - """Ignore expected failures logs during test execution.""" - if loganalyzer: - for duthost in duthosts: - loganalyzer[duthost.hostname].ignore_regex.extend( - [ - r".* ERR monit\[\d+\]: 'routeCheck' status failed \(255\) -- Failure results:.*", - ] - ) - - return - @pytest.fixture(scope="module") -def common_setup_teardown(ptfhost, duthosts): - logger.info("########### Setup for lag testing ###########") +def common_setup_teardown(copy_acstests_directory, copy_ptftests_directory, ptfhost, duthosts): # noqa F811 - ptfhost.shell("mkdir -p {}".format(TEST_DIR)) - # Copy PTF test into PTF-docker for test LACP DU - test_files = ['lag_test.py', 'acs_base_test.py', 'router_utils.py'] - for test_file in test_files: - src = "../ansible/roles/test/files/acstests/%s" % test_file - dst = TEST_DIR + test_file - ptfhost.copy(src=src, dest=dst) + logger.info("########### Setup for lag testing ###########") yield ptfhost - ptfhost.file(path=TEST_DIR, state="absent") # NOTE: As test_lag always causes the route_check to fail and route_check # takes more than 3 cycles(15mins) to alert, the testcase in the nightly after # the test_lag will suffer from the monit alert, so let's config reload the @@ -126,7 +102,8 @@ def __verify_lag_lacp_timing(self, lacp_timer, exp_iface): 'ether_type': 0x8809, 'interval_count': 3 } - ptf_runner(self.ptfhost, TEST_DIR, "lag_test.LacpTimingTest", '/root/ptftests', params=params) + ptf_runner(self.ptfhost, 'acstests', "lag_test.LacpTimingTest", + '/root/ptftests', params=params, is_python3=True) def __verify_lag_minlink(self, host, lag_name, lag_facts, neighbor_intf, deselect_time, wait_timeout=30): @@ -368,7 +345,7 @@ def test_lag(common_setup_teardown, duthosts, tbinfo, nbrhosts, fanouthosts, @pytest.fixture(scope='function') -def ignore_expected_loganalyzer_exceptions_lag2(duthosts, rand_one_dut_hostname, loganalyzer): +def ignore_expected_loganalyzer_exceptions_lag(duthosts, rand_one_dut_hostname, loganalyzer): """ Ignore expected failures logs during test execution. @@ -379,14 +356,14 @@ def ignore_expected_loganalyzer_exceptions_lag2(duthosts, rand_one_dut_hostname, rand_one_dut_hostname: Hostname of a random chosen dut loganalyzer: Loganalyzer utility fixture """ - # When loganalyzer is disabled, the object could be None - duthost = duthosts[rand_one_dut_hostname] - if loganalyzer: - ignoreRegex = [ - # Valid test_lag_db_status and test_lag_db_status_with_po_update - ".*ERR swss[0-9]*#orchagent: :- getPortOperSpeed.*", - ] - loganalyzer[duthost.hostname].ignore_regex.extend(ignoreRegex) + ignoreRegex = [ + r".*ERR swss[0-9]*#orchagent: :- getPortOperSpeed.*", + r".* ERR monit\[\d+\]: 'routeCheck' status failed \(255\) -- Failure results:.*", + ] + + for duthost in duthosts.frontend_nodes: + if duthost.loganalyzer: + duthost.loganalyzer.ignore_regex.extend(ignoreRegex) @pytest.fixture(scope='function') @@ -473,7 +450,7 @@ def check_link_is_down(asichost, po_intf): def test_lag_db_status(duthosts, enum_dut_portchannel_with_completeness_level, - ignore_expected_loganalyzer_exceptions_lag2): + ignore_expected_loganalyzer_exceptions_lag): # Test state_db status for lag interfaces dut_name, dut_lag = decode_dut_port_name(enum_dut_portchannel_with_completeness_level) logger.info("Start test_lag_db_status test on dut {} for lag {}".format(dut_name, dut_lag)) @@ -524,7 +501,7 @@ def test_lag_db_status(duthosts, enum_dut_portchannel_with_completeness_level, # Retrieve lag_facts after no shutdown interface asichost.startup_interface(po_intf) # Sometimes, it has to wait seconds for booting up interface - pytest_assert(wait_until(60, 1, 0, check_link_is_up, duthost, asichost, po_intf, port_info, lag_name), + pytest_assert(wait_until(180, 1, 0, check_link_is_up, duthost, asichost, po_intf, port_info, lag_name), "{} member {}'s status or netdev_oper_status in state_db is not up." .format(lag_name, po_intf)) finally: @@ -547,7 +524,7 @@ def test_lag_db_status(duthosts, enum_dut_portchannel_with_completeness_level, def test_lag_db_status_with_po_update(duthosts, teardown, enum_dut_portchannel_with_completeness_level, - ignore_expected_loganalyzer_exceptions_lag2): + ignore_expected_loganalyzer_exceptions_lag): """ test port channel add/deletion and check interface status in state_db """ diff --git a/tests/pc/test_lag_member.py b/tests/pc/test_lag_member.py index 2b7e5797f1..1dd5def445 100644 --- a/tests/pc/test_lag_member.py +++ b/tests/pc/test_lag_member.py @@ -399,6 +399,7 @@ def most_common_port_speed(duthost): port_status = cfg_facts["PORT"] number_of_lag_member = HWSKU_INTF_NUMBERS_DICT.get(duthost.facts["hwsku"], DEAFULT_NUMBER_OF_MEMBER_IN_LAG) src_vlan_id = get_vlan_id(cfg_facts, number_of_lag_member) + pytest_require(src_vlan_id != -1, "Can't get usable vlan concluding enough member") src_vlan_members = cfg_facts["VLAN_MEMBER"]["Vlan{}".format(src_vlan_id)] # specific LAG interface from t0-56-po2vlan topo, which can't be tested src_vlan_members.pop('PortChannel201', None) diff --git a/tests/pc/test_po_update.py b/tests/pc/test_po_update.py index fd142b0b92..53c1ce99aa 100644 --- a/tests/pc/test_po_update.py +++ b/tests/pc/test_po_update.py @@ -13,7 +13,7 @@ from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert -from tests.voq.voq_helpers import verify_no_routes_from_nexthop +from tests.common.helpers.voq_helpers import verify_no_routes_from_nexthop pytestmark = [ pytest.mark.topology('any'), diff --git a/tests/pc/test_po_voq.py b/tests/pc/test_po_voq.py index 3a264867a6..4f0bc7d20f 100644 --- a/tests/pc/test_po_voq.py +++ b/tests/pc/test_po_voq.py @@ -1,6 +1,6 @@ import pytest import tests.common.helpers.voq_lag as voq_lag -from tests.voq.voq_helpers import verify_no_routes_from_nexthop +from tests.common.helpers.voq_helpers import verify_no_routes_from_nexthop import logging logger = logging.getLogger(__name__) diff --git a/tests/pc/test_retry_count.py b/tests/pc/test_retry_count.py index 60404ec1fa..77c5925c3f 100644 --- a/tests/pc/test_retry_count.py +++ b/tests/pc/test_retry_count.py @@ -291,7 +291,8 @@ def test_kill_teamd_lag_up(self, duthost, nbrhosts, higher_retry_count_on_peers, pytest_assert(not status["runner"]["selected"], "lag member is still up") -def test_peer_retry_count_disabled(duthost, nbrhosts, higher_retry_count_on_peers, disable_retry_count_on_peer): +def test_peer_retry_count_disabled(duthost, nbrhosts, higher_retry_count_on_peers, disable_retry_count_on_peer, + collect_techsupport_all_nbrs): """ Test that peers reset the retry count to 3 when the feature is disabled """ @@ -364,7 +365,8 @@ def test_kill_teamd_peer_lag_up(self, duthost, nbrhosts, higher_retry_count_on_p pytest_assert(not status["runner"]["selected"], "lag member is still up") -def test_dut_retry_count_disabled(duthost, nbrhosts, higher_retry_count_on_dut, disable_retry_count_on_dut): +def test_dut_retry_count_disabled(duthost, nbrhosts, higher_retry_count_on_dut, disable_retry_count_on_dut, + collect_techsupport_all_nbrs): """ Test that DUT resets the retry count to 3 when the feature is disabled """ diff --git a/tests/pfcwd/conftest.py b/tests/pfcwd/conftest.py index 85b63e0af6..738c6fa7d0 100644 --- a/tests/pfcwd/conftest.py +++ b/tests/pfcwd/conftest.py @@ -5,8 +5,9 @@ from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 from tests.common.fixtures.ptfhost_utils import set_ptf_port_mapping_mode # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import pause_garp_service # noqa F401 from tests.common.mellanox_data import is_mellanox_device as isMellanoxDevice -from .files.pfcwd_helper import TrafficPorts, set_pfc_timers, select_test_ports +from tests.common.helpers.pfcwd_helper import TrafficPorts, set_pfc_timers, select_test_ports from tests.common.utilities import str2bool logger = logging.getLogger(__name__) @@ -103,11 +104,11 @@ def setup_pfc_test( Yields: setup_info: dictionary containing pfc timers, generated test ports and selected test ports """ - SUPPORTED_T1_TOPOS = {"t1-lag", "t1-64-lag", "t1-56-lag"} + SUPPORTED_T1_TOPOS = {"t1-lag", "t1-64-lag", "t1-56-lag", "t1-28-lag"} duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] mg_facts = duthost.get_extended_minigraph_facts(tbinfo) port_list = list(mg_facts['minigraph_ports'].keys()) - neighbors = conn_graph_facts['device_conn'][duthost.hostname] + neighbors = conn_graph_facts['device_conn'].get(duthost.hostname, {}) dut_eth0_ip = duthost.mgmt_ip vlan_nw = None @@ -209,14 +210,29 @@ def setup_dut_test_params( # icmp_responder need to be paused during the test because the test case # configures static IP address on ptf host and sends ICMP reply to DUT. -@pytest.fixture(scope="module") -def pause_icmp_responder(ptfhost): - icmp_responder_status = ptfhost.shell("supervisorctl status icmp_responder", module_ignore_errors=True)["stdout"] - if "RUNNING" not in icmp_responder_status: - yield - return - ptfhost.shell("supervisorctl stop icmp_responder", module_ignore_errors=True) +@pytest.fixture(scope="module", autouse=True) +def pfcwd_pause_service(ptfhost): + needs_resume = {"icmp_responder": False, "garp_service": False} + + out = ptfhost.shell("supervisorctl status icmp_responder", module_ignore_errors=True).get("stdout", "") + if 'RUNNING' in out: + needs_resume["icmp_responder"] = True + ptfhost.shell("supervisorctl stop icmp_responder") + + out = ptfhost.shell("supervisorctl status garp_service", module_ignore_errors=True).get("stdout", "") + if 'RUNNING' in out: + needs_resume["garp_service"] = True + ptfhost.shell("supervisorctl stop garp_service") + + logger.debug("pause_service needs_resume {}".format(needs_resume)) yield - ptfhost.shell("supervisorctl restart icmp_responder", module_ignore_errors=True) + if needs_resume["icmp_responder"]: + ptfhost.shell("supervisorctl start icmp_responder") + needs_resume["icmp_responder"] = False + if needs_resume["garp_service"]: + ptfhost.shell("supervisorctl start garp_service") + needs_resume["garp_service"] = False + + logger.debug("pause_service needs_resume {}".format(needs_resume)) diff --git a/tests/pfcwd/files/pfc_detect_mellanox.lua b/tests/pfcwd/files/pfc_detect_mellanox.lua new file mode 100644 index 0000000000..826a577d62 --- /dev/null +++ b/tests/pfcwd/files/pfc_detect_mellanox.lua @@ -0,0 +1,124 @@ +-- KEYS - queue IDs +-- ARGV[1] - counters db index +-- ARGV[2] - counters table name +-- ARGV[3] - poll time interval (milliseconds) +-- return queue Ids that satisfy criteria + +local counters_db = ARGV[1] +local counters_table_name = ARGV[2] +local poll_time = tonumber(ARGV[3]) * 1000 + +local rets = {} + +redis.call('SELECT', counters_db) + +-- Record the polling time +local timestamp_last = redis.call('HGET', 'TIMESTAMP', 'pfcwd_poll_timestamp_last') +local timestamp_struct = redis.call('TIME') +local timestamp_current = timestamp_struct[1] + timestamp_struct[2] / 1000000 +local timestamp_string = tostring(timestamp_current) +redis.call('HSET', 'TIMESTAMP', 'pfcwd_poll_timestamp_last', timestamp_string) +local real_poll_time = poll_time +if timestamp_last ~= false then + real_poll_time = (timestamp_current - tonumber(timestamp_last)) * 1000000 +end + +-- Iterate through each queue +local n = table.getn(KEYS) +for i = n, 1, -1 do + local counter_keys = redis.call('HKEYS', counters_table_name .. ':' .. KEYS[i]) + local counter_num = 0 + local old_counter_num = 0 + local is_deadlock = false + local pfc_wd_status = redis.call('HGET', counters_table_name .. ':' .. KEYS[i], 'PFC_WD_STATUS') + local pfc_wd_action = redis.call('HGET', counters_table_name .. ':' .. KEYS[i], 'PFC_WD_ACTION') + + local big_red_switch_mode = redis.call('HGET', counters_table_name .. ':' .. KEYS[i], 'BIG_RED_SWITCH_MODE') + if not big_red_switch_mode and (pfc_wd_status == 'operational' or pfc_wd_action == 'alert') then + local detection_time = redis.call('HGET', counters_table_name .. ':' .. KEYS[i], 'PFC_WD_DETECTION_TIME') + if detection_time then + detection_time = tonumber(detection_time) + local time_left = redis.call('HGET', counters_table_name .. ':' .. KEYS[i], 'PFC_WD_DETECTION_TIME_LEFT') + if not time_left then + time_left = detection_time + else + time_left = tonumber(time_left) + end + + local queue_index = redis.call('HGET', 'COUNTERS_QUEUE_INDEX_MAP', KEYS[i]) + local port_id = redis.call('HGET', 'COUNTERS_QUEUE_PORT_MAP', KEYS[i]) + -- If there is no entry in COUNTERS_QUEUE_INDEX_MAP or COUNTERS_QUEUE_PORT_MAP then + -- it means KEYS[i] queue is inserted into FLEX COUNTER DB but the corresponding + -- maps haven't been updated yet. + if queue_index and port_id then + local pfc_rx_pkt_key = 'SAI_PORT_STAT_PFC_' .. queue_index .. '_RX_PKTS' + local pfc_duration_key = 'SAI_PORT_STAT_PFC_' .. queue_index .. '_RX_PAUSE_DURATION_US' + + -- Get all counters + local occupancy_bytes = redis.call('HGET', counters_table_name .. ':' .. KEYS[i], 'SAI_QUEUE_STAT_CURR_OCCUPANCY_BYTES') + local packets = redis.call('HGET', counters_table_name .. ':' .. KEYS[i], 'SAI_QUEUE_STAT_PACKETS') + local pfc_rx_packets = redis.call('HGET', counters_table_name .. ':' .. port_id, pfc_rx_pkt_key) + local pfc_duration = redis.call('HGET', counters_table_name .. ':' .. port_id, pfc_duration_key) + + if occupancy_bytes and packets and pfc_rx_packets and pfc_duration then + occupancy_bytes = tonumber(occupancy_bytes) + packets = tonumber(packets) + pfc_rx_packets = tonumber(pfc_rx_packets) + pfc_duration = tonumber(pfc_duration) + + local packets_last = redis.call('HGET', counters_table_name .. ':' .. KEYS[i], 'SAI_QUEUE_STAT_PACKETS_last') + local pfc_rx_packets_last = redis.call('HGET', counters_table_name .. ':' .. port_id, pfc_rx_pkt_key .. '_last') + local pfc_duration_last = redis.call('HGET', counters_table_name .. ':' .. port_id, pfc_duration_key .. '_last') + -- DEBUG CODE START. Uncomment to enable + local debug_storm = redis.call('HGET', counters_table_name .. ':' .. KEYS[i], 'DEBUG_STORM') + -- DEBUG CODE END. + + -- If this is not a first run, then we have last values available + if packets_last and pfc_rx_packets_last and pfc_duration_last then + packets_last = tonumber(packets_last) + pfc_rx_packets_last = tonumber(pfc_rx_packets_last) + pfc_duration_last = tonumber(pfc_duration_last) + local storm_condition = (pfc_duration - pfc_duration_last) > (poll_time * 0.8) + + -- Check actual condition of queue being in PFC storm + if (occupancy_bytes > 0 and packets - packets_last == 0 and pfc_rx_packets - pfc_rx_packets_last > 0) or + -- DEBUG CODE START. Uncomment to enable + (debug_storm == "enabled") or + -- DEBUG CODE END. + (occupancy_bytes == 0 and packets - packets_last == 0 and storm_condition) then + if time_left <= poll_time then + redis.call('HDEL', counters_table_name .. ':' .. port_id, pfc_rx_pkt_key .. '_last') + redis.call('HDEL', counters_table_name .. ':' .. port_id, pfc_duration_key .. '_last') + local occupancy_string = '"occupancy","' .. tostring(occupancy_bytes) .. '",' + local packets_string = '"packets","' .. tostring(packets) .. '","packets_last","' .. tostring(packets_last) .. '",' + local pfc_rx_packets_string = '"pfc_rx_packets","' .. tostring(pfc_rx_packets) .. '","pfc_rx_packets_last","' .. tostring(pfc_rx_packets_last) .. '",' + local storm_condition_string = '"pfc_duration","' .. tostring(pfc_duration) .. '","pfc_duration_last","' .. tostring(pfc_duration_last) .. '",' + local timestamps = '"timestamp","' .. timestamp_string .. '","timestamp_last","' .. timestamp_last .. '","real_poll_time","' .. real_poll_time .. '"' + redis.call('PUBLISH', 'PFC_WD_ACTION', '["' .. KEYS[i] .. '","storm",' .. occupancy_string .. packets_string .. pfc_rx_packets_string .. storm_condition_string .. timestamps .. ']') + is_deadlock = true + time_left = detection_time + else + time_left = time_left - poll_time + end + else + if pfc_wd_action == 'alert' and pfc_wd_status ~= 'operational' then + redis.call('PUBLISH', 'PFC_WD_ACTION', '["' .. KEYS[i] .. '","restore"]') + end + time_left = detection_time + end + end + + -- Save values for next run + redis.call('HSET', counters_table_name .. ':' .. KEYS[i], 'SAI_QUEUE_STAT_PACKETS_last', packets) + redis.call('HSET', counters_table_name .. ':' .. KEYS[i], 'PFC_WD_DETECTION_TIME_LEFT', time_left) + if is_deadlock == false then + redis.call('HSET', counters_table_name .. ':' .. port_id, pfc_rx_pkt_key .. '_last', pfc_rx_packets) + redis.call('HSET', counters_table_name .. ':' .. port_id, pfc_duration_key .. '_last', pfc_duration) + end + end + end + end + end +end + +return rets diff --git a/tests/pfcwd/test_pfcwd_all_port_storm.py b/tests/pfcwd/test_pfcwd_all_port_storm.py index 5c652c7cfc..8cffd960b6 100644 --- a/tests/pfcwd/test_pfcwd_all_port_storm.py +++ b/tests/pfcwd/test_pfcwd_all_port_storm.py @@ -6,11 +6,14 @@ from tests.common.fixtures.conn_graph_facts import enum_fanout_graph_facts # noqa F401 from tests.common.helpers.pfc_storm import PFCMultiStorm from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer -from .files.pfcwd_helper import start_wd_on_ports, start_background_traffic # noqa F401 -from .files.pfcwd_helper import EXPECT_PFC_WD_DETECT_RE, EXPECT_PFC_WD_RESTORE_RE, fetch_vendor_specific_diagnosis_re -from .files.pfcwd_helper import send_background_traffic +from tests.common.helpers.pfcwd_helper import start_wd_on_ports, start_background_traffic # noqa F401 +from tests.common.helpers.pfcwd_helper import EXPECT_PFC_WD_DETECT_RE, EXPECT_PFC_WD_RESTORE_RE, \ + fetch_vendor_specific_diagnosis_re +from tests.common.helpers.pfcwd_helper import send_background_traffic +from tests.common import config_reload TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates") +FILE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "files") pytestmark = [ pytest.mark.disable_loganalyzer, @@ -26,6 +29,50 @@ def pfc_queue_idx(): yield 3 # Hardcoded in the testcase as well. +@pytest.fixture(scope='module') +def degrade_pfcwd_detection(duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts): + """ + A fixture to degrade PFC Watchdog detection logic. + It's requried because leaf fanout switch can't generate enough PFC pause to trigger + PFC storm on all ports. + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + dut_asic_type = duthost.facts["asic_type"].lower() + skip_fixture = False + if dut_asic_type != "mellanox": + skip_fixture = True + # The workaround is not applicable for Mellanox leaf-fanout running ONYX or SONiC + # as we can leverage ASIC to generate PFC pause frames + for fanouthost in list(fanouthosts.values()): + fanout_os = fanouthost.get_fanout_os() + if fanout_os == 'onyx' or fanout_os == 'sonic' and fanouthost.facts['asic_type'] == "mellanox": + skip_fixture = True + break + if skip_fixture: + yield + return + logger.info("--- Degrade PFCWD detection logic --") + SRC_FILE = FILE_DIR + "/pfc_detect_mellanox.lua" + DST_FILE = "/usr/share/swss/pfc_detect_mellanox.lua" + # Backup original PFC Watchdog detection script + cmd = "docker exec -i swss cp {} {}.bak".format(DST_FILE, DST_FILE) + duthost.shell(cmd) + # Copy the new script to DUT + duthost.copy(src=SRC_FILE, dest='/tmp') + # Copy the new script to swss container + cmd = "docker cp /tmp/pfc_detect_mellanox.lua swss:{}".format(DST_FILE) + duthost.shell(cmd) + # Reload DUT to apply the new script + config_reload(duthost, safe_reload=True, check_intf_up_ports=True, wait_for_bgp=True) + yield + # Restore the original PFC Watchdog detection script + cmd = "docker exec -i swss cp {}.bak {}".format(DST_FILE, DST_FILE) + duthost.shell(cmd) + config_reload(duthost, safe_reload=True, check_intf_up_ports=True, wait_for_bgp=True) + # Cleanup + duthost.file(path='/tmp/pfc_detect_mellanox.lua', state='absent') + + @pytest.fixture(scope='class', autouse=True) def stop_pfcwd(duthosts, enum_rand_one_per_hwsku_frontend_hostname): """ @@ -120,7 +167,7 @@ def set_storm_params(duthost, fanout_graph, fanouthosts, peer_params): return storm_hndle -@pytest.mark.usefixtures('stop_pfcwd', 'storm_test_setup_restore', 'start_background_traffic') +@pytest.mark.usefixtures('degrade_pfcwd_detection', 'stop_pfcwd', 'storm_test_setup_restore', 'start_background_traffic') # noqa E501 class TestPfcwdAllPortStorm(object): """ PFC storm test class """ def run_test(self, duthost, storm_hndle, expect_regex, syslog_marker, action): diff --git a/tests/pfcwd/test_pfcwd_cli.py b/tests/pfcwd/test_pfcwd_cli.py new file mode 100644 index 0000000000..e504d782e4 --- /dev/null +++ b/tests/pfcwd/test_pfcwd_cli.py @@ -0,0 +1,475 @@ +import datetime +import logging +import pytest +import time + +from tests.common.fixtures.conn_graph_facts import enum_fanout_graph_facts # noqa F401 +from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.pfc_storm import PFCStorm +from tests.common.helpers.pfcwd_helper import start_wd_on_ports +from tests.common.helpers.pfcwd_helper import has_neighbor_device +from tests.ptf_runner import ptf_runner +from tests.common import constants +from tests.common.dualtor.dual_tor_utils import is_tunnel_qos_remap_enabled, dualtor_ports # noqa F401 +from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m # noqa F401, E501 +from tests.common.helpers.pfcwd_helper import send_background_traffic, check_pfc_storm_state, parser_show_pfcwd_stat +from tests.common.utilities import wait_until + +pytestmark = [ + pytest.mark.topology("t0", "t1") +] + +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope='function', autouse=True) +def stop_pfcwd(duthosts, enum_rand_one_per_hwsku_frontend_hostname): + """ + Fixture that stops PFC Watchdog before each test run + + Args: + duthost(AnsibleHost) : dut instance + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + logger.info("--- Stop Pfcwd --") + duthost.command("pfcwd stop") + + yield + logger.info("--- Start Pfcwd --") + duthost.command("pfcwd start_default") + + +class SetupPfcwdFunc(object): + def parse_test_port_info(self): + """ + Parse the test port information into a dict + {port_id: port_type} + """ + self.port_id_to_type_map = dict() + for _, v in self.test_ports_info.items(): + self.port_id_to_type_map[v['test_port_id']] = v['test_port_type'] + + """ Test setup per port """ + def setup_test_params(self, port, vlan, init=False, detect=True): + """ + Sets up test parameters associated with a DUT port + + Args: + port(string) : DUT port + vlan(dict) : DUT vlan info + init(bool) : If the fanout needs to be initialized or not + """ + logger.info("--- Setting up test params for port {} ---".format(port)) + self.parse_test_port_info() + self.setup_port_params(port, init=init, detect=detect) + self.resolve_arp(vlan, self.is_dualtor) + self.storm_setup(init=init, detect=detect) + + def setup_port_params(self, port, init=False, detect=True): + """ + Gather all the parameters needed for storm generation and ptf test based off the DUT port + + Args: + port(string) : DUT port + """ + self.pfc_wd = dict() + self.pfc_wd['test_pkt_count'] = 100 + self.pfc_wd['queue_index'] = 4 + self.pfc_wd['frames_number'] = 100000000 + self.pfc_wd['test_port_ids'] = list() + self.peer_device = self.ports[port]['peer_device'] + self.pfc_wd['test_port'] = port + self.pfc_wd['rx_port'] = self.ports[port]['rx_port'] + self.pfc_wd['test_neighbor_addr'] = self.ports[port]['test_neighbor_addr'] + self.pfc_wd['rx_neighbor_addr'] = self.ports[port]['rx_neighbor_addr'] + self.pfc_wd['test_port_id'] = self.ports[port]['test_port_id'] + self.pfc_wd['rx_port_id'] = self.ports[port]['rx_port_id'] + self.pfc_wd['port_type'] = self.ports[port]['test_port_type'] + if self.pfc_wd['port_type'] == "portchannel": + self.pfc_wd['test_port_ids'] = self.ports[port]['test_portchannel_members'] + elif self.pfc_wd['port_type'] in ["vlan", "interface"]: + self.pfc_wd['test_port_ids'] = self.pfc_wd['test_port_id'] + self.pfc_wd['test_port_vlan_id'] = self.ports[port].get('test_port_vlan_id') + self.pfc_wd['rx_port_vlan_id'] = self.ports[port].get('rx_port_vlan_id') + self.pfc_wd['port_id_to_type_map'] = self.port_id_to_type_map + self.queue_oid = self.dut.get_queue_oid(port, self.pfc_wd['queue_index']) + + def update_queue(self, port): + """ + Switch between queue 3 and 4 during the test + + Args: + port(string) : DUT port + """ + if self.pfc_wd['queue_index'] == 4: + self.pfc_wd['queue_index'] = self.pfc_wd['queue_index'] - 1 + else: + self.pfc_wd['queue_index'] = self.pfc_wd['queue_index'] + 1 + logger.info("Current queue: {}".format(self.pfc_wd['queue_index'])) + self.queue_oid = self.dut.get_queue_oid(port, self.pfc_wd['queue_index']) + + def resolve_arp(self, vlan, is_dualtor=False): + """ + Populate ARP info for the DUT vlan port + + Args: + vlan(dict) : DUT vlan info + """ + if self.pfc_wd['port_type'] == "vlan": + self.ptf.script("./scripts/remove_ip.sh") + ptf_port = 'eth%s' % self.pfc_wd['test_port_id'] + if self.pfc_wd['test_port_vlan_id'] is not None: + ptf_port += (constants.VLAN_SUB_INTERFACE_SEPARATOR + self.pfc_wd['test_port_vlan_id']) + self.ptf.command("ip neigh flush all") + self.ptf.command("ip -6 neigh flush all") + self.dut.command("ip neigh flush all") + self.dut.command("ip -6 neigh flush all") + self.ptf.command("ifconfig {} {}".format(ptf_port, self.pfc_wd['test_neighbor_addr'])) + self.ptf.command("ping {} -c 10".format(vlan['addr'])) + + if is_dualtor: + self.dut.command("docker exec -i swss arping {} -c 5".format(self.pfc_wd['test_neighbor_addr']), module_ignore_errors=True) # noqa: E501 + else: + self.dut.command("docker exec -i swss arping {} -c 5".format(self.pfc_wd['test_neighbor_addr'])) + + def storm_setup(self, init=False, detect=True): + """ + Prepare fanout for the storm generation + + Args: + init(bool): if the storm class needs to be initialized or not + """ + # new peer device + if not self.peer_dev_list or self.peer_device not in self.peer_dev_list: + peer_info = {'peerdevice': self.peer_device, + 'hwsku': self.fanout_info[self.peer_device]['device_info']['HwSku'], + 'pfc_fanout_interface': self.neighbors[self.pfc_wd['test_port']]['peerport'] + } + self.peer_dev_list[self.peer_device] = peer_info['hwsku'] + + if self.dut.topo_type == 't2' and self.fanout[self.peer_device].os == 'sonic': + gen_file = 'pfc_gen_t2.py' + pfc_send_time = 60 + else: + gen_file = 'pfc_gen.py' + pfc_send_time = None + + # get pfc storm handle + if init and detect: + self.storm_hndle = PFCStorm(self.dut, self.fanout_info, self.fanout, + pfc_queue_idx=self.pfc_wd['queue_index'], + pfc_frames_number=self.pfc_wd['frames_number'], + pfc_send_period=pfc_send_time, + pfc_gen_file=gen_file, + peer_info=peer_info) + self.storm_hndle.update_queue_index(self.pfc_wd['queue_index']) + self.storm_hndle.update_peer_info(peer_info) + self.storm_hndle.deploy_pfc_gen() + + # peer device already exists. only interface changes + else: + peer_info = {'peerdevice': self.peer_device, + 'hwsku': self.peer_dev_list[self.peer_device], + 'pfc_fanout_interface': self.neighbors[self.pfc_wd['test_port']]['peerport'] + } + + self.storm_hndle.update_queue_index(self.pfc_wd['queue_index']) + self.storm_hndle.update_peer_info(peer_info) + + +class SendVerifyTraffic(): + """ PTF test """ + def __init__(self, ptf, router_mac, tx_mac, pfc_params, is_dualtor): + """ + Args: + ptf(AnsibleHost) : ptf instance + router_mac(string) : router mac address + ptf_params(dict) : all PFC test params specific to the DUT port + """ + self.ptf = ptf + self.router_mac = router_mac + self.tx_mac = tx_mac + self.pfc_queue_index = pfc_params['queue_index'] + self.pfc_wd_test_pkt_count = pfc_params['test_pkt_count'] + self.pfc_wd_rx_port_id = pfc_params['rx_port_id'] + self.pfc_wd_test_port = pfc_params['test_port'] + self.pfc_wd_test_port_id = pfc_params['test_port_id'] + self.pfc_wd_test_port_ids = pfc_params['test_port_ids'] + self.pfc_wd_test_neighbor_addr = pfc_params['test_neighbor_addr'] + self.pfc_wd_rx_neighbor_addr = pfc_params['rx_neighbor_addr'] + self.pfc_wd_test_port_vlan_id = pfc_params['test_port_vlan_id'] + self.pfc_wd_rx_port_vlan_id = pfc_params['rx_port_vlan_id'] + self.port_id_to_type_map = pfc_params['port_id_to_type_map'] + self.port_type = pfc_params['port_type'] + if is_dualtor: + self.vlan_mac = "00:aa:bb:cc:dd:ee" + else: + self.vlan_mac = router_mac + + def send_tx_egress(self, action, verify): + """ + Send traffic with test port as the egress and verify if the packets get forwarded + or dropped based on the action + + Args: + action(string) : PTF test action + """ + logger.info("Check for egress {} on Tx port {}".format(action, self.pfc_wd_test_port)) + dst_port = "[" + str(self.pfc_wd_test_port_id) + "]" + if action == "forward" and type(self.pfc_wd_test_port_ids) == list: + dst_port = "".join(str(self.pfc_wd_test_port_ids)).replace(',', '') + ptf_params = {'router_mac': self.router_mac, + 'vlan_mac': self.vlan_mac, + 'queue_index': self.pfc_queue_index, + 'pkt_count': self.pfc_wd_test_pkt_count, + 'port_src': self.pfc_wd_rx_port_id[0], + 'port_dst': dst_port, + 'ip_dst': self.pfc_wd_test_neighbor_addr, + 'port_type': self.port_id_to_type_map[self.pfc_wd_rx_port_id[0]], + 'wd_action': action if verify else "dontcare"} + if self.pfc_wd_rx_port_vlan_id is not None: + ptf_params['port_src_vlan_id'] = self.pfc_wd_rx_port_vlan_id + if self.pfc_wd_test_port_vlan_id is not None: + ptf_params['port_dst_vlan_id'] = self.pfc_wd_test_port_vlan_id + log_format = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + log_file = "/tmp/pfc_wd.PfcWdTest.{}.log".format(log_format) + ptf_runner(self.ptf, "ptftests", "pfc_wd.PfcWdTest", "ptftests", params=ptf_params, + log_file=log_file, is_python3=True) + + def send_rx_ingress(self, action, verify): + """ + Send traffic with test port as the ingress and verify if the packets get forwarded + or dropped based on the action + + Args: + action(string) : PTF test action + """ + logger.info("Check for ingress {} on Rx port {}".format(action, self.pfc_wd_test_port)) + if type(self.pfc_wd_rx_port_id) == list: + dst_port = "".join(str(self.pfc_wd_rx_port_id)).replace(',', '') + else: + dst_port = "[ " + str(self.pfc_wd_rx_port_id) + " ]" + ptf_params = {'router_mac': self.tx_mac, + 'vlan_mac': self.vlan_mac, + 'queue_index': self.pfc_queue_index, + 'pkt_count': self.pfc_wd_test_pkt_count, + 'port_src': self.pfc_wd_test_port_id, + 'port_dst': dst_port, + 'ip_dst': self.pfc_wd_rx_neighbor_addr, + 'port_type': self.port_id_to_type_map[self.pfc_wd_test_port_id], + 'wd_action': action if verify else "dontcare"} + if self.pfc_wd_rx_port_vlan_id is not None: + ptf_params['port_dst_vlan_id'] = self.pfc_wd_rx_port_vlan_id + if self.pfc_wd_test_port_vlan_id is not None: + ptf_params['port_src_vlan_id'] = self.pfc_wd_test_port_vlan_id + log_format = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + log_file = "/tmp/pfc_wd.PfcWdTest.{}.log".format(log_format) + ptf_runner(self.ptf, "ptftests", "pfc_wd.PfcWdTest", "ptftests", params=ptf_params, + log_file=log_file, is_python3=True) + + +class TestPfcwdFunc(SetupPfcwdFunc): + """ Test PFC function and supporting methods """ + def storm_detect_path(self, dut, port, action): + """ + Storm detection action and associated verifications + + Args: + dut(AnsibleHost) : DUT instance + port(string) : DUT port + action(string) : PTF test action + + Returns: + loganalyzer(Loganalyzer) : instance + """ + restore_time = self.timers['pfc_wd_restore_time_large'] + detect_time = self.timers['pfc_wd_detect_time'] + + selected_test_ports = [self.pfc_wd['rx_port'][0]] + test_ports_info = {self.pfc_wd['rx_port'][0]: self.pfc_wd} + queues = [self.storm_hndle.pfc_queue_idx] + + with send_background_traffic(dut, self.ptf, queues, selected_test_ports, test_ports_info): + if action != "dontcare": + start_wd_on_ports(dut, port, restore_time, detect_time, action) + + self.storm_hndle.start_storm() + + logger.info("Verify if PFC storm is detected on port {}".format(port)) + pytest_assert( + wait_until(30, 2, 5, check_pfc_storm_state, dut, port, self.storm_hndle.pfc_queue_idx, "storm"), + "PFC storm state did not change as expected" + ) + + def storm_restore_path(self, dut, port): + """ + Storm restoration action and associated verifications + + Args: + loganalyzer(Loganalyzer) : loganalyzer instance + port(string) : DUT port + action(string) : PTF test action + """ + + self.storm_hndle.stop_storm() + time.sleep(self.timers['pfc_wd_wait_for_restore_time']) + # storm restore + logger.info("Verify if PFC storm is restored on port {}".format(port)) + pytest_assert( + wait_until(30, 2, 5, check_pfc_storm_state, dut, port, self.storm_hndle.pfc_queue_idx, "restore"), + "PFC storm state did not change as expected" + ) + + def run_test(self, dut, port, action): + """ + Test method that invokes the storm detection and restoration path which includes the traffic + test and associated counter verifications + + Args: + dut(AnsibleHost) : DUT instance + port(string) : DUT port + action(string) : PTF test action + """ + pfcwd_stat = self.dut.show_and_parse('show pfcwd stat') + logger.info("before storm start: pfcwd_stat {}".format(pfcwd_stat)) + + logger.info("--- Storm detection path for port {} ---".format(port)) + self.storm_detect_path(dut, port, action) + # record the initial state of the DUT + pfcwd_stat_init = parser_show_pfcwd_stat(dut, port, self.pfc_wd['queue_index']) + logger.debug("pfcwd_stat_init {}".format(pfcwd_stat_init)) + + pytest_assert(("storm" in pfcwd_stat_init[0]['status']), "PFC storm status not detected") + pytest_assert( + ((int(pfcwd_stat_init[0]['storm_detect_count']) - int(pfcwd_stat_init[0]['restored_count'])) == 1), + "PFC storm detect count not correct" + ) + + # send traffic to egress port + self.traffic_inst.send_tx_egress(self.tx_action, False) + pfcwd_stat_after_tx = parser_show_pfcwd_stat(dut, port, self.pfc_wd['queue_index']) + logger.debug("pfcwd_stat_after_tx {}".format(pfcwd_stat_after_tx)) + # check count, drop: tx_drop_count; forward: tx_ok_count + if self.tx_action == "drop": + tx_drop_count_init = int(pfcwd_stat_init[0]['tx_drop_count']) + tx_drop_count_check = int(pfcwd_stat_after_tx[0]['tx_drop_count']) + logger.info("tx_drop_count {} -> {}".format(tx_drop_count_init, tx_drop_count_check)) + pytest_assert( + ((tx_drop_count_check - tx_drop_count_init) >= self.pfc_wd['test_pkt_count']), + "PFC storm Tx ok count not correct" + ) + elif self.tx_action == "forward": + tx_ok_count_init = int(pfcwd_stat_init[0]['tx_ok_count']) + tx_ok_count_check = int(pfcwd_stat_after_tx[0]['tx_ok_count']) + logger.info("tx_ok_count {} -> {}".format(tx_ok_count_init, tx_ok_count_check)) + pytest_assert( + ((tx_ok_count_check - tx_ok_count_init) >= self.pfc_wd['test_pkt_count']), + "PFC storm Tx ok count not correct" + ) + + # send traffic to ingress port + time.sleep(3) + self.traffic_inst.send_rx_ingress(self.rx_action, False) + pfcwd_stat_after_rx = parser_show_pfcwd_stat(dut, port, self.pfc_wd['queue_index']) + logger.debug("pfcwd_stat_after_rx {}".format(pfcwd_stat_after_rx)) + # check count, drop: rx_drop_count; forward: rx_ok_count + if self.rx_action == "drop": + rx_drop_count_init = int(pfcwd_stat_init[0]['rx_drop_count']) + rx_drop_count_check = int(pfcwd_stat_after_rx[0]['rx_drop_count']) + logger.info("rx_drop_count {} -> {}".format(rx_drop_count_init, rx_drop_count_check)) + pytest_assert( + ((rx_drop_count_check - rx_drop_count_init) >= self.pfc_wd['test_pkt_count']), + "PFC storm Rx drop count not correct" + ) + elif self.rx_action == "forward": + rx_ok_count_init = int(pfcwd_stat_init[0]['rx_ok_count']) + rx_ok_count_check = int(pfcwd_stat_after_rx[0]['rx_ok_count']) + logger.info("rx_ok_count {} -> {}".format(rx_ok_count_init, rx_ok_count_check)) + pytest_assert( + ((rx_ok_count_check - rx_ok_count_init) >= self.pfc_wd['test_pkt_count']), + "PFC storm Rx ok count not correct" + ) + + logger.info("--- Storm restoration path for port {} ---".format(port)) + self.storm_restore_path(dut, port) + + def set_traffic_action(self, duthost, action): + action = action if action != "dontcare" else "drop" + if duthost.facts["asic_type"] in ["mellanox", "cisco-8000", "innovium"] or is_tunnel_qos_remap_enabled(duthost): + self.rx_action = "forward" + else: + self.rx_action = action + self.tx_action = action + + def test_pfcwd_show_stat(self, request, setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts, ptfhost, # noqa F811 + duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, + setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally, + toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811 + """ + PFCwd CLI show pfcwd stats test + + Args: + request(object) : pytest request object + setup_pfc_test(fixture) : Module scoped autouse fixture for PFCwd + enum_fanout_graph_facts(fixture) : fanout graph info + ptfhost(AnsibleHost) : ptf host instance + duthost(AnsibleHost) : DUT instance + fanouthosts(AnsibleHost): fanout instance + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + setup_info = setup_pfc_test + setup_dut_info = setup_dut_test_params + self.fanout_info = enum_fanout_graph_facts + self.ptf = ptfhost + self.dut = duthost + self.fanout = fanouthosts + self.timers = setup_info['pfc_timers'] + self.ports = setup_info['selected_test_ports'] + self.test_ports_info = setup_info['test_ports'] + if self.dut.topo_type == 't2': + key, value = list(self.ports.items())[0] + self.ports = {key: value} + self.neighbors = setup_info['neighbors'] + self.peer_dev_list = dict() + self.storm_hndle = None + self.rx_action = None + self.tx_action = None + self.is_dualtor = setup_dut_info['basicParams']['is_dualtor'] + + # skip the pytest when the device does not have neighbors + # 'rx_port' being None indicates there are no ports available to receive frames for pfc storm + if not has_neighbor_device(setup_pfc_test): + pytest.skip("Test skipped: No neighbors detected as 'rx_port' is None for selected test ports," + " which is necessary for PFCwd test setup.") + + # for idx, port in enumerate(self.ports): + port = list(self.ports.keys())[0] + logger.info("--- Testing various Pfcwd actions on {} ---".format(port)) + self.setup_test_params(port, setup_info['vlan'], init=True) + self.traffic_inst = SendVerifyTraffic( + self.ptf, + duthost.get_dut_iface_mac(self.pfc_wd['rx_port'][0]), + duthost.get_dut_iface_mac(self.pfc_wd['test_port']), + self.pfc_wd, + self.is_dualtor) + + pfc_wd_restore_time_large = request.config.getoption("--restore-time") + # wait time before we check the logs for the 'restore' signature. 'pfc_wd_restore_time_large' is in ms. + self.timers['pfc_wd_wait_for_restore_time'] = int(pfc_wd_restore_time_large / 1000 * 2) + actions = ['drop', 'forward'] + for action in actions: + logger.info("--- Pfcwd port {} set action {} ---".format(port, action)) + try: + self.set_traffic_action(duthost, action) + logger.info("Pfcwd action {} on port {}: Tx traffic action {}, Rx traffic action {} ". + format(action, port, self.tx_action, self.rx_action)) + self.run_test(self.dut, port, action) + except Exception as e: + pytest.fail(str(e)) + + finally: + if self.storm_hndle: + logger.info("--- Stop pfc storm on port {}".format(port)) + self.storm_hndle.stop_storm() + logger.info("--- Stop PFC WD ---") + self.dut.command("pfcwd stop") diff --git a/tests/pfcwd/test_pfcwd_function.py b/tests/pfcwd/test_pfcwd_function.py index 987db27374..92c5d12e01 100644 --- a/tests/pfcwd/test_pfcwd_function.py +++ b/tests/pfcwd/test_pfcwd_function.py @@ -10,14 +10,17 @@ from tests.common.helpers.assertions import pytest_assert, pytest_require from tests.common.helpers.pfc_storm import PFCStorm from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer -from .files.pfcwd_helper import start_wd_on_ports -from .files.pfcwd_helper import EXPECT_PFC_WD_DETECT_RE, EXPECT_PFC_WD_RESTORE_RE, fetch_vendor_specific_diagnosis_re +from tests.common.helpers.pfcwd_helper import start_wd_on_ports +from tests.common.helpers.pfcwd_helper import EXPECT_PFC_WD_DETECT_RE, EXPECT_PFC_WD_RESTORE_RE, \ + fetch_vendor_specific_diagnosis_re +from tests.common.helpers.pfcwd_helper import has_neighbor_device from tests.ptf_runner import ptf_runner from tests.common import port_toggle from tests.common import constants from tests.common.dualtor.dual_tor_utils import is_tunnel_qos_remap_enabled, dualtor_ports # noqa F401 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m # noqa F401, E501 -from .files.pfcwd_helper import send_background_traffic +from tests.common.helpers.pfcwd_helper import send_background_traffic, check_pfc_storm_state +from tests.common.utilities import wait_until PTF_PORT_MAPPING_MODE = 'use_orig_interface' @@ -432,6 +435,10 @@ def resolve_arp(self, vlan, is_dualtor=False): ptf_port = 'eth%s' % self.pfc_wd['test_port_id'] if self.pfc_wd['test_port_vlan_id'] is not None: ptf_port += (constants.VLAN_SUB_INTERFACE_SEPARATOR + self.pfc_wd['test_port_vlan_id']) + self.ptf.command("ip neigh flush all") + self.ptf.command("ip -6 neigh flush all") + self.dut.command("ip neigh flush all") + self.dut.command("ip -6 neigh flush all") self.ptf.command("ifconfig {} {}".format(ptf_port, self.pfc_wd['test_neighbor_addr'])) self.ptf.command("ping {} -c 10".format(vlan['addr'])) @@ -728,7 +735,19 @@ def storm_detect_path(self, dut, port, action): if self.pfc_wd['fake_storm']: PfcCmd.set_storm_status(dut, self.queue_oid, "enabled") - time.sleep(5) + if dut.facts['asic_type'] == "mellanox": + # On Mellanox platform, more time is required for PFC storm being triggered + # as PFC pause sent from Non-Mellanox leaf fanout is not continuous sometimes. + PFC_STORM_TIMEOUT = 30 + pytest_assert( + wait_until( + PFC_STORM_TIMEOUT, 2, 5, + check_pfc_storm_state, dut, port, self.storm_hndle.pfc_queue_idx, "storm" + ), + "PFC storm state did not change as expected" + ) + else: + time.sleep(5) # storm detect logger.info("Verify if PFC storm is detected on port {}".format(port)) @@ -838,7 +857,7 @@ def set_traffic_action(self, duthost, action): def test_pfcwd_actions(self, request, fake_storm, setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts, # noqa F811 ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally, # noqa F811 - pause_icmp_responder, toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811 + toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811 """ PFCwd functional test @@ -872,6 +891,12 @@ def test_pfcwd_actions(self, request, fake_storm, setup_pfc_test, setup_dut_test self.tx_action = None self.is_dualtor = setup_dut_info['basicParams']['is_dualtor'] + # skip the pytest when the device does not have neighbors + # 'rx_port' being None indicates there are no ports available to receive frames for pfc storm + if not has_neighbor_device(setup_pfc_test): + pytest.skip("Test skipped: No neighbors detected as 'rx_port' is None for selected test ports," + " which is necessary for PFCwd test setup.") + for idx, port in enumerate(self.ports): logger.info("") logger.info("--- Testing various Pfcwd actions on {} ---".format(port)) @@ -912,7 +937,7 @@ def test_pfcwd_actions(self, request, fake_storm, setup_pfc_test, setup_dut_test def test_pfcwd_multi_port(self, request, fake_storm, setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts, # noqa F811 ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally, # noqa F811 - pause_icmp_responder, toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811 + toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811 """ Tests pfcwd behavior when 2 ports are under pfc storm one after the other @@ -961,6 +986,12 @@ def test_pfcwd_multi_port(self, request, fake_storm, setup_pfc_test, setup_dut_t self.set_traffic_action(duthost, "drop") self.stats = PfcPktCntrs(self.dut, self.rx_action, self.tx_action) + # skip the pytest when the device does not have neighbors + # 'rx_port' being None indicates there are no ports available to receive frames for pfc storm + if not has_neighbor_device(setup_pfc_test): + pytest.skip("Test skipped: No neighbors detected as 'rx_port' is None for selected test ports," + " which is necessary for PFCwd test setup.") + for count in range(2): try: for idx, port in enumerate(selected_ports): @@ -990,7 +1021,7 @@ def test_pfcwd_multi_port(self, request, fake_storm, setup_pfc_test, setup_dut_t def test_pfcwd_mmu_change(self, request, fake_storm, setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts, # noqa F811 ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, dualtor_ports, # noqa F811 setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally, # noqa F811 - pause_icmp_responder, toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811 + toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811 """ Tests if mmu changes impact Pfcwd functionality @@ -1043,6 +1074,12 @@ def test_pfcwd_mmu_change(self, request, fake_storm, setup_pfc_test, setup_dut_t self.set_traffic_action(duthost, "drop") self.stats = PfcPktCntrs(self.dut, self.rx_action, self.tx_action) + # skip the pytest when the device does not have neighbors + # 'rx_port' being None indicates there are no ports available to receive frames for pfc storm + if not has_neighbor_device(setup_pfc_test): + pytest.skip("Test skipped: No neighbors detected as 'rx_port' is None for selected test ports," + " which is necessary for PFCwd test setup.") + try: for idx, mmu_action in enumerate(MMU_ACTIONS): self.traffic_inst = SendVerifyTraffic( @@ -1081,7 +1118,7 @@ def test_pfcwd_mmu_change(self, request, fake_storm, setup_pfc_test, setup_dut_t def test_pfcwd_port_toggle(self, request, fake_storm, setup_pfc_test, setup_dut_test_params, enum_fanout_graph_facts, # noqa F811 tbinfo, ptfhost, duthosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m_unconditionally, # noqa F811 - pause_icmp_responder, toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811 + toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m): # noqa F811 """ Test PfCWD functionality after toggling port @@ -1127,6 +1164,12 @@ def test_pfcwd_port_toggle(self, request, fake_storm, setup_pfc_test, setup_dut_ self.is_dualtor = setup_dut_info['basicParams']['is_dualtor'] action = "dontcare" + # skip the pytest when the device does not have neighbors + # 'rx_port' being None indicates there are no ports available to receive frames for pfc storm + if not has_neighbor_device(setup_pfc_test): + pytest.skip("Test skipped: No neighbors detected as 'rx_port' is None for selected test ports," + " which is necessary for PFCwd test setup.") + for idx, port in enumerate(self.ports): logger.info("") logger.info("--- Testing port toggling with PFCWD enabled on {} ---".format(port)) @@ -1215,6 +1258,12 @@ def test_pfcwd_no_traffic( self.is_dualtor = setup_dut_info['basicParams']['is_dualtor'] self.fake_storm = False # Not needed for this test. + # skip the pytest when the device does not have neighbors + # 'rx_port' being None indicates there are no ports available to receive frames for pfc storm + if not has_neighbor_device(setup_pfc_test): + pytest.skip("Test skipped: No neighbors detected as 'rx_port' is None for selected test ports," + " which is necessary for PFCwd test setup.") + for idx, port in enumerate(self.ports): logger.info("") logger.info("--- Testing non-Trafific Pfcwd actions on {} ---".format(port)) diff --git a/tests/pfcwd/test_pfcwd_timer_accuracy.py b/tests/pfcwd/test_pfcwd_timer_accuracy.py index a8418f3b68..72fdabb322 100644 --- a/tests/pfcwd/test_pfcwd_timer_accuracy.py +++ b/tests/pfcwd/test_pfcwd_timer_accuracy.py @@ -6,16 +6,18 @@ from tests.common.fixtures.conn_graph_facts import enum_fanout_graph_facts # noqa F401 from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.pfc_storm import PFCStorm -from .files.pfcwd_helper import start_wd_on_ports, start_background_traffic # noqa F401 +from tests.common.helpers.pfcwd_helper import start_wd_on_ports, start_background_traffic # noqa F401 from tests.common.plugins.loganalyzer import DisableLogrotateCronContext -from .files.pfcwd_helper import send_background_traffic +from tests.common.helpers.pfcwd_helper import send_background_traffic pytestmark = [ pytest.mark.topology('any') ] +ITERATION_NUM = 20 + logger = logging.getLogger(__name__) @@ -184,7 +186,7 @@ def run_test(self, setup_info): with send_background_traffic(self.dut, self.ptf, queues, selected_test_ports, test_ports_info): self.storm_handle.start_storm() logger.info("Wait for queue to recover from PFC storm") - time.sleep(8) + time.sleep(32) self.storm_handle.stop_storm() time.sleep(16) @@ -206,6 +208,9 @@ def run_test(self, setup_info): self.all_restore_time.append(real_restore_time) dut_detect_restore_time = storm_restore_ms - storm_detect_ms + logger.info( + "Iteration all_dut_detect_time list {} and length {}".format( + ",".join(str(i) for i in self.all_detect_time), len(self.all_detect_time))) self.all_dut_detect_restore_time.append(dut_detect_restore_time) logger.info( "Iteration all_dut_detect_restore_time list {} and length {}".format( @@ -218,32 +223,58 @@ def verify_pfcwd_timers(self): self.all_detect_time.sort() self.all_restore_time.sort() logger.info("Verify that real detection time is not greater than configured") + logger.info("all detect time {}".format(self.all_detect_time)) + logger.info("all restore time {}".format(self.all_restore_time)) + + check_point = ITERATION_NUM // 2 - 1 config_detect_time = self.timers['pfc_wd_detect_time'] + self.timers['pfc_wd_poll_time'] + # Loose the check if two conditions are met + # 1. Leaf-fanout is Non-Onyx or non-Mellanox SONiC devices + # 2. Device is Mellanox plaform, Loose the check + # 3. Device is broadcom plaform, add half of polling time as compensation for the detect config time + # It's because the pfc_gen.py running on leaf-fanout can't guarantee the PFCWD is triggered consistently + logger.debug("dut asic_type {}".format(self.dut.facts['asic_type'])) + for fanouthost in list(self.fanout.values()): + if fanouthost.get_fanout_os() != "onyx" or \ + fanouthost.get_fanout_os() == "sonic" and fanouthost.facts['asic_type'] != "mellanox": + if self.dut.facts['asic_type'] == "mellanox": + logger.info("Loose the check for non-Onyx or non-Mellanox leaf-fanout testbed") + check_point = ITERATION_NUM // 3 - 1 + break + elif self.dut.facts['asic_type'] == "broadcom": + logger.info("Configuring detect time for broadcom DUT") + config_detect_time = ( + self.timers['pfc_wd_detect_time'] + + self.timers['pfc_wd_poll_time'] + + (self.timers['pfc_wd_poll_time'] // 2) + ) + break + err_msg = ("Real detection time is greater than configured: Real detect time: {} " - "Expected: {} (wd_detect_time + wd_poll_time)".format(self.all_detect_time[9], + "Expected: {} (wd_detect_time + wd_poll_time)".format(self.all_detect_time[check_point], config_detect_time)) - pytest_assert(self.all_detect_time[9] < config_detect_time, err_msg) + pytest_assert(self.all_detect_time[check_point] < config_detect_time, err_msg) if self.timers['pfc_wd_poll_time'] < self.timers['pfc_wd_detect_time']: logger.info("Verify that real detection time is not less than configured") err_msg = ("Real detection time is less than configured: Real detect time: {} " - "Expected: {} (wd_detect_time)".format(self.all_detect_time[9], + "Expected: {} (wd_detect_time)".format(self.all_detect_time[check_point], self.timers['pfc_wd_detect_time'])) - pytest_assert(self.all_detect_time[9] > self.timers['pfc_wd_detect_time'], err_msg) + pytest_assert(self.all_detect_time[check_point] > self.timers['pfc_wd_detect_time'], err_msg) if self.timers['pfc_wd_poll_time'] < self.timers['pfc_wd_restore_time']: logger.info("Verify that real restoration time is not less than configured") err_msg = ("Real restoration time is less than configured: Real restore time: {} " - "Expected: {} (wd_restore_time)".format(self.all_restore_time[9], + "Expected: {} (wd_restore_time)".format(self.all_restore_time[check_point], self.timers['pfc_wd_restore_time'])) - pytest_assert(self.all_restore_time[9] > self.timers['pfc_wd_restore_time'], err_msg) + pytest_assert(self.all_restore_time[check_point] > self.timers['pfc_wd_restore_time'], err_msg) logger.info("Verify that real restoration time is less than configured") config_restore_time = self.timers['pfc_wd_restore_time'] + self.timers['pfc_wd_poll_time'] err_msg = ("Real restoration time is greater than configured: Real restore time: {} " - "Expected: {} (wd_restore_time + wd_poll_time)".format(self.all_restore_time[9], + "Expected: {} (wd_restore_time + wd_poll_time)".format(self.all_restore_time[check_point], config_restore_time)) - pytest_assert(self.all_restore_time[9] < config_restore_time, err_msg) + pytest_assert(self.all_restore_time[check_point] < config_restore_time, err_msg) def verify_pfcwd_timers_t2(self): """ @@ -286,7 +317,7 @@ def retrieve_timestamp(self, pattern): return int(timestamp_ms) def test_pfcwd_timer_accuracy(self, duthosts, ptfhost, enum_rand_one_per_hwsku_frontend_hostname, - pfcwd_timer_setup_restore): + pfcwd_timer_setup_restore, fanouthosts): """ Tests PFCwd timer accuracy @@ -300,6 +331,7 @@ def test_pfcwd_timer_accuracy(self, duthosts, ptfhost, enum_rand_one_per_hwsku_f self.timers = setup_info['timers'] self.dut = duthost self.ptf = ptfhost + self.fanout = fanouthosts self.all_detect_time = list() self.all_restore_time = list() self.all_dut_detect_restore_time = list() @@ -310,7 +342,7 @@ def test_pfcwd_timer_accuracy(self, duthosts, ptfhost, enum_rand_one_per_hwsku_f self.run_test(setup_info) self.verify_pfcwd_timers_t2() else: - for i in range(1, 20): + for i in range(1, ITERATION_NUM): logger.info("--- Pfcwd Timer Test iteration #{}".format(i)) cmd = "show pfc counters" diff --git a/tests/pfcwd/test_pfcwd_warm_reboot.py b/tests/pfcwd/test_pfcwd_warm_reboot.py index 144e1e03e9..67b67c264a 100644 --- a/tests/pfcwd/test_pfcwd_warm_reboot.py +++ b/tests/pfcwd/test_pfcwd_warm_reboot.py @@ -16,8 +16,10 @@ from tests.common.utilities import InterruptableThread from tests.common.utilities import join_all from tests.ptf_runner import ptf_runner -from .files.pfcwd_helper import EXPECT_PFC_WD_DETECT_RE, EXPECT_PFC_WD_RESTORE_RE -from .files.pfcwd_helper import send_background_traffic +from tests.common.helpers.pfcwd_helper import EXPECT_PFC_WD_DETECT_RE, EXPECT_PFC_WD_RESTORE_RE +from tests.common.helpers.pfcwd_helper import send_background_traffic +from tests.common.helpers.pfcwd_helper import has_neighbor_device +from tests.common.utilities import wait_until TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates") TESTCASE_INFO = {'no_storm': {'test_sequence': ["detect", "restore", "warm-reboot", "detect", "restore"], @@ -439,14 +441,21 @@ def run_test(self, port, queue, detect=True, storm_start=True, first_detect_afte self.traffic_inst.verify_wd_func(self.dut, detect=detect) @pytest.fixture(autouse=True) - def pfcwd_wb_test_cleanup(self): + def pfcwd_wb_test_cleanup(self, setup_pfc_test): """ Cleanup method + + Args: + setup_pfc_test(fixture): module scoped autouse fixture """ yield # stop all threads that might stuck in wait DUT_ACTIVE.set() + # if there are no neighbor devices detected, exit the cleanup function early + if not has_neighbor_device(setup_pfc_test): + return + for thread in self.storm_threads: thread_exception = thread.join(timeout=0.1, suppress_exception=True) @@ -512,8 +521,13 @@ def pfcwd_wb_helper(self, fake_storm, testcase_actions, setup_pfc_test, enum_fan self.storm_threads = [] for t_idx, test_action in enumerate(testcase_actions): + logger.info("Index {} test_action {}".format(t_idx, test_action)) if 'warm-reboot' in test_action: reboot(self.dut, localhost, reboot_type="warm", wait_warmboot_finalizer=True) + + assert wait_until(300, 20, 20, self.dut.critical_services_fully_started), \ + "All critical services should fully started!" + continue # Need to wait some time after warm-reboot for the counters to be created @@ -604,6 +618,12 @@ def test_pfcwd_wb(self, fake_storm, testcase_action, setup_pfc_test, enum_fanout localhost(AnsibleHost) : localhost instance fanouthosts(AnsibleHost): fanout instance """ + # skip the pytest when the device does not have neighbors + # 'rx_port_id' being None indicates there are no ports available to receive frames for the fake storm + if not has_neighbor_device(setup_pfc_test): + pytest.skip("Test skipped: No neighbors detected as 'rx_port_id' is None for selected test ports," + " which is necessary for PFCwd test setup.") + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] logger.info("--- {} ---".format(TESTCASE_INFO[testcase_action]['desc'])) self.pfcwd_wb_helper(fake_storm, TESTCASE_INFO[testcase_action]['test_sequence'], setup_pfc_test, diff --git a/tests/platform_tests/api/test_chassis_fans.py b/tests/platform_tests/api/test_chassis_fans.py index 5c1a301376..b94f6b97a1 100644 --- a/tests/platform_tests/api/test_chassis_fans.py +++ b/tests/platform_tests/api/test_chassis_fans.py @@ -5,7 +5,7 @@ from tests.common.helpers.platform_api import chassis, fan from .platform_api_test_base import PlatformApiTestBase -from tests.platform_tests.thermal_control_test_helper import start_thermal_control_daemon, stop_thermal_control_daemon +from tests.common.helpers.thermal_control_test_helper import start_thermal_control_daemon, stop_thermal_control_daemon ################################################### # TODO: Remove this after we transition to Python 3 @@ -240,9 +240,11 @@ def test_set_fans_speed(self, duthosts, enum_rand_one_per_hwsku_hostname, localh target_speed = random.randint(speed_minimum, speed_maximum) speed = fan.get_speed(platform_api_conn, i) + speed_delta = abs(speed-target_speed) speed_set = fan.set_speed(platform_api_conn, i, target_speed) # noqa F841 - time.sleep(self.get_fan_facts(duthost, i, 5, "speed", "delay")) + time_wait = 10 if speed_delta > 40 else 5 + time.sleep(self.get_fan_facts(duthost, i, time_wait, "speed", "delay")) act_speed = fan.get_speed(platform_api_conn, i) under_speed = fan.is_under_speed(platform_api_conn, i) diff --git a/tests/platform_tests/api/test_fan_drawer_fans.py b/tests/platform_tests/api/test_fan_drawer_fans.py index f2b9112e2b..03f6f0eac1 100644 --- a/tests/platform_tests/api/test_fan_drawer_fans.py +++ b/tests/platform_tests/api/test_fan_drawer_fans.py @@ -5,7 +5,7 @@ import pytest from tests.common.helpers.platform_api import chassis, fan_drawer, fan_drawer_fan -from tests.platform_tests.thermal_control_test_helper import start_thermal_control_daemon, stop_thermal_control_daemon +from tests.common.helpers.thermal_control_test_helper import start_thermal_control_daemon, stop_thermal_control_daemon from .platform_api_test_base import PlatformApiTestBase ################################################### @@ -306,9 +306,11 @@ def test_set_fans_speed(self, duthosts, enum_rand_one_per_hwsku_hostname, localh target_speed = random.randint(speed_minimum, speed_maximum) speed = fan_drawer_fan.get_speed(platform_api_conn, j, i) + speed_delta = abs(speed-target_speed) speed_set = fan_drawer_fan.set_speed(platform_api_conn, j, i, target_speed) # noqa F841 - time.sleep(self.get_fan_facts(duthost, j, i, 5, "speed", "delay")) + time_wait = 10 if speed_delta > 40 else 5 + time.sleep(self.get_fan_facts(duthost, j, i, time_wait, "speed", "delay")) act_speed = fan_drawer_fan.get_speed(platform_api_conn, j, i) under_speed = fan_drawer_fan.is_under_speed(platform_api_conn, j, i) diff --git a/tests/platform_tests/broadcom/files/ser_injector.py b/tests/platform_tests/broadcom/files/ser_injector.py index 6e67ac3901..4d912a9b6a 100644 --- a/tests/platform_tests/broadcom/files/ser_injector.py +++ b/tests/platform_tests/broadcom/files/ser_injector.py @@ -123,6 +123,11 @@ 'EGR_ZONE_1_DOT1P_MAPPING_TABLE_4.eABLE_4.epipe0', 'EGR_ZONE_3_DOT1P_MAPPING_TABLE_1.epipe0', 'EGR_FLEX_CONTAINER_UPDATE_PROFILE_1.epipe0', 'EGR_ZONE_3_DOT1P_MAPPING_TABLE_3.epipe0', 'EGR_VLAN_CONTROL_2.epipe0', 'EGR_ZONE_1_DOT1P_MAPPING_TABLE_4.epipe0', + 'MMU_MTRO_CONFIG_L1_MEM.mmu_sed0', + 'MMU_MTRO_CONFIG_L1_MEM_A.mmu_sed0', 'MMU_MTRO_CONFIG_L1_MEM_B.mmu_sed0', + 'MMU_MTRO_CONFIG_L1_MEM_A_PIPE0.mmu_sed0', 'MMU_MTRO_CONFIG_L1_MEM_B_PIPE0.mmu_sed0', + 'MMU_MTRO_CONFIG_L1_MEM_PIPE0.mmu_sed0', 'MMU_MTRO_CONFIG_L1_MEM_PIPE1.mmu_sed0', + 'MMU_MTRO_CONFIG_L1_MEM_PIPE2.mmu_sed0', 'MMU_MTRO_CONFIG_L1_MEM_PIPE3.mmu_sed0', ], 'timeout_basic': [ 'EGR_ZONE_0_EDITOR_CONTROL_TCAM.epipe0', 'DLB_ECMP_FLOWSET_MEMBER.ipipe0', @@ -269,7 +274,11 @@ 'IS_TDM_CALENDAR0.ipipe0', 'IS_TDM_CALENDAR1.ipipe0', 'IS_TDM_CALENDAR0_PIPE0.ipipe0', 'IS_TDM_CALENDAR0_PIPE1.ipipe0', 'IS_TDM_CALENDAR0_PIPE2.ipipe0', 'IS_TDM_CALENDAR0_PIPE3.ipipe0', 'IS_TDM_CALENDAR1_PIPE0.ipipe0', 'IS_TDM_CALENDAR1_PIPE1.ipipe0', 'IS_TDM_CALENDAR1_PIPE2.ipipe0', - 'IS_TDM_CALENDAR1_PIPE3.ipipe0', + 'IS_TDM_CALENDAR1_PIPE3.ipipe0', 'MMU_MTRO_CONFIG_L1_MEM.mmu_sed0', + 'MMU_MTRO_CONFIG_L1_MEM_A.mmu_sed0', 'MMU_MTRO_CONFIG_L1_MEM_B.mmu_sed0', + 'MMU_MTRO_CONFIG_L1_MEM_A_PIPE0.mmu_sed0', 'MMU_MTRO_CONFIG_L1_MEM_B_PIPE0.mmu_sed0', + 'MMU_MTRO_CONFIG_L1_MEM_PIPE0.mmu_sed0', 'MMU_MTRO_CONFIG_L1_MEM_PIPE1.mmu_sed0', + 'MMU_MTRO_CONFIG_L1_MEM_PIPE2.mmu_sed0', 'MMU_MTRO_CONFIG_L1_MEM_PIPE3.mmu_sed0', ], 'timeout_basic': [ ], @@ -380,7 +389,8 @@ def get_asic_name(): if ("Broadcom Limited Device b960" in output or "Broadcom Limited Broadcom BCM56960" in output or "Broadcom Inc. and subsidiaries Device b960" in output or - "Broadcom Inc. and subsidiaries Broadcom BCM56960" in output): + "Broadcom Inc. and subsidiaries Broadcom BCM56960" in output or + "Broadcom Inc. and subsidiaries BCM56960" in output): asic = "th" elif ("Broadcom Limited Device b971" in output or "Broadcom Inc. and subsidiaries Device b971" in output): diff --git a/tests/platform_tests/broadcom/test_ser.py b/tests/platform_tests/broadcom/test_ser.py index 3483ed5199..fc5a4911c6 100644 --- a/tests/platform_tests/broadcom/test_ser.py +++ b/tests/platform_tests/broadcom/test_ser.py @@ -147,7 +147,31 @@ def test_ser(duthosts, rand_one_dut_hostname, enum_asic_index): else: logger.info('Test complete failed with timeout') + pattern_asic = re.compile(r'SER test on ASIC :(.*)') + match_asic = pattern_asic.search(get_log_cmd_stdout) + result_asic = match_asic.group(0) if match_asic else None + + pattern_failed = re.compile(r'SER Test failed for memories (.*)') + match_failed_memories = pattern_failed.search(get_log_cmd_stdout) + result_failed_memories = match_failed_memories.group(0) if match_failed_memories else None + + pattern_timeout = re.compile(r'SER Test timed out for memories (.*)') + match_timed_out_memories = pattern_timeout.search(get_log_cmd_stdout) + result_timed_out_memories = match_timed_out_memories.group(0) if match_timed_out_memories else None + logger.info('result_asic {}; \n' + 'result_failed_memories {}; \n' + 'result_timed_out_memories {}'.format( + result_asic, result_failed_memories, result_timed_out_memories) + ) + logger.debug("test ser script output: \n {}".format(get_log_cmd_response['stdout_lines'])) time.sleep(5) assert not_timeout, 'ser_injector scirpt timeout' - assert False, 'ser_injector scirpt failed ' + assert False, ( + 'ser_injector script failed; \n' + 'result_asic {}; \n' + 'result_failed_memories {}; \n' + 'result_timed_out_memories {}'.format( + result_asic, result_failed_memories, result_timed_out_memories + ) + ) diff --git a/tests/platform_tests/cli/test_show_platform.py b/tests/platform_tests/cli/test_show_platform.py index a6669064de..f4ecbbc580 100644 --- a/tests/platform_tests/cli/test_show_platform.py +++ b/tests/platform_tests/cli/test_show_platform.py @@ -34,6 +34,7 @@ THERMAL_CONTROL_TEST_WAIT_TIME = 65 THERMAL_CONTROL_TEST_CHECK_INTERVAL = 5 +VPD_DATA_FILE = "/var/run/hw-management/eeprom/vpd_data" @pytest.fixture(scope='module') @@ -93,6 +94,12 @@ def test_show_platform_summary(duthosts, enum_rand_one_per_hwsku_hostname, dut_v if len(unexpected_fields) != 0: expected_fields_values.add(expected_num_asic) + if duthost.facts["asic_type"] in ["mellanox"]: + # For Mellanox devices, we validate the hw-revision using the value at VPD_DATA_FILE + vpd_data = duthost.command(f"cat {VPD_DATA_FILE}")["stdout_lines"] + hw_rev_expected = util.parse_colon_speparated_lines(vpd_data)["REV"] + expected_fields_values.add(hw_rev_expected) + actual_fields_values = set(summary_dict.values()) diff_fields_values = expected_fields_values.difference(actual_fields_values) pytest_assert((len(diff_fields_values) == 0 or (len(diff_fields_values) == 1 and diff_fields_values.pop() is None)), @@ -107,10 +114,21 @@ def test_platform_serial_no(duthosts, enum_rand_one_per_hwsku_hostname, dut_vars """ duthost = duthosts[enum_rand_one_per_hwsku_hostname] cmd = "sudo decode-syseeprom -s" - get_serial_no_cmd = duthost.command(cmd) + get_serial_no_cmd = duthost.command(cmd, module_ignore_errors=True) + # For kvm testbed, when executing command `sudo decode-syseeprom -s` + # On some images, it will return the expected Error `ModuleNotFoundError: No module named 'sonic_platform'` + # and get the expected return code 2 + # On some images, it will return the expected Error `Failed to read system EEPROM info in syseeprom on 'vlab-01'` + # and get the expected return code 0 + # So let this function return in advance + if duthost.facts["asic_type"] == "vs" and (get_serial_no_cmd["rc"] == 1 or get_serial_no_cmd["rc"] == 0): + return + assert get_serial_no_cmd['rc'] == 0, "Run command '{}' failed".format(cmd) + logging.info("Verifying output of '{}' on '{}' ...".format(get_serial_no_cmd, duthost.hostname)) get_serial_no_output = get_serial_no_cmd["stdout"].replace('\x00', '') - expected_serial_no = dut_vars['serial'] + expected_serial_no = dut_vars.get('serial', "") + pytest_assert(get_serial_no_output == expected_serial_no, "Expected serial_no '{}' is not matching with {} in syseeprom on '{}'". format(expected_serial_no, get_serial_no_output, duthost.hostname)) @@ -124,8 +142,15 @@ def test_show_platform_syseeprom(duthosts, enum_rand_one_per_hwsku_hostname, dut skip_release_for_platform(duthost, ["202012", "201911", "201811"], ["arista_7050", "arista_7260", "arista_7060"]) cmd = " ".join([CMD_SHOW_PLATFORM, "syseeprom"]) + syseeprom_cmd = duthost.command(cmd, module_ignore_errors=True) + # For kvm testbed, command `show platform syseeprom` will return the expected Error + # `ModuleNotFoundError: No module named 'sonic_platform'` + # So let this function return in advance + if duthost.facts["asic_type"] == "vs" and syseeprom_cmd["rc"] == 1: + return + assert syseeprom_cmd['rc'] == 0, "Run command '{}' failed".format(cmd) + logging.info("Verifying output of '{}' on '{}' ...".format(cmd, duthost.hostname)) - syseeprom_cmd = duthost.command(cmd) syseeprom_output = syseeprom_cmd["stdout"] syseeprom_output_lines = syseeprom_cmd["stdout_lines"] @@ -237,7 +262,16 @@ def test_show_platform_psustatus(duthosts, enum_supervisor_dut_hostname): cmd = " ".join([CMD_SHOW_PLATFORM, "psustatus"]) logging.info("Verifying output of '{}' on '{}' ...".format(cmd, duthost.hostname)) - psu_status_output_lines = duthost.command(cmd)["stdout_lines"] + psu_status_output = duthost.command(cmd, module_ignore_errors=True) + + # For kvm testbed, there is no key "PSU_INFO" in "STATE_DB" + # And it will raise Error when executing such command + # We will return in advance if this test case is running on kvm testbed + if duthost.facts["asic_type"] == "vs" and psu_status_output["rc"] == 1: + return + assert psu_status_output['rc'] == 0, "Run command '{}' failed".format(cmd) + + psu_status_output_lines = psu_status_output["stdout_lines"] psu_line_pattern = get_dut_psu_line_pattern(duthost) @@ -271,7 +305,17 @@ def test_show_platform_psustatus_json(duthosts, enum_supervisor_dut_hostname): cmd = " ".join([CMD_SHOW_PLATFORM, "psustatus", "--json"]) logging.info("Verifying output of '{}' ...".format(cmd)) - psu_status_output = duthost.command(cmd)["stdout"] + psu_status_output = duthost.command(cmd, module_ignore_errors=True) + + # For kvm testbed, there is no key "PSU_INFO" in "STATE_DB" + # And it will raise Error when executing such command + # We will return in advance if this test case is running on kvm testbed + if duthost.facts["asic_type"] == "vs" and psu_status_output["rc"] == 1: + return + assert psu_status_output['rc'] == 0, "Run command '{}' failed".format(cmd) + + psu_status_output = psu_status_output["stdout"] + psu_info_list = json.loads(psu_status_output) # TODO: Compare against expected platform-specific output @@ -339,7 +383,8 @@ def check_fan_status(duthost, cmd): config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] if not fans and config_facts['DEVICE_METADATA']['localhost'].get('switch_type', '') == 'dpu': return True - + if duthost.facts["asic_type"] == "vs": + return True # Check that all fans are showing valid status and also at-least one PSU is OK. num_fan_ok = 0 for a_fan in list(fans.values()): @@ -447,8 +492,16 @@ def test_show_platform_firmware_status(duthosts, enum_rand_one_per_hwsku_hostnam cmd = " ".join([CMD_SHOW_PLATFORM, "firmware", "status"]) + firmware_output = duthost.command(cmd, module_ignore_errors=True) + # For kvm testbed, command `show platform firmware status` will return the expected Error + # `ModuleNotFoundError: No module named 'sonic_platform'` + # So let this function return in advance + if duthost.facts["asic_type"] == "vs" and firmware_output["rc"] == 1: + return + assert firmware_output['rc'] == 0, "Run command '{}' failed".format(cmd) + logging.info("Verifying output of '{}' on '{}' ...".format(cmd, duthost.hostname)) - firmware_output_lines = duthost.command(cmd)["stdout_lines"] + firmware_output_lines = firmware_output["stdout_lines"] verify_show_platform_firmware_status_output(firmware_output_lines, duthost.hostname) # TODO: Test values against platform-specific expected data @@ -464,6 +517,12 @@ def test_show_platform_pcieinfo(duthosts, enum_rand_one_per_hwsku_hostname): logging.info("Verifying output of '{}' on '{}' ...".format(cmd, duthost.hostname)) pcieinfo_output_lines = duthost.command(cmd)["stdout_lines"] + # For kvm testbed, there is no file `/usr/share/sonic/device/x86_64-kvm_x86_64-r0/pcie.yaml` + # So running such command will get the error `No such file or directory` + # Return in advance if this test case is running on kvm testbed + if duthost.facts["asic_type"] == "vs": + return + passed_check_regexp = r'\[Passed\]|PASSED' for line in pcieinfo_output_lines[1:]: error_message = "Failed to validate output of command '{}' line: '{}'".format(cmd, line) diff --git a/tests/platform_tests/conftest.py b/tests/platform_tests/conftest.py index 74b15476c4..10ee343ba2 100644 --- a/tests/platform_tests/conftest.py +++ b/tests/platform_tests/conftest.py @@ -1,50 +1,10 @@ -import glob import json import pytest import os -import re import logging -from collections import OrderedDict -from datetime import datetime - -from tests.platform_tests.reboot_timing_constants import SERVICE_PATTERNS, OTHER_PATTERNS,\ - SAIREDIS_PATTERNS, OFFSET_ITEMS, TIME_SPAN_ITEMS, REQUIRED_PATTERNS from tests.common.mellanox_data import is_mellanox_device -from tests.common.broadcom_data import is_broadcom_device -from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer -from tests.common.plugins.sanity_check.recover import neighbor_vm_restore from .args.counterpoll_cpu_usage_args import add_counterpoll_cpu_usage_args -from .mellanox.mellanox_thermal_control_test_helper import suspend_hw_tc_service, resume_hw_tc_service - - -TEMPLATES_DIR = os.path.join(os.path.dirname( - os.path.realpath(__file__)), "templates") - -FMT = "%b %d %H:%M:%S.%f" -FMT_YEAR = "%Y %b %d %H:%M:%S.%f" -FMT_SHORT = "%b %d %H:%M:%S" -FMT_ALT = "%Y-%m-%dT%H:%M:%S.%f%z" - -LOGS_ON_TMPFS_PLATFORMS = [ - "x86_64-arista_7050_qx32", - "x86_64-arista_7050_qx32s", - "x86_64-arista_7060_cx32s", - "x86_64-arista_7260cx3_64", - "x86_64-arista_7050cx3_32s", - "x86_64-mlnx_msn2700-r0", - "x86_64-dell_s6100_c2538-r0", - "armhf-nokia_ixs7215_52x-r0" -] - - -def _parse_timestamp(timestamp): - for format in [FMT, FMT_YEAR, FMT_SHORT, FMT_ALT]: - try: - time = datetime.strptime(timestamp, format) - return time - except ValueError: - continue - raise ValueError("Unable to parse {} with any known format".format(timestamp)) +from tests.common.helpers.mellanox_thermal_control_test_helper import suspend_hw_tc_service, resume_hw_tc_service @pytest.fixture(autouse=True, scope="module") @@ -58,7 +18,7 @@ def skip_on_simx(duthosts, rand_one_dut_hostname): @pytest.fixture(scope="module") -def xcvr_skip_list(duthosts, dpu_npu_port_list): +def xcvr_skip_list(duthosts, dpu_npu_port_list, tbinfo): intf_skip_list = {} for dut in duthosts: platform = dut.facts['platform'] @@ -89,6 +49,12 @@ def xcvr_skip_list(duthosts, dpu_npu_port_list): logging.debug('Skipping sfp interfaces: {}'.format(sfp_list)) intf_skip_list[dut.hostname].extend(sfp_list) + # For Mx topo, skip the SFP interfaces because they are admin down + if tbinfo['topo']['name'] == "mx" and hwsku in ["Arista-720DT-G48S4", "Nokia-7215"]: + sfp_list = ['Ethernet48', 'Ethernet49', 'Ethernet50', 'Ethernet51'] + logging.debug('Skipping sfp interfaces: {}'.format(sfp_list)) + intf_skip_list[dut.hostname].extend(sfp_list) + return intf_skip_list @@ -114,554 +80,6 @@ def bring_up_dut_interfaces(request, duthosts, enum_rand_one_per_hwsku_frontend_ duthost.command("sudo config interface {} startup {}".format(namespace_arg, port)) -def get_state_times(timestamp, state, state_times, first_after_offset=None): - time = timestamp.strftime(FMT) - state_name = state.split("|")[0].strip() - state_status = state.split("|")[1].strip() - state_dict = state_times.get(state_name, {"timestamp": {}}) - timestamps = state_dict.get("timestamp") - if state_status in timestamps: - state_dict[state_status + - " count"] = state_dict.get(state_status+" count") + 1 - # capture last occcurence - useful in calculating events end time - state_dict["last_occurence"] = time - elif first_after_offset: - state_dict[state_status+" count"] = 1 - # capture the first occurence as the one after offset timestamp and ignore the ones before - # this is useful to find time after a specific instance, for eg. - kexec time or FDB disable time. - if _parse_timestamp(first_after_offset) < _parse_timestamp(time): - timestamps[state_status] = time - else: - # only capture timestamp of first occurence of the entity. Otherwise, just increment the count above. - # this is useful in capturing start point. Eg., first neighbor entry, LAG ready, etc. - state_dict[state_status+" count"] = 1 - timestamps[state_status] = time - return {state_name: state_dict} - - -def get_report_summary(duthost, analyze_result, reboot_type, reboot_oper, base_os_version): - time_spans = analyze_result.get("time_span", {}) - time_spans_summary = OrderedDict() - kexec_offsets = analyze_result.get("offset_from_kexec", {}) - reboot_start_time = analyze_result.get( - "reboot_time", {}).get("timestamp", {}).get("Start") - kexec_offsets_summary = OrderedDict() - for entity in OFFSET_ITEMS: - time_taken = "" - if entity in kexec_offsets: - time_taken = kexec_offsets.get(entity).get("time_taken", "") - elif entity in time_spans: - timestamp = time_spans.get(entity).get("timestamp", {}) - marker_start_time = timestamp.get( - "Start") if "Start" in timestamp else timestamp.get("Started") - if reboot_start_time and reboot_start_time != "N/A" and marker_start_time: - time_taken = (_parse_timestamp(marker_start_time) - - _parse_timestamp(reboot_start_time)).total_seconds() - kexec_offsets_summary.update({entity.lower(): str(time_taken)}) - - for entity in TIME_SPAN_ITEMS: - time_taken = "" - if entity in time_spans: - time_taken = time_spans.get(entity, {}).get("time_span", "") - elif entity in kexec_offsets: - marker_first_time = kexec_offsets.get( - entity).get("timestamp", {}).get("Start") - marker_last_time = kexec_offsets.get(entity).get("last_occurence") - if marker_first_time and marker_last_time: - time_taken = (_parse_timestamp(marker_last_time) - - _parse_timestamp(marker_first_time)).total_seconds() - time_spans_summary.update({entity.lower(): str(time_taken)}) - - lacp_sessions_dict = analyze_result.get("controlplane") - lacp_sessions_waittime = lacp_sessions_dict.pop("lacp_sessions")\ - if lacp_sessions_dict and "lacp_sessions" in lacp_sessions_dict else None - controlplane_summary = {"downtime": "", - "arp_ping": "", "lacp_session_max_wait": ""} - if duthost.facts['platform'] != 'x86_64-kvm_x86_64-r0': - if lacp_sessions_waittime and len(lacp_sessions_waittime) > 0: - # Filter out None values and then fine the maximum - filtered_lacp_sessions_waittime = [value for value in lacp_sessions_waittime.values() if value is not None] - if filtered_lacp_sessions_waittime: - max_lacp_session_wait = max(filtered_lacp_sessions_waittime) - else: - max_lacp_session_wait = None - analyze_result.get( - "controlplane", controlplane_summary).update( - {"lacp_session_max_wait": max_lacp_session_wait}) - - result_summary = { - "reboot_type": "{}-{}".format(reboot_type, reboot_oper) if reboot_oper else reboot_type, - "hwsku": duthost.facts["hwsku"], - "base_ver": base_os_version[0] if base_os_version and len(base_os_version) else "", - "target_ver": get_current_sonic_version(duthost), - "dataplane": analyze_result.get("dataplane", {"downtime": "", "lost_packets": ""}), - "controlplane": analyze_result.get("controlplane", controlplane_summary), - "time_span": time_spans_summary, - "offset_from_kexec": kexec_offsets_summary - } - return result_summary - - -def get_kexec_time(duthost, messages, result): - reboot_pattern = re.compile( - r'.* NOTICE (?:admin|root): Rebooting with /sbin/kexec -e to.*...') - reboot_time = "N/A" - logging.info("FINDING REBOOT PATTERN") - for message in messages: - # Get timestamp of reboot - Rebooting string - if re.search(reboot_pattern, message): - logging.info( - "FOUND REBOOT PATTERN for {}".format(duthost.hostname)) - delim = "{}|{}".format(duthost.hostname, "sonic") - reboot_time = _parse_timestamp(re.split(delim, message)[ - 0].strip()).strftime(FMT) - continue - result["reboot_time"] = { - "timestamp": {"Start": reboot_time}, - } - - -def analyze_log_file(duthost, messages, result, offset_from_kexec): - service_restart_times = dict() - derived_patterns = OTHER_PATTERNS.get("COMMON") - service_patterns = dict() - # get platform specific regexes - if is_broadcom_device(duthost): - derived_patterns.update(OTHER_PATTERNS.get("BRCM")) - elif is_mellanox_device(duthost): - derived_patterns.update(OTHER_PATTERNS.get("MLNX")) - # get image specific regexes - if "20191130" in get_current_sonic_version(duthost): - derived_patterns.update(OTHER_PATTERNS.get("201911")) - service_patterns.update(SERVICE_PATTERNS.get("201911")) - else: - derived_patterns.update(OTHER_PATTERNS.get("LATEST")) - service_patterns.update(SERVICE_PATTERNS.get("LATEST")) - - if not messages: - logging.error("Expected messages not found in syslog") - return None - - def service_time_check(message, status): - delim = "{}|{}".format(duthost.hostname, "sonic") - time = _parse_timestamp(re.split(delim, message)[0].strip()) - time = time.strftime(FMT) - service_name = message.split(status + " ")[1].split()[0] - service_name = service_name.upper() - if service_name == "ROUTER": - service_name = "RADV" - service_dict = service_restart_times.get( - service_name, {"timestamp": {}}) - timestamps = service_dict.get("timestamp") - if status in timestamps: - service_dict[status + - " count"] = service_dict.get(status+" count") + 1 - else: - service_dict[status+" count"] = 1 - timestamps[status] = time - service_restart_times.update({service_name: service_dict}) - - for message in messages: - # Get stopping to started timestamps for services (swss, bgp, etc) - for status, pattern in list(service_patterns.items()): - if re.search(pattern, message): - service_time_check(message, status) - break - # Get timestamps of all other entities - for state, pattern in list(derived_patterns.items()): - if re.search(pattern, message): - delim = "{}|{}".format(duthost.hostname, "sonic") - timestamp = _parse_timestamp( - re.split(delim, message)[0].strip()) - state_name = state.split("|")[0].strip() - if state_name + "|End" not in list(derived_patterns.keys()): - if "FDB_EVENT_OTHER_MAC_EXPIRY" in state_name or "FDB_EVENT_SCAPY_MAC_EXPIRY" in state_name: - fdb_aging_disable_start = service_restart_times.get("FDB_AGING_DISABLE", {})\ - .get("timestamp", {}).get("Start") - if not fdb_aging_disable_start: - break - first_after_offset = fdb_aging_disable_start - else: - first_after_offset = result.get("reboot_time", {}).get( - "timestamp", {}).get("Start") - state_times = get_state_times(timestamp, state, offset_from_kexec, - first_after_offset=first_after_offset) - offset_from_kexec.update(state_times) - else: - state_times = get_state_times( - timestamp, state, service_restart_times) - service_restart_times.update(state_times) - if "PORT_READY" not in state_name: - # If PORT_READY, don't break out of the for-loop here, because we want to - # try to match the other regex as well - break - # Calculate time that services took to stop/start - for _, timings in list(service_restart_times.items()): - timestamps = timings["timestamp"] - timings["stop_time"] = (_parse_timestamp(timestamps["Stopped"]) - - _parse_timestamp(timestamps["Stopping"])).total_seconds() \ - if "Stopped" in timestamps and "Stopping" in timestamps else None - - timings["start_time"] = (_parse_timestamp(timestamps["Started"]) - - _parse_timestamp(timestamps["Starting"])).total_seconds() \ - if "Started" in timestamps and "Starting" in timestamps else None - - if "Started" in timestamps and "Stopped" in timestamps: - timings["time_span"] = (_parse_timestamp(timestamps["Started"]) - - _parse_timestamp(timestamps["Stopped"])).total_seconds() - elif "Start" in timestamps and "End" in timestamps: - if "last_occurence" in timings: - timings["time_span"] = (_parse_timestamp(timings["last_occurence"]) - - _parse_timestamp(timestamps["Start"])).total_seconds() - else: - timings["time_span"] = (_parse_timestamp(timestamps["End"]) - - _parse_timestamp(timestamps["Start"])).total_seconds() - - result["time_span"].update(service_restart_times) - result["offset_from_kexec"] = offset_from_kexec - return result - - -def analyze_sairedis_rec(messages, result, offset_from_kexec): - sai_redis_state_times = dict() - for message in messages: - for state, pattern in list(SAIREDIS_PATTERNS.items()): - if re.search(pattern, message): - timestamp = datetime.strptime(message.split( - "|")[0].strip(), "%Y-%m-%d.%H:%M:%S.%f") - state_name = state.split("|")[0].strip() - reboot_time = result.get("reboot_time", {}).get( - "timestamp", {}).get("Start") - if state_name + "|End" not in list(SAIREDIS_PATTERNS.keys()): - if "FDB_EVENT_OTHER_MAC_EXPIRY" in state_name or "FDB_EVENT_SCAPY_MAC_EXPIRY" in state_name: - fdb_aging_disable_start = result.get("time_span", {}).get("FDB_AGING_DISABLE", {})\ - .get("timestamp", {}).get("Start") - if not fdb_aging_disable_start: - break - # Ignore MAC learning events before FDB aging disable, as MAC learning is still allowed - log_time = timestamp.strftime(FMT) - if _parse_timestamp(log_time) < _parse_timestamp(fdb_aging_disable_start): - break - first_after_offset = fdb_aging_disable_start - else: - first_after_offset = result.get("reboot_time", {}).get( - "timestamp", {}).get("Start") - state_times = get_state_times(timestamp, state, offset_from_kexec, - first_after_offset=first_after_offset) - offset_from_kexec.update(state_times) - else: - state_times = get_state_times(timestamp, state, sai_redis_state_times, - first_after_offset=reboot_time) - sai_redis_state_times.update(state_times) - - for _, timings in list(sai_redis_state_times.items()): - timestamps = timings["timestamp"] - if "Start" in timestamps and "End" in timestamps: - timings["time_span"] = (_parse_timestamp(timestamps["End"]) - - _parse_timestamp(timestamps["Start"])).total_seconds() - - result["time_span"].update(sai_redis_state_times) - result["offset_from_kexec"] = offset_from_kexec - - -def get_data_plane_report(analyze_result, reboot_type, log_dir, reboot_oper): - report = {"controlplane": {"arp_ping": "", "downtime": ""}, - "dataplane": {"lost_packets": "", "downtime": ""}} - files = glob.glob1(log_dir, '*reboot*-report.json') - if files: - filepath = "{}/{}".format(log_dir, files[0]) - with open(filepath) as json_file: - report = json.load(json_file) - analyze_result.update(report) - - -def verify_mac_jumping(test_name, timing_data, verification_errors): - mac_jumping_other_addr = timing_data.get("offset_from_kexec", {})\ - .get("FDB_EVENT_OTHER_MAC_EXPIRY", {}).get("Start count", 0) - mac_jumping_scapy_addr = timing_data.get("offset_from_kexec", {})\ - .get("FDB_EVENT_SCAPY_MAC_EXPIRY", {}).get("Start count", 0) - mac_expiry_start = timing_data.get("offset_from_kexec", {}).get("FDB_EVENT_OTHER_MAC_EXPIRY", {})\ - .get("timestamp", {}).get("Start") - fdb_aging_disable_start = timing_data.get("time_span", {}).get("FDB_AGING_DISABLE", {})\ - .get("timestamp", {}).get("Start") - fdb_aging_disable_end = timing_data.get("time_span", {}).get("FDB_AGING_DISABLE", {})\ - .get("timestamp", {}).get("End") - - if "mac_jump" in test_name: - # MAC jumping allowed - allow Scapy default MAC to jump - logging.info("MAC jumping is allowed. Jump count for expected mac: {}, unexpected MAC: {}" - .format(mac_jumping_scapy_addr, mac_jumping_other_addr)) - if not mac_jumping_scapy_addr: - verification_errors.append( - "MAC jumping not detected when expected for address: 00-06-07-08-09-0A") - else: - # MAC jumping not allowed - do not allow the SCAPY default MAC to jump - if mac_jumping_scapy_addr: - verification_errors.append("MAC jumping is not allowed. Jump count for scapy mac: {}, other MAC: {}" - .format(mac_jumping_scapy_addr, mac_jumping_other_addr)) - if mac_jumping_other_addr: - # In both mac jump allowed and denied cases unexpected MAC addresses should NOT jump between - # the window that starts when SAI is instructed to disable MAC learning (warmboot shutdown path) - # and ends when SAI is instructed to enable MAC learning (warmboot recovery path) - logging.info("Mac expiry for unexpected addresses started at {}".format(mac_expiry_start) + - " and FDB learning enabled at {}".format(fdb_aging_disable_end)) - if _parse_timestamp(mac_expiry_start) > _parse_timestamp(fdb_aging_disable_start) and\ - _parse_timestamp(mac_expiry_start) < _parse_timestamp(fdb_aging_disable_end): - verification_errors.append( - "Mac expiry detected during the window when FDB ageing was disabled") - - -def verify_required_events(duthost, event_counters, timing_data, verification_errors): - for key in ["time_span", "offset_from_kexec"]: - for pattern in REQUIRED_PATTERNS.get(key): - if pattern == 'PORT_READY': - observed_start_count = timing_data.get( - key, {}).get(pattern, {}).get("Start-changes-only count", 0) - else: - observed_start_count = timing_data.get( - key, {}).get(pattern, {}).get("Start count", 0) - observed_end_count = timing_data.get( - key, {}).get(pattern, {}).get("End count", 0) - expected_count = event_counters.get(pattern) - # If we're checking PORT_READY, allow any number of PORT_READY messages between 0 and the number of ports. - # Some platforms appear to have a random number of these messages, other platforms have however many ports - # are up. - if observed_start_count != expected_count and ( - pattern != 'PORT_READY' or observed_start_count > expected_count): - verification_errors.append("FAIL: Event {} was found {} times, when expected exactly {} times". - format(pattern, observed_start_count, expected_count)) - if key == "time_span" and observed_start_count != observed_end_count: - verification_errors.append("FAIL: Event {} counters did not match. ".format(pattern) + - "Started {} times, and ended {} times". - format(observed_start_count, observed_end_count)) - - -def overwrite_script_to_backup_logs(duthost, reboot_type, bgpd_log): - # find the fast/warm-reboot script path - reboot_script_path = duthost.shell('which {}'.format( - "{}-reboot".format(reboot_type)))['stdout'] - # backup original script - duthost.shell("cp {} {}".format( - reboot_script_path, reboot_script_path + ".orig")) - # find the anchor string inside fast/warm-reboot script - rebooting_log_line = "debug.*Rebooting with.*to.*" - # Create a backup log command to be inserted right after the anchor string defined above - backup_log_cmds = "cat /var/log/syslog.1 /var/log/syslog > /host/syslog.99 || true;" +\ - "cat /var/log/swss/sairedis.rec.1 /var/log/swss/sairedis.rec > /host/sairedis.rec.99 || true;" +\ - "cat /var/log/swss/swss.rec.1 /var/log/swss/swss.rec > /host/swss.rec.99 || true;" +\ - "cat {}.1 {} > /host/bgpd.log.99 || true".format(bgpd_log, bgpd_log) - # Do find-and-replace on fast/warm-reboot script to insert the backup_log_cmds string - insert_backup_command = "sed -i '/{}/a {}' {}".format( - rebooting_log_line, backup_log_cmds, reboot_script_path) - duthost.shell(insert_backup_command) - - -def get_current_sonic_version(duthost): - return duthost.shell('sonic_installer list | grep Current | cut -f2 -d " "')['stdout'] - - -@pytest.fixture() -def advanceboot_loganalyzer(duthosts, enum_rand_one_per_hwsku_frontend_hostname, request): - """ - Advance reboot log analysis. - This fixture starts log analysis at the beginning of the test. At the end, - the collected expect messages are verified and timing of start/stop is calculated. - - Args: - duthosts : List of DUT hosts - enum_rand_one_per_hwsku_frontend_hostname: hostname of a randomly selected DUT - """ - duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - test_name = request.node.name - if "upgrade_path" in test_name: - reboot_type_source = request.config.getoption("--upgrade_type") - else: - reboot_type_source = test_name - if "warm" in reboot_type_source: - reboot_type = "warm" - elif "fast" in reboot_type_source: - reboot_type = "fast" - else: - reboot_type = "unknown" - # Currently, advanced reboot test would skip for kvm platform if the test has no device_type marker for vs. - # Doing the same skip logic in this fixture to avoid running loganalyzer without the test executed - if duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0': - device_marks = [arg for mark in request.node.iter_markers( - name='device_type') for arg in mark.args] - if 'vs' not in device_marks: - pytest.skip('Testcase not supported for kvm') - platform = duthost.facts["platform"] - logs_in_tmpfs = list() - - loganalyzer = LogAnalyzer( - ansible_host=duthost, marker_prefix="test_advanced_reboot_{}".format(test_name)) - base_os_version = list() - - def bgpd_log_handler(preboot=False): - # check current OS version post-reboot. This can be different than preboot OS version in case of upgrade - current_os_version = get_current_sonic_version(duthost) - if preboot: - if 'SONiC-OS-201811' in current_os_version: - bgpd_log = "/var/log/quagga/bgpd.log" - else: - bgpd_log = "/var/log/frr/bgpd.log" - additional_files = {'/var/log/swss/sairedis.rec': '', bgpd_log: ''} - loganalyzer.additional_files = list(additional_files.keys()) - loganalyzer.additional_start_str = list(additional_files.values()) - return bgpd_log - else: - # log_analyzer may start with quagga and end with frr, and frr.log might still have old logs. - # To avoid missing preboot log, or analyzing old logs, combine quagga and frr log into new file - duthost.shell("cat {} {} | sort -n > {}".format( - "/var/log/quagga/bgpd.log", "/var/log/frr/bgpd.log", "/var/log/bgpd.log"), module_ignore_errors=True) - loganalyzer.additional_files = [ - '/var/log/swss/sairedis.rec', '/var/log/bgpd.log'] - - def pre_reboot_analysis(): - log_filesystem = duthost.shell( - "df --output=fstype -h /var/log")['stdout'] - logs_in_tmpfs.append( - True if (log_filesystem and "tmpfs" in log_filesystem) else False) - base_os_version.append(get_current_sonic_version(duthost)) - bgpd_log = bgpd_log_handler(preboot=True) - if platform in LOGS_ON_TMPFS_PLATFORMS or (len(logs_in_tmpfs) > 0 and logs_in_tmpfs[0] is True): - # For small disk devices, /var/log in mounted in tmpfs. - # Hence, after reboot the preboot logs are lost. - # For log_analyzer to work, it needs logs from the shutdown path - # Below method inserts a step in reboot script to back up logs to /host/ - overwrite_script_to_backup_logs(duthost, reboot_type, bgpd_log) - marker = loganalyzer.init() - loganalyzer.load_common_config() - - ignore_file = os.path.join(TEMPLATES_DIR, "ignore_boot_messages") - expect_file = os.path.join(TEMPLATES_DIR, "expect_boot_messages") - ignore_reg_exp = loganalyzer.parse_regexp_file(src=ignore_file) - expect_reg_exp = loganalyzer.parse_regexp_file(src=expect_file) - - loganalyzer.ignore_regex.extend(ignore_reg_exp) - loganalyzer.expect_regex = [] - loganalyzer.expect_regex.extend(expect_reg_exp) - loganalyzer.match_regex = [] - return marker - - def post_reboot_analysis(marker, event_counters=None, reboot_oper=None, log_dir=None): - bgpd_log_handler() - if platform in LOGS_ON_TMPFS_PLATFORMS or (len(logs_in_tmpfs) > 0 and logs_in_tmpfs[0] is True): - restore_backup = "mv /host/syslog.99 /var/log/; " +\ - "mv /host/sairedis.rec.99 /var/log/swss/; " +\ - "mv /host/swss.rec.99 /var/log/swss/; " +\ - "mv /host/bgpd.log.99 /var/log/" - duthost.shell(restore_backup, module_ignore_errors=True) - # find the fast/warm-reboot script path - reboot_script_path = duthost.shell('which {}'.format( - "{}-reboot".format(reboot_type)))['stdout'] - # restore original script. If the ".orig" file does not exist (upgrade path case), ignore the error. - duthost.shell("mv {} {}".format(reboot_script_path + ".orig", reboot_script_path), - module_ignore_errors=True) - # For mac jump test, the log message we care about is uaually combined with other messages in one line, - # which makes the length of the line longer than 1000 and get dropped by Logananlyzer. So we need to increase - # the max allowed length. - # The regex library in Python 2 takes very long time (over 10 minutes) to process long lines. In our test, - # most of the combined log message for mac jump test is around 5000 characters. So we set the max allowed - # length to 6000. - result = loganalyzer.analyze(marker, fail=False, maximum_log_length=6000) - analyze_result = {"time_span": dict(), "offset_from_kexec": dict()} - offset_from_kexec = dict() - - # Parsing sairedis shall happen after parsing syslog because FDB_AGING_DISABLE is required - # when analysing sairedis.rec log, so we need to sort the keys - key_list = ["syslog", "bgpd.log", "sairedis.rec"] - for i in range(0, len(key_list)): - for message_key in list(result["expect_messages"].keys()): - if key_list[i] in message_key: - key_list[i] = message_key - break - - for key in key_list: - messages = result["expect_messages"][key] - if "syslog" in key: - get_kexec_time(duthost, messages, analyze_result) - reboot_start_time = analyze_result.get( - "reboot_time", {}).get("timestamp", {}).get("Start") - if not reboot_start_time or reboot_start_time == "N/A": - logging.error("kexec regex \"Rebooting with /sbin/kexec\" not found in syslog. " + - "Skipping log_analyzer checks..") - return - syslog_messages = messages - elif "bgpd.log" in key: - bgpd_log_messages = messages - elif "sairedis.rec" in key: - sairedis_rec_messages = messages - - # analyze_sairedis_rec() use information from syslog and must be called after analyzing syslog - analyze_log_file(duthost, syslog_messages, - analyze_result, offset_from_kexec) - analyze_log_file(duthost, bgpd_log_messages, - analyze_result, offset_from_kexec) - analyze_sairedis_rec(sairedis_rec_messages, - analyze_result, offset_from_kexec) - - for marker, time_data in list(analyze_result["offset_from_kexec"].items()): - marker_start_time = time_data.get("timestamp", {}).get("Start") - reboot_start_time = analyze_result.get( - "reboot_time", {}).get("timestamp", {}).get("Start") - if reboot_start_time and reboot_start_time != "N/A" and marker_start_time: - time_data["time_taken"] = (_parse_timestamp(marker_start_time) - - _parse_timestamp(reboot_start_time)).total_seconds() - else: - time_data["time_taken"] = "N/A" - - if reboot_oper and not isinstance(reboot_oper, str): - reboot_oper = type(reboot_oper).__name__ - get_data_plane_report(analyze_result, reboot_type, - log_dir, reboot_oper) - result_summary = get_report_summary( - duthost, analyze_result, reboot_type, reboot_oper, base_os_version) - logging.info(json.dumps(analyze_result, indent=4)) - logging.info(json.dumps(result_summary, indent=4)) - if reboot_oper: - report_file_name = request.node.name + "_" + reboot_oper + "_report.json" - summary_file_name = request.node.name + "_" + reboot_oper + "_summary.json" - else: - report_file_name = request.node.name + "_report.json" - summary_file_name = request.node.name + "_summary.json" - - report_file_dir = os.path.realpath((os.path.join(os.path.dirname(__file__), - "../logs/platform_tests/"))) - report_file_path = report_file_dir + "/" + report_file_name - summary_file_path = report_file_dir + "/" + summary_file_name - if not os.path.exists(report_file_dir): - os.makedirs(report_file_dir) - with open(report_file_path, 'w') as fp: - json.dump(analyze_result, fp, indent=4) - with open(summary_file_path, 'w') as fp: - json.dump(result_summary, fp, indent=4) - - # After generating timing data report, do some checks on the timing data - verification_errors = list() - verify_mac_jumping(test_name, analyze_result, verification_errors) - if duthost.facts['platform'] != 'x86_64-kvm_x86_64-r0': - # TBD: expand this verification to KVM - extra port events in KVM which need to be filtered - verify_required_events( - duthost, event_counters, analyze_result, verification_errors) - return verification_errors - - yield pre_reboot_analysis, post_reboot_analysis - - -@pytest.fixture() -def advanceboot_neighbor_restore(duthosts, enum_rand_one_per_hwsku_frontend_hostname, nbrhosts, tbinfo): - """ - This fixture is invoked at the test teardown for advanced-reboot SAD cases. - If a SAD case fails or crashes for some reason, the neighbor VMs can be left in - a bad state. This fixture will restore state of neighbor interfaces, portchannels - and BGP sessions that were shutdown during the test. - """ - yield - duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - neighbor_vm_restore(duthost, nbrhosts, tbinfo) - - @pytest.fixture() def capture_interface_counters(duthosts, enum_rand_one_per_hwsku_frontend_hostname): duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] diff --git a/tests/platform_tests/counterpoll/test_counterpoll_watermark.py b/tests/platform_tests/counterpoll/test_counterpoll_watermark.py index 4479e9168d..28b3bd8ffa 100644 --- a/tests/platform_tests/counterpoll/test_counterpoll_watermark.py +++ b/tests/platform_tests/counterpoll/test_counterpoll_watermark.py @@ -55,9 +55,9 @@ @pytest.fixture(scope='module') -def dut_vars(duthosts, enum_rand_one_per_hwsku_hostname, request): +def dut_vars(duthosts, enum_rand_one_per_hwsku_frontend_hostname, request): inv_files = get_inventory_files(request) - dut_vars = get_host_visible_vars(inv_files, enum_rand_one_per_hwsku_hostname) + dut_vars = get_host_visible_vars(inv_files, enum_rand_one_per_hwsku_frontend_hostname) yield dut_vars @@ -65,7 +65,7 @@ def check_counters_populated(duthost, key): return bool(redis_get_keys(duthost, 'COUNTERS_DB', key)) -def test_counterpoll_queue_watermark_pg_drop(duthosts, localhost, enum_rand_one_per_hwsku_hostname, dut_vars, +def test_counterpoll_queue_watermark_pg_drop(duthosts, localhost, enum_rand_one_per_hwsku_frontend_hostname, dut_vars, backup_and_restore_config_db): # noqa F811 """ @summary: Verify FLEXCOUNTERS_DB and COUNTERS_DB content after `counterpoll queue/watermark/queue enable` @@ -84,7 +84,7 @@ def test_counterpoll_queue_watermark_pg_drop(duthosts, localhost, enum_rand_one_ no WATERMARK or QUEUE stats in FLEX_COUNTER_DB 4. enables all three counterpolls (queue,watermark,pg-drop) and count stats per type """ - duthost = duthosts[enum_rand_one_per_hwsku_hostname] + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] skip_release(duthost, ["202205", "202111", "202106", "202012", "201911", "201811", "201803"]) counted_dict = {} @@ -124,7 +124,8 @@ def test_counterpoll_queue_watermark_pg_drop(duthosts, localhost, enum_rand_one_ # Delay to allow the counterpoll to generate the maps in COUNTERS_DB with allure.step("waiting {} seconds for counterpoll to generate maps in COUNTERS_DB"): delay = RELEVANT_MAPS[tested_counterpoll][DELAY] - wait_until(120, 5, delay, check_counters_populated, duthost, MAPS_LONG_PREFIX.format('*')) + pytest_assert(wait_until(120, 5, delay, check_counters_populated, duthost, MAPS_LONG_PREFIX.format('*')), + "COUNTERS_DB failed to populate") # verify QUEUE or PG maps are generated into COUNTERS_DB after enabling relevant counterpoll with allure.step("Verifying MAPS in COUNTERS_DB on {}...".format(duthost.hostname)): maps_dict = RELEVANT_MAPS[tested_counterpoll] diff --git a/tests/platform_tests/daemon/test_pcied.py b/tests/platform_tests/daemon/test_pcied.py index a59c100c0e..8f5d459f3d 100644 --- a/tests/platform_tests/daemon/test_pcied.py +++ b/tests/platform_tests/daemon/test_pcied.py @@ -52,6 +52,18 @@ def setup(duthosts, rand_one_dut_hostname): pytest.skip("{} is not enabled in {} {}".format(daemon_name, duthost.facts['platform'], duthost.os_version)) +@pytest.fixture(scope="module", autouse=True) +def get_pcie_devices_tbl_key(duthosts, rand_one_dut_hostname, setup): + duthost = duthosts[rand_one_dut_hostname] + skip_release(duthost, ["201811", "201911"]) + pytest_assert(wait_until(30, 10, 0, check_pcie_devices_table_ready, duthost), + "PCIE_DEVICES table is empty") + command_output = duthost.shell("sonic-db-cli STATE_DB KEYS '*' | grep PCIE_DEVICES") + + global pcie_devices_status_tbl_key + pcie_devices_status_tbl_key = command_output["stdout"] + + @pytest.fixture(scope="module", autouse=True) def teardown_module(duthosts, rand_one_dut_hostname): duthost = duthosts[rand_one_dut_hostname] @@ -81,18 +93,6 @@ def check_pcie_devices_table_ready(duthost): return False -@pytest.fixture(scope="module", autouse=True) -def get_pcie_devices_tbl_key(duthosts, rand_one_dut_hostname): - duthost = duthosts[rand_one_dut_hostname] - skip_release(duthost, ["201811", "201911"]) - pytest_assert(wait_until(30, 10, 0, check_pcie_devices_table_ready, duthost), - "PCIE_DEVICES table is empty") - command_output = duthost.shell("sonic-db-cli STATE_DB KEYS '*' | grep PCIE_DEVICES") - - global pcie_devices_status_tbl_key - pcie_devices_status_tbl_key = command_output["stdout"] - - def collect_data(duthost): keys = duthost.shell('sonic-db-cli STATE_DB KEYS "PCIE_DEVICE|*"')['stdout_lines'] diff --git a/tests/platform_tests/fwutil/fwutil_common.py b/tests/platform_tests/fwutil/fwutil_common.py index 3df7aceba6..19709fd15e 100644 --- a/tests/platform_tests/fwutil/fwutil_common.py +++ b/tests/platform_tests/fwutil/fwutil_common.py @@ -43,8 +43,7 @@ def get_hw_revision(duthost): def power_cycle(duthost=None, pdu_ctrl=None, delay_time=60): - if pdu_ctrl is None: - pytest.skip("No PSU controller for %s, skipping" % duthost.hostname) + assert pdu_ctrl, "pdu_ctrl is not ready, fail test" all_outlets = pdu_ctrl.get_outlet_status() @@ -240,9 +239,13 @@ def call_fwutil(duthost, localhost, pdu_ctrl, fw_pkg, component=None, next_image # Only one chassis chassis = list(init_versions["chassis"].keys())[0] paths = get_install_paths(duthost, fw_pkg, init_versions, chassis, component) - current = duthost.shell('sonic_installer list | grep Current | cut -f2 -d " "')['stdout'] if component not in paths: pytest.skip("No available firmware to install on {}. Skipping".format(component)) + boot_type = boot if boot else paths[component]["reboot"][0] + if boot_type == POWER_CYCLE: + assert pdu_ctrl, "pdu_ctrl is not ready, fail test" + + current = duthost.shell('sonic_installer list | grep Current | cut -f2 -d " "')['stdout'] allure.step("Upload firmware to DUT") generate_config(duthost, paths, init_versions) @@ -278,7 +281,6 @@ def call_fwutil(duthost, localhost, pdu_ctrl, fw_pkg, component=None, next_image logger.info("Running install command: {}".format(command)) task, res = duthost.command(command, module_ignore_errors=True, module_async=True) - boot_type = boot if boot else paths[component]["reboot"][0] allure.step("Perform Neccesary Reboot") timeout = max([v.get("timeout", TIMEOUT) for k, v in list(paths.items())]) diff --git a/tests/platform_tests/link_flap/link_flap_utils.py b/tests/platform_tests/link_flap/link_flap_utils.py index 6d7a2d9c12..16349b819c 100644 --- a/tests/platform_tests/link_flap/link_flap_utils.py +++ b/tests/platform_tests/link_flap/link_flap_utils.py @@ -1,56 +1,14 @@ """ Test utils used by the link flap tests. """ -import time import logging import random -from tests.common.platform.device_utils import fanout_switch_port_lookup -from tests.common.utilities import wait_until -from tests.common.helpers.assertions import pytest_assert +from tests.common.platform.device_utils import fanout_switch_port_lookup, __get_dut_if_status logger = logging.getLogger(__name__) -def __get_dut_if_status(dut, ifname=None): - """ - Get interface status on the DUT. - - Args: - dut: DUT host object - ifname: Interface of DUT - exp_state: State of DUT's port ('up' or 'down') - verbose: Logging port state. - - Returns: - Interface state - """ - if not ifname: - status = dut.show_interface(command='status')['ansible_facts']['int_status'] - else: - status = dut.show_interface(command='status', interfaces=[ifname])['ansible_facts']['int_status'] - return status - - -def __check_if_status(dut, dut_port, exp_state, verbose=False): - """ - Check interface status on the DUT. - - Args: - dut: DUT host object - dut_port: Port of DUT - exp_state: State of DUT's port ('up' or 'down') - verbose: Logging port state. - - Returns: - Bool value which confirm port state - """ - status = __get_dut_if_status(dut, dut_port)[dut_port] - if verbose: - logger.debug("Interface status : %s", status) - return status['oper_state'] == exp_state - - def __build_candidate_list(candidates, fanout, fanout_port, dut_port, status): """ Add candidates to list for link flap test. @@ -134,71 +92,6 @@ def check_portchannel_status(dut, dut_port_channel, exp_state, verbose=False): return status['oper_state'] == exp_state -def toggle_one_link(dut, dut_port, fanout, fanout_port, watch=False, check_status=True): - """ - Toggle one link on the fanout. - - Args: - dut: DUT host object - dut_port: Port of DUT - fanout: Fanout host object - fanout_port: Port of fanout - watch: Logging system state - """ - - sleep_time = 90 - logger.info("Testing link flap on %s", dut_port) - if check_status: - pytest_assert(__check_if_status(dut, dut_port, 'up', verbose=True), - "Fail: dut port {}: link operational down".format(dut_port)) - - logger.info("Shutting down fanout switch %s port %s connecting to %s", fanout.hostname, fanout_port, dut_port) - - need_recovery = True - try: - fanout.shutdown(fanout_port) - if check_status: - pytest_assert(wait_until(sleep_time, 1, 0, __check_if_status, dut, dut_port, 'down', True), - "dut port {} didn't go down as expected".format(dut_port)) - - if watch: - time.sleep(1) - watch_system_status(dut) - - logger.info("Bring up fanout switch %s port %s connecting to %s", fanout.hostname, fanout_port, dut_port) - fanout.no_shutdown(fanout_port) - need_recovery = False - - if check_status: - pytest_assert(wait_until(sleep_time, 1, 0, __check_if_status, dut, dut_port, 'up', True), - "dut port {} didn't go up as expected".format(dut_port)) - finally: - if need_recovery: - fanout.no_shutdown(fanout_port) - if check_status: - wait_until(sleep_time, 1, 0, __check_if_status, dut, dut_port, 'up', True) - - -def watch_system_status(dut): - """ - Watch DUT's system status - - Args: - dut: DUT host object - """ - # Watch memory status - memory_output = dut.shell("show system-memory")["stdout"] - logger.info("Memory Status: %s", memory_output) - - # Watch orchagent CPU utilization - orch_cpu = dut.shell("show processes cpu | grep orchagent | awk '{print $9}'")["stdout"] - logger.info("Orchagent CPU Util: %s", orch_cpu) - - # Watch Redis Memory - redis_memory = dut.shell("redis-cli info memory | grep used_memory_human")["stdout"] - logger.info("Redis Memory: %s", redis_memory) - - def check_orch_cpu_utilization(dut, orch_cpu_threshold): """ Compare orchagent CPU utilization diff --git a/tests/platform_tests/link_flap/test_cont_link_flap.py b/tests/platform_tests/link_flap/test_cont_link_flap.py index ea7a2abfa2..62743ed4cc 100644 --- a/tests/platform_tests/link_flap/test_cont_link_flap.py +++ b/tests/platform_tests/link_flap/test_cont_link_flap.py @@ -8,14 +8,19 @@ import logging import pytest +import time +import math + +from collections import defaultdict from tests.common.helpers.assertions import pytest_assert, pytest_require from tests.common import port_toggle -from tests.platform_tests.link_flap.link_flap_utils import build_test_candidates, toggle_one_link,\ +from tests.platform_tests.link_flap.link_flap_utils import build_test_candidates,\ check_orch_cpu_utilization, check_bgp_routes from tests.common.utilities import wait_until from tests.common.devices.eos import EosHost from tests.common.devices.sonic import SonicHost +from tests.common.platform.device_utils import toggle_one_link pytestmark = [ pytest.mark.disable_loganalyzer, @@ -28,6 +33,25 @@ class TestContLinkFlap(object): TestContLinkFlap class for continuous link flap """ + def get_frr_daemon_memory_usage(self, duthost, daemon): + frr_daemon_memory_per_asics = {} + + for asic in duthost.asics: + frr_daemon_memory_output = asic.run_vtysh(f'-c "show memory {daemon}"')["stdout"] + + logging.info( + f"{daemon}{('-' + asic.namespace) if asic.namespace else ''} memory status: \n%s", + frr_daemon_memory_output + ) + + frr_daemon_memory = asic.run_vtysh( + f'-c "show memory {daemon}" | grep "Used ordinary blocks"' + )["stdout"].split()[-2] + + frr_daemon_memory_per_asics[asic.asic_index] = frr_daemon_memory + + return frr_daemon_memory_per_asics + def test_cont_link_flap(self, request, duthosts, nbrhosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, bring_up_dut_interfaces, tbinfo): """ @@ -38,10 +62,10 @@ def test_cont_link_flap(self, request, duthosts, nbrhosts, enum_rand_one_per_hws to cause BGP Flaps. 2.) Flap all interfaces on peer (FanOutLeaf) one by one 1-3 iteration to cause BGP Flaps. - 3.) Watch for memory (show system-memory) ,orchagent CPU Utilization - and Redis_memory. + 3.) Watch for memory (show system-memory), FRR daemons memory(vtysh -c "show memory bgp/zebra"), + orchagent CPU Utilization and Redis_memory. - Pass Criteria: All routes must be re-learned with < 5% increase in Redis and + Pass Criteria: All routes must be re-learned with < 5% increase in Redis/FRR memory usage and ORCH agent CPU consumption below threshold after 3 mins after stopping flaps. """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] @@ -69,6 +93,13 @@ def test_cont_link_flap(self, request, duthosts, nbrhosts, enum_rand_one_per_hws logging.info("IPv4 routes: start {}, summary {}".format(start_time_ipv4_route_counts, sumv4)) logging.info("IPv6 routes: start {}, summary {}".format(start_time_ipv6_route_counts, sumv6)) + # Record FRR daemons memory status at start + frr_demons_to_check = ['bgpd', 'zebra'] + start_time_frr_daemon_memory = {} + for daemon in frr_demons_to_check: + start_time_frr_daemon_memory[daemon] = self.get_frr_daemon_memory_usage(duthost, daemon) + logging.info(f"{daemon} memory usage at start: \n%s", start_time_frr_daemon_memory[daemon]) + # Make Sure Orch CPU < orch_cpu_threshold before starting test. logging.info("Make Sure orchagent CPU utilization is less that %d before link flap", orch_cpu_threshold) pytest_assert(wait_until(100, 2, 0, check_orch_cpu_utilization, duthost, orch_cpu_threshold), @@ -124,10 +155,51 @@ def test_cont_link_flap(self, request, duthosts, nbrhosts, enum_rand_one_per_hws pytest.fail(str(failmsg)) + # Wait 30s for the memory usage to be stable + time.sleep(30) + # Record memory status at end memory_output = duthost.shell("show system-memory")["stdout"] logging.info("Memory Status at end: %s", memory_output) + # Check the FRR daemons memory usage at end + end_time_frr_daemon_memory = {} + incr_frr_daemon_memory_threshold = defaultdict(lambda: {}) + + for daemon in frr_demons_to_check: + for asic_index, asic_frr_memory in start_time_frr_daemon_memory[daemon].items(): + incr_frr_daemon_memory_threshold[daemon][asic_index] = 10 if tbinfo["topo"]["type"] in ["m0", "mx"]\ + else 5 + + min_threshold_percent = 1 / float(asic_frr_memory) * 100 + + if min_threshold_percent > incr_frr_daemon_memory_threshold[daemon][asic_index]: + incr_frr_daemon_memory_threshold[daemon][asic_index] = math.ceil(min_threshold_percent) + + logging.info(f"The memory increment threshold for frr daemon {daemon}-asic{asic_index} " + f"is {incr_frr_daemon_memory_threshold[daemon][asic_index]}%") + + for daemon in frr_demons_to_check: + # Record FRR daemon memory status at end + end_time_frr_daemon_memory[daemon] = self.get_frr_daemon_memory_usage(duthost, daemon) + logging.info(f"{daemon} memory usage at end: \n%s", end_time_frr_daemon_memory[daemon]) + + # Calculate diff in FRR daemon memory + for asic_index, end_frr_memory in end_time_frr_daemon_memory[daemon].items(): + incr_frr_daemon_memory = float(end_frr_memory) - float(start_time_frr_daemon_memory[daemon][asic_index]) + + daemon_name = daemon if not duthost.is_multi_asic else f"{daemon}-asic{asic_index}" + logging.info(f"{daemon_name} absolute difference: %d", incr_frr_daemon_memory) + + # Check FRR daemon memory only if it is increased else default to pass + if incr_frr_daemon_memory > 0: + percent_incr_frr_daemon_memory = \ + (incr_frr_daemon_memory / float(start_time_frr_daemon_memory[daemon][asic_index])) * 100 + logging.info(f"{daemon_name} memory percentage increase: %d", percent_incr_frr_daemon_memory) + pytest_assert(percent_incr_frr_daemon_memory < incr_frr_daemon_memory_threshold[daemon][asic_index], + f"{daemon_name} memory increase more than expected: " + f"{incr_frr_daemon_memory_threshold[daemon][asic_index]}%") + # Record orchagent CPU utilization at end orch_cpu = duthost.shell( "COLUMNS=512 show processes cpu | grep orchagent | awk '{print $1, $9}'")["stdout_lines"] diff --git a/tests/platform_tests/link_flap/test_link_flap.py b/tests/platform_tests/link_flap/test_link_flap.py index 730d8c7104..b52320cd3a 100644 --- a/tests/platform_tests/link_flap/test_link_flap.py +++ b/tests/platform_tests/link_flap/test_link_flap.py @@ -4,17 +4,16 @@ import logging import pytest -from tests.platform_tests.link_flap.link_flap_utils import toggle_one_link, check_orch_cpu_utilization -from tests.common.platform.device_utils import fanout_switch_port_lookup -from tests.common.helpers.assertions import pytest_assert +from tests.platform_tests.link_flap.link_flap_utils import check_orch_cpu_utilization, build_test_candidates +from tests.common.platform.device_utils import toggle_one_link +from tests.common.helpers.assertions import pytest_assert, pytest_require from tests.common.utilities import wait_until logger = logging.getLogger(__name__) pytestmark = [ pytest.mark.disable_loganalyzer, - pytest.mark.topology('any'), - pytest.mark.device_type('physical') + pytest.mark.topology('any') ] @@ -49,12 +48,8 @@ def test_link_flap(request, duthosts, rand_one_dut_hostname, tbinfo, fanouthosts loop_times = get_loop_times - port_lists = get_port_list(duthost, tbinfo) - - candidates = [] - for port in port_lists: - fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, duthost.hostname, port) - candidates.append((port, fanout, fanout_port)) + candidates = build_test_candidates(duthost, fanouthosts, 'all_ports') + pytest_require(candidates, "Didn't find any port that is admin up and present in the connection graph") for loop_time in range(0, loop_times): watch = False diff --git a/tests/platform_tests/mellanox/check_sysfs.py b/tests/platform_tests/mellanox/check_sysfs.py index 82a192d26d..4999695056 100644 --- a/tests/platform_tests/mellanox/check_sysfs.py +++ b/tests/platform_tests/mellanox/check_sysfs.py @@ -163,6 +163,10 @@ def check_sysfs(dut): logging.info("Check SFP related sysfs") for sfp_id, sfp_info in list(sysfs_facts['sfp_info'].items()): + # Skip when the sfp is missing + if not sfp_info["temp_fault"]: + continue + assert sfp_info["temp_fault"] == '0', "SFP%d temp fault" % int(sfp_id) sfp_temp = float(sfp_info['temp']) if sfp_info['temp'] != '0' else 0 sfp_temp_crit = float( diff --git a/tests/platform_tests/mellanox/test_psu_power_threshold.py b/tests/platform_tests/mellanox/test_psu_power_threshold.py index 9a1c220a32..3213777da1 100644 --- a/tests/platform_tests/mellanox/test_psu_power_threshold.py +++ b/tests/platform_tests/mellanox/test_psu_power_threshold.py @@ -6,8 +6,8 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.mellanox_data import get_platform_data from tests.common.utilities import wait_until # noqa F401 -from tests.platform_tests.thermal_control_test_helper import mocker_factory # noqa F401 -from .mellanox_thermal_control_test_helper import MockerHelper, PsuPowerThresholdMocker # noqa F401 +from tests.common.helpers.thermal_control_test_helper import mocker_factory # noqa F401 +from tests.common.helpers.mellanox_thermal_control_test_helper import MockerHelper, PsuPowerThresholdMocker # noqa F401 pytestmark = [ pytest.mark.asic('mellanox'), diff --git a/tests/platform_tests/mellanox/test_reboot_cause.py b/tests/platform_tests/mellanox/test_reboot_cause.py index e77915e2c1..04c8701cbb 100644 --- a/tests/platform_tests/mellanox/test_reboot_cause.py +++ b/tests/platform_tests/mellanox/test_reboot_cause.py @@ -2,7 +2,7 @@ import logging import pytest from tests.common.reboot import REBOOT_TYPE_BIOS, REBOOT_TYPE_ASIC, check_reboot_cause -from tests.platform_tests.thermal_control_test_helper import mocker_factory # noqa: F401 +from tests.common.helpers.thermal_control_test_helper import mocker_factory # noqa: F401 pytestmark = [ pytest.mark.asic('mellanox'), diff --git a/tests/platform_tests/sensors_utils/psu_data.yaml b/tests/platform_tests/sensors_utils/psu_data.yaml new file mode 100644 index 0000000000..6da9d62d82 --- /dev/null +++ b/tests/platform_tests/sensors_utils/psu_data.yaml @@ -0,0 +1,313 @@ +sensors_checks: + MTEF-AC-A: + alarms: + fan: + - dps460-i2c-*-*/PSU-* Fan 1/fan1_alarm + - dps460-i2c-*-*/PSU-* Fan 1/fan1_fault + power: + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in2_lcrit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Pwr (in)/power1_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_max_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Curr (in)/curr1_max_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Curr (in)/curr1_crit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_crit_alarm + temp: + - dps460-i2c-*-*/PSU-* Temp 1/temp1_max_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_max_alarm + compares: + power: [ ] + temp: + - - dps460-i2c-*-*/PSU-* Temp 1/temp1_input + - dps460-i2c-*-*/PSU-* Temp 1/temp1_max + - - dps460-i2c-*-*/PSU-* Temp 2/temp2_input + - dps460-i2c-*-*/PSU-* Temp 2/temp2_max + non_zero: + fan: [ ] + power: [ ] + temp: [ ] + psu_skips: { } + sensor_skip_per_version: { } + MTEF-AC-C: + alarms: + fan: + - dps460-i2c-*-*/PSU-* Fan 1/fan1_alarm + - dps460-i2c-*-*/PSU-* Fan 1/fan1_fault + power: + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_min_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_max_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_lcrit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_crit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_min_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_lcrit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_crit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Pwr (in)/power1_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_cap_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_crit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Curr (in)/curr1_max_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Curr (in)/curr1_crit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_lcrit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_crit_alarm + temp: + - dps460-i2c-*-*/PSU-* Temp 1/temp1_max_alarm + - dps460-i2c-*-*/PSU-* Temp 1/temp1_min_alarm + - dps460-i2c-*-*/PSU-* Temp 1/temp1_crit_alarm + - dps460-i2c-*-*/PSU-* Temp 1/temp1_lcrit_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_max_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_min_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_crit_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_lcrit_alarm + - dps460-i2c-*-*/PSU-* Temp 3/temp3_max_alarm + - dps460-i2c-*-*/PSU-* Temp 3/temp3_min_alarm + - dps460-i2c-*-*/PSU-* Temp 3/temp3_crit_alarm + - dps460-i2c-*-*/PSU-* Temp 3/temp3_lcrit_alarm + compares: + power: [ ] + temp: + - - dps460-i2c-*-*/PSU-* Temp 1/temp1_input + - dps460-i2c-*-*/PSU-* Temp 1/temp1_crit + - - dps460-i2c-*-*/PSU-* Temp 2/temp2_input + - dps460-i2c-*-*/PSU-* Temp 2/temp2_crit + - - dps460-i2c-*-*/PSU-* Temp 3/temp3_input + - dps460-i2c-*-*/PSU-* Temp 3/temp3_crit + non_zero: + fan: [ ] + power: [ ] + temp: [ ] + psu_skips: { } + sensor_skip_per_version: { } + MTEF-AC-E: + alarms: + fan: + - dps460-i2c-*-*/PSU-* Fan 1/fan1_alarm + - dps460-i2c-*-*/PSU-* Fan 1/fan1_fault + power: + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Pwr (in)/power1_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_max_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Curr (in)/curr1_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_max_alarm + temp: + - dps460-i2c-*-*/PSU-* Temp 1/temp1_max_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_max_alarm + - dps460-i2c-*-*/PSU-* Temp 3/temp3_max_alarm + compares: + power: [ ] + temp: + - - dps460-i2c-*-*/PSU-* Temp 1/temp1_input + - dps460-i2c-*-*/PSU-* Temp 1/temp1_max + - - dps460-i2c-*-*/PSU-* Temp 2/temp2_input + - dps460-i2c-*-*/PSU-* Temp 2/temp2_max + - - dps460-i2c-*-*/PSU-* Temp 3/temp3_input + - dps460-i2c-*-*/PSU-* Temp 3/temp3_max + non_zero: + fan: [ ] + power: [ ] + temp: [ ] + psu_skips: { } + sensor_skip_per_version: { } + MTEF-AC-F: + alarms: + fan: + - dps460-i2c-*-*/PSU-* Fan 1/fan1_alarm + - dps460-i2c-*-*/PSU-* Fan 1/fan1_fault + power: + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_min_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_max_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_lcrit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_crit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_min_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_lcrit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_crit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Pwr (in)/power1_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_crit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Curr (in)/curr1_max_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Curr (in)/curr1_crit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_crit_alarm + temp: + - dps460-i2c-*-*/PSU-* Temp 1/temp1_max_alarm + - dps460-i2c-*-*/PSU-* Temp 1/temp1_crit_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_max_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_crit_alarm + - dps460-i2c-*-*/PSU-* Temp 3/temp3_max_alarm + - dps460-i2c-*-*/PSU-* Temp 3/temp3_crit_alarm + compares: + power: [ ] + temp: + - - dps460-i2c-*-*/PSU-* Temp 1/temp1_input + - dps460-i2c-*-*/PSU-* Temp 1/temp1_crit + - - dps460-i2c-*-*/PSU-* Temp 2/temp2_input + - dps460-i2c-*-*/PSU-* Temp 2/temp2_crit + - - dps460-i2c-*-*/PSU-* Temp 3/temp3_input + - dps460-i2c-*-*/PSU-* Temp 3/temp3_crit + non_zero: + fan: [ ] + power: [ ] + temp: [ ] + psu_skips: { } + sensor_skip_per_version: { } + MTEF-AC-M: + alarms: + fan: + - pmbus-i2c-*-*/PSU-* Fan1/fan1_alarm + - pmbus-i2c-*-*/PSU-* Fan1/fan1_fault + power: + - pmbus-i2c-*-*/PSU-* 220V Rail(in)/in1_min_alarm + - pmbus-i2c-*-*/PSU-* 220V Rail(in)/in1_max_alarm + - pmbus-i2c-*-*/PSU-* 220V Rail(in)/in1_lcrit_alarm + - pmbus-i2c-*-*/PSU-* 220V Rail(in)/in1_crit_alarm + - pmbus-i2c-*-*/PSU-* 12V Rail(out)/in2_min_alarm + - pmbus-i2c-*-*/PSU-* 12V Rail(out)/in2_lcrit_alarm + - pmbus-i2c-*-*/PSU-* 12V Rail(out)/in2_crit_alarm + - pmbus-i2c-*-*/PSU-* 220V Rail Pwr(in)/power1_alarm + - pmbus-i2c-*-*/PSU-* 12V Rail Pwr(out)/power2_cap_alarm + - pmbus-i2c-*-*/PSU-* 12V Rail Pwr(out)/power2_max_alarm + - pmbus-i2c-*-*/PSU-* 220V Rail Curr(in)/curr1_max_alarm + - pmbus-i2c-*-*/PSU-* 12V Rail Curr(out)/curr2_max_alarm + - pmbus-i2c-*-*/PSU-* 12V Rail Curr(out)/curr2_crit_alarm + temp: + - pmbus-i2c-*-*/PSU-* Temp1/temp1_max_alarm + - pmbus-i2c-*-*/PSU-* Temp1/temp1_min_alarm + - pmbus-i2c-*-*/PSU-* Temp1/temp1_crit_alarm + - pmbus-i2c-*-*/PSU-* Temp1/temp1_lcrit_alarm + - pmbus-i2c-*-*/PSU-* Temp2/temp2_max_alarm + - pmbus-i2c-*-*/PSU-* Temp2/temp2_min_alarm + - pmbus-i2c-*-*/PSU-* Temp2/temp2_crit_alarm + - pmbus-i2c-*-*/PSU-* Temp2/temp2_lcrit_alarm + - pmbus-i2c-*-*/PSU-* Temp3/temp3_max_alarm + - pmbus-i2c-*-*/PSU-* Temp3/temp3_min_alarm + - pmbus-i2c-*-*/PSU-* Temp3/temp3_crit_alarm + - pmbus-i2c-*-*/PSU-* Temp3/temp3_lcrit_alarm + compares: + power: [ ] + temp: + - - pmbus-i2c-*-*/PSU-* Temp1/temp1_input + - pmbus-i2c-*-*/PSU-* Temp1/temp1_crit + - - pmbus-i2c-*-*/PSU-* Temp2/temp2_input + - pmbus-i2c-*-*/PSU-* Temp2/temp2_crit + - - pmbus-i2c-*-*/PSU-* Temp3/temp3_input + - pmbus-i2c-*-*/PSU-* Temp3/temp3_crit + non_zero: + fan: [ ] + power: [ ] + temp: [ ] + psu_skips: { } + sensor_skip_per_version: { } + 930-9SPSU-00RA-00B: + alarms: + fan: + - dps460-i2c-*-*/PSU-* Fan 1/fan1_alarm + - dps460-i2c-*-*/PSU-* Fan 1/fan1_fault + power: + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_min_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_max_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_lcrit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_crit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Pwr (in)/power1_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_crit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Curr (in)/curr1_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_crit_alarm + temp: + - dps460-i2c-*-*/PSU-* Temp 1/temp1_max_alarm + - dps460-i2c-*-*/PSU-* Temp 1/temp1_crit_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_max_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_crit_alarm + - dps460-i2c-*-*/PSU-* Temp 3/temp3_max_alarm + - dps460-i2c-*-*/PSU-* Temp 3/temp3_crit_alarm + compares: + power: [ ] + temp: + - - dps460-i2c-*-*/PSU-* Temp 1/temp1_input + - dps460-i2c-*-*/PSU-* Temp 1/temp1_crit + - - dps460-i2c-*-*/PSU-* Temp 2/temp2_input + - dps460-i2c-*-*/PSU-* Temp 2/temp2_crit + - - dps460-i2c-*-*/PSU-* Temp 3/temp3_input + - dps460-i2c-*-*/PSU-* Temp 3/temp3_crit + non_zero: + fan: [ ] + power: [ ] + temp: [ ] + psu_skips: { } + sensor_skip_per_version: { } + MTEF-AC-C-SS1: + alarms: + fan: + - dps460-i2c-*-*/PSU-* Fan 1/fan1_alarm + - dps460-i2c-*-*/PSU-* Fan 1/fan1_fault + power: + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_min_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_max_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_lcrit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_crit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_min_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_lcrit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in3_crit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Pwr (in)/power1_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_crit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Curr (in)/curr1_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_crit_alarm + temp: + - dps460-i2c-*-*/PSU-* Temp 1/temp1_max_alarm + - dps460-i2c-*-*/PSU-* Temp 1/temp1_crit_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_max_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_crit_alarm + - dps460-i2c-*-*/PSU-* Temp 3/temp3_max_alarm + - dps460-i2c-*-*/PSU-* Temp 3/temp3_crit_alarm + compares: + power: [ ] + temp: + - - dps460-i2c-*-*/PSU-* Temp 1/temp1_input + - dps460-i2c-*-*/PSU-* Temp 1/temp1_crit + - - dps460-i2c-*-*/PSU-* Temp 2/temp2_input + - dps460-i2c-*-*/PSU-* Temp 2/temp2_crit + - - dps460-i2c-*-*/PSU-* Temp 3/temp3_input + - dps460-i2c-*-*/PSU-* Temp 3/temp3_crit + non_zero: + fan: [ ] + power: [ ] + temp: [ ] + psu_skips: { } + sensor_skip_per_version: { } + MTEF-AC-G: + alarms: + fan: + - dps460-i2c-*-*/PSU-* Fan 1/fan1_alarm + - dps460-i2c-*-*/PSU-* Fan 1/fan1_fault + power: + - dps460-i2c-*-*/PSU-* 220V Rail (in)/in1_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in2_lcrit_alarm + - dps460-i2c-*-*/PSU-* 12V Rail (out)/in2_crit_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Pwr (in)/power1_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Pwr (out)/power2_max_alarm + - dps460-i2c-*-*/PSU-* 220V Rail Curr (in)/curr1_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_max_alarm + - dps460-i2c-*-*/PSU-* 12V Rail Curr (out)/curr2_lcrit_alarm + temp: + - dps460-i2c-*-*/PSU-* Temp 1/temp1_max_alarm + - dps460-i2c-*-*/PSU-* Temp 2/temp2_max_alarm + compares: + power: [ ] + temp: + - - dps460-i2c-*-*/PSU-* Temp 1/temp1_input + - dps460-i2c-*-*/PSU-* Temp 1/temp1_max + - - dps460-i2c-*-*/PSU-* Temp 2/temp2_input + - dps460-i2c-*-*/PSU-* Temp 2/temp2_max + non_zero: + fan: [ ] + power: [ ] + temp: [ ] + psu_skips: { } + sensor_skip_per_version: { } diff --git a/tests/platform_tests/sensors_utils/psu_sensor_utils.py b/tests/platform_tests/sensors_utils/psu_sensor_utils.py new file mode 100644 index 0000000000..effb1951f8 --- /dev/null +++ b/tests/platform_tests/sensors_utils/psu_sensor_utils.py @@ -0,0 +1,324 @@ +import json +import logging +import os +import re +import yaml + +PSU_SENSORS_DATA_FILE = "psu_data.yaml" +PSU_SENSORS_JSON_FILE = "psu_sensors.json" +MISSING_PSU = "N/A" +PSU_NUM_SENSOR_PATTERN = r'PSU-(\d+)(?:\([A-Z]\))?' +SKIPPED_CHECK_TYPES = ["psu_skips", "sensor_skip_per_version"] +PSU_FAN_DIR_PATTERN = r'-(PSF|PSR)-' +# PSUs in this dictionary are only supported with the specified manufacturers +PSU_MANUFACTURER_SUPPORT = {'MTEF-AC-G': ['ACBEL']} +VPD_DATA_PATH_FORMAT = "/var/run/hw-management/eeprom/psu{PSU_INDEX}_vpd" +PSU_VPD_MANUFACTURER_FIELD = 'MFR_NAME' +logger = logging.getLogger() + + +def update_sensor(sensor_path, psu_num, bus_num, bus_address, psu_side): + """ + Updates the sensor_path (original format can be seen in psu-data.yml) to contain platform related information + :param sensor_path: Sensor path as taken from psu-data.yml (contains * where platform related data is needed) + :param psu_num: The psu number + :param bus_num: bus number of sensors from this psu number + :param bus_address: bus address of sensors from this psu number + :param psu_side: psu_side of this psu number, could be empty + :returns: full parsed psu sensor path + """ + # Some of the sensors from sku-sensors-data.yml are parsed together as a list and not separately as a string, + # to make the processing identical, we process both cases as a list + paths_to_add = [] + add_as_list = False + if isinstance(sensor_path, list): + sensor_paths = sensor_path + add_as_list = True + else: + sensor_paths = [sensor_path] + for path in sensor_paths: + path_with_bus_data = path.replace("*-*", f"{bus_num}-{bus_address}") + if psu_side: + path_with_psu_num = path_with_bus_data.replace("*", f"{psu_num}({psu_side})") + else: + path_with_psu_num = path_with_bus_data.replace("*", f"{psu_num}") + paths_to_add.append(path_with_psu_num) + if add_as_list: + return paths_to_add + return paths_to_add[0] + + +def is_psu_sensor(sensor, psu_sensor_prefix): + """ + The function returns whether the current sensor is a psu sensor + :param sensor: a sensor path in the format of sensors-sku-data.yml file + :param psu_sensor_prefix: psu sensor prefix of the platform - for example dps460-i2c + :return: True if sensor is a psu sensor and False otherwise + """ + psu_sensor_pattern = rf'^{psu_sensor_prefix}-.*-.*' + # Sometimes (line in compares section, the yaml will convert the sensors to list of sensor paths) + if isinstance(sensor, list): + return all([re.match(psu_sensor_pattern, sensor_path) for sensor_path in sensor]) + else: + return bool(re.match(psu_sensor_pattern, sensor)) + + +def parse_num_from_sensor_path(sensor_path): + """ + Parses the psu number from the sensor_path + :param sensor_path: a sensor path in the format of sensors-sku-data.yml file + :return: The slot of the PSU sensor_path is a part of or None if no slot was found (means it's not a PSU sensor) + """ + match = re.search(PSU_NUM_SENSOR_PATTERN, sensor_path) + if match: + return match.group(1) + else: + logger.error(f"Couldn't find PSU number in {sensor_path}") + + +def should_replace_sensor(sensor_path, psu_nums_to_remove, psu_sensor_prefix): + """ + The function returns whether the sensor_path is related to a PSU slot in psu_nums_to_replace + :param sensor_path: a sensor path in the format of sensors-sku-data.yml file + :param psu_nums_to_remove: set of psu numbers whose numbers we want to remove + :param psu_sensor_prefix: psu sensor prefix of the platform - for example dps460-i2c + :return: a dictionary of installed PSUs entries, mapping psu slots (numbers) to the psu models + + """ + if not is_psu_sensor(sensor_path, psu_sensor_prefix): + return False + + if isinstance(sensor_path, list): + return all([parse_num_from_sensor_path(path) in psu_nums_to_remove for path in sensor_path]) + else: + return parse_num_from_sensor_path(sensor_path) in psu_nums_to_remove + + +def update_sensor_data(alarm_data, psu_platform_data, psu_numbers): + """ + The function updates the alarm_data according to psu_platform_data for each of the psu_numbers listed + :param alarm_data: a list of psu sensors of some alarm_type + :param psu_platform_data: A dictionary containing for each psu number, the bus number, bus address and PSU slot side + (empty if doesn't exist) + :param psu_numbers: A list of numbers we want to retrieve psu info from psu_platform_data + :return: a dictionary of installed PSUs entries, mapping psu slots (numbers) to the psu models + """ + updated_alarm_data = [] + for psu_num in psu_numbers: + bus_num, bus_address, psu_side = psu_platform_data[psu_num] + updated_alarm_data.extend([update_sensor(sensor_path, psu_num, bus_num, bus_address, psu_side) for + sensor_path in alarm_data]) + return updated_alarm_data + + +def parse_psu_manufacturer(duthost, psu_index): + """ + The function parses the psu manufacturer of the psu installed at the given index from the vpd_data + :param duthost: duthost fixture + :param psu_index: Index of the psu + :return: The psu manufacturer name as it appears in the psu vpd_data + """ + vpd_data_path = VPD_DATA_PATH_FORMAT.format(PSU_INDEX=psu_index) + manufacturer_cmd = f"cat {vpd_data_path} | grep {PSU_VPD_MANUFACTURER_FIELD}" + psu_manufacturer_line = duthost.shell(manufacturer_cmd)["stdout"] + manufacturer_name_ind = 1 + return psu_manufacturer_line.split(':')[manufacturer_name_ind].strip() + + +class SensorHelper: + """ + Helper class to the test_sensors tests + """ + + def __init__(self, duthost): + """ + Setup important variables of the class + """ + + self.missing_psus = None + self.supports_dynamic_psus = False + self.psu_dict = None + self.uncovered_psus = None + self.psu_platform_data = None + self.psu_sensors_checks = None + self.duthost = duthost + self.platform = self.duthost.facts['platform'] + self.fetch_psu_data() + self.read_psus_from_dut() + + def fetch_psu_data(self): + """ + Parses psu_data and psu_sensor files into needed variables + """ + psu_sensors_data_file_fullpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), + PSU_SENSORS_DATA_FILE) + psu_sensors_data = yaml.safe_load(open(psu_sensors_data_file_fullpath).read()) + self.psu_sensors_checks = psu_sensors_data['sensors_checks'] + psu_sensors_json_file_fullpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), + PSU_SENSORS_JSON_FILE) + with open(psu_sensors_json_file_fullpath) as file: + psu_json_mapping = json.load(file) + if self.platform in psu_json_mapping: + self.psu_platform_data = psu_json_mapping[self.platform] + self.supports_dynamic_psus = True + else: + logger.warning(f"Platform {self.platform} does not support dynamic testing of PSU sensors. " + f"Test will run without fetching psu sensors dynamically.") + + def read_psus_from_dut(self): + """ + The function reads the psus installed on the dut and initialized 2 fields of the class. + The first field is a dictionary called psu_dict that consists of entries {psuNum: psuModel} of PSU installed on + the dut that exist in the dictionary field psu_sensors_checks . + The second field is a set called uncovered_psus and consists of psu models not in the dictionary field + psu_sensors_checks + """ + self.psu_dict = dict() + self.uncovered_psus = set() + self.missing_psus = set() + if self.supports_dynamic_psus: + psu_data = json.loads(self.duthost.shell('show platform psu --json')['stdout']) + for psu in psu_data: + psu_index, psu_model = psu["index"], psu["model"] + # Ignore PSR/PSF part, as we don't care if the fan is reversed (PSR) or not (PSF) + psu_model_no_fan_dir = re.sub(PSU_FAN_DIR_PATTERN, '-', psu_model) + if self.is_supported_psu_model(psu_index, psu_model_no_fan_dir): + self.psu_dict[psu_index] = psu_model_no_fan_dir + elif psu["model"] == MISSING_PSU: + self.missing_psus.add(psu_index) + logger.warning(f"Slot {psu_index} is missing a PSU.") + else: + self.uncovered_psus.add(psu_model_no_fan_dir) + + def platform_supports_dynamic_psu(self): + """ + Getter function for the field supports_dynamic_psus + """ + return self.supports_dynamic_psus + + def get_missing_psus(self): + """ + Getter function for the field missing_psus + """ + return self.missing_psus + + def get_psu_index_model_dict(self): + """ + Getter function for self.psu_dict created in the function read_psus_from_dut. + """ + return self.psu_dict + + def get_uncovered_psus(self): + """ + Returns a set of psus on the platform that do not have their sensors in psu_data.yml + """ + return self.uncovered_psus + + def remove_psu_checks(self, sensor_checks, psu_nums_to_remove): + """ + This function removes all psu sensor_checks of a certain psu from the platform sensor_checks + :param sensor_checks: the sensors data fetched from sensors-sku-data.yml file + :param psu_nums_to_remove: set of psu numbers whose numbers we want to remove from sensor_checks + """ + for check_type, checks in sensor_checks.items(): + if check_type in SKIPPED_CHECK_TYPES: + continue + for alarm_hw_type, alarm_data in checks.items(): + platform_sensors = [] + for sensor_path in alarm_data: + if not should_replace_sensor(sensor_path, psu_nums_to_remove, self.get_sensor_psu_prefix()): + platform_sensors.append(sensor_path) + else: + logger.debug(f"Removed PSU sensor - {sensor_path}") + checks[alarm_hw_type] = platform_sensors + + def get_sensor_psu_prefix(self): + """ + This function will fetch the sensor bus pattern prefix from psu_sensors_data. + :return: sensor psu sensor prefix without bus num and address of dut - i.e., dps460-i2c of dps460-i2c-4-58" + """ + psu_bus_path = list(self.psu_platform_data["default"]["chip"].keys())[0] # grab some key from the chip part + # the psu_bus_path will look something like dps460-i2c-*-58 - we want to generalize it - dps460-i2c-*-* + psu_bus_parts = psu_bus_path.split('-')[:2] # Split the string by '-', the prefix is the first 2 words + return '-'.join(psu_bus_parts) + + def update_psu_sensors(self, sensors_checks, psu_models_to_replace, hardware_version): + """ + This function adds to sensor_checks the PSU sensors fetched in runtime from psu-sensors.yml file. + :param sensors_checks: the sensors data fetched from sensors-sku-data.yml file, after removal of psu sensors + :param psu_models_to_replace: set of psu numbers whose numbers we want to remove from sensor_checks + :param hardware_version: hardware version as retrieved from mellanox_data.get_hardware_version + """ + psu_platform_data = self.parse_psu_json_mapping(set(psu_models_to_replace.keys()), hardware_version) + # create mapping from psu_models to sets of psu numbers matching them + psu_nums_per_psu = dict() + for psu_num, psu_model in psu_models_to_replace.items(): + psu_nums_per_psu.setdefault(psu_model, set()).add(psu_num) + + # For each psu, update the generalized psu sensors in psu_sensors_data with this psu platform related data + for psu_model, psu_nums in psu_nums_per_psu.items(): + # Grab the generalized checks we need to update for each psu slot that matches the model + psu_sensors_checks = self.psu_sensors_checks[psu_model] + for check_type, checks in sensors_checks.items(): + if check_type in SKIPPED_CHECK_TYPES: + continue + for alarm_hw_type, alarm_data in checks.items(): + psu_alarm_data = psu_sensors_checks[check_type][alarm_hw_type] + updated_alarm_data = update_sensor_data(psu_alarm_data, psu_platform_data, psu_nums) + checks[alarm_hw_type].extend(updated_alarm_data) + + def parse_psu_json_mapping(self, psu_nums_to_replace, hardware_version): + """ + This function returns a dictionary that contains for each PSU slot in the device, the bus number, + bus address and psu side + :param psu_json_mapping: mapping from platform to relevant data regarding the psu sensors + :param psu_nums_to_replace: set of psu numbers we want to fetch sensors for + :param platform: the platform of the dut + :param hardware_version: hardware version as retrieved from mellanox_data.get_hardware_version + :returns: A dictionary containing for each psu number, the bus number, bus address and PSU slot + side (empty if doesn't exist) + """ + + psu_json_data = dict() + hw_type = hardware_version if hardware_version in self.psu_platform_data.keys() else "default" + bus_data = self.psu_platform_data[hw_type]["bus"] + chip_data = self.psu_platform_data[hw_type]["chip"] + for chip_key, chip_value in chip_data.items(): + if len(chip_value) == 1: # means we have no side in the sensor path + psu_num = chip_value[0] + psu_side = "" + else: + psu_num, psu_side = chip_value + bus_address = chip_key.split('-')[-1] + if psu_num in psu_nums_to_replace: + bus_number = chip_key.split('-')[-2] # we try to get it from the bus_data of chip part but it can be * + if bus_number == '*': # if the bus data is same for all slots, it will be * in the chip part and we + # take it from general bus_data part + bus_number = bus_data[0].split('-')[1] + psu_json_data[psu_num] = (bus_number, bus_address, psu_side) + return psu_json_data + + def is_supported_psu_model(self, psu_index, psu_model): + """ + This function returns whether the given psu_model should be supported or not + :param psu_index: The index the psu is installed in + :param psu_model: A psu model (without fan direction) + :returns: A boolean stating whether the psu dynamic feature is supported with this psu model + """ + covered_psus = set(self.psu_sensors_checks.keys()) + + return psu_model in covered_psus and self.is_psu_manufacturer_supported(psu_index, psu_model) + + def is_psu_manufacturer_supported(self, psu_index, psu_model): + """ + This function returns whether the given psu_model is installed with supported manufacturers + :param psu_index: The index the psu is installed in + :param psu_model: A psu model (without fan direction) + :returns: A boolean stating whether the psu dynamic feature is supported with this psu model its manufacturer + """ + manufacturer_supported = True + if psu_model in PSU_MANUFACTURER_SUPPORT: + supported_psu_manufacturers = PSU_MANUFACTURER_SUPPORT[psu_model] + psu_manufacturer = parse_psu_manufacturer(self.duthost, psu_index) + manufacturer_supported &= psu_manufacturer in supported_psu_manufacturers + return manufacturer_supported diff --git a/tests/platform_tests/sensors_utils/psu_sensors.json b/tests/platform_tests/sensors_utils/psu_sensors.json new file mode 100644 index 0000000000..1188d5411a --- /dev/null +++ b/tests/platform_tests/sensors_utils/psu_sensors.json @@ -0,0 +1,424 @@ +{ + "x86_64-mlnx_msn2410-r0": { + "default": { + "bus": [ + "i2c-10", + "i2c-1-mux (chan_id 10)" + ], + "chip": { + "dps460-i2c-*-58": [ + "2", + "R" + ], + "dps460-i2c-*-59": [ + "1", + "L" + ] + } + } + }, + "x86_64-mlnx_msn2700-r0": { + "default": { + "bus": [ + "i2c-10", + "i2c-1-mux (chan_id 10)" + ], + "chip": { + "dps460-i2c-*-58": [ + "2", + "R" + ], + "dps460-i2c-*-59": [ + "1", + "L" + ] + } + } + }, + "x86_64-mlnx_msn2700a1-r0": { + "default": { + "bus": [ + "i2c-10", + "i2c-1-mux (chan_id 1)" + ], + "chip": { + "dps460-i2c-*-58": [ + "2", + "R" + ], + "dps460-i2c-*-59": [ + "1", + "L" + ] + } + } + }, + "x86_64-mlnx_msn2740-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "2", + "R" + ], + "dps460-i2c-*-59": [ + "1", + "L" + ] + } + } + }, + "x86_64-mlnx_msn3420-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "2" + ], + "dps460-i2c-*-59": [ + "1" + ] + } + } + }, + "x86_64-mlnx_msn3700-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "2" + ], + "dps460-i2c-*-59": [ + "1" + ] + } + }, + "respined": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "1" + ], + "dps460-i2c-*-59": [ + "2" + ] + } + }, + "swb-respined": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "1" + ], + "dps460-i2c-*-59": [ + "2" + ] + } + } + }, + "x86_64-mlnx_msn3700c-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "2" + ], + "dps460-i2c-*-59": [ + "1" + ] + } + }, + "respined": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "1" + ], + "dps460-i2c-*-59": [ + "2" + ] + } + }, + "swb-respined": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "1" + ], + "dps460-i2c-*-59": [ + "2" + ] + } + } + }, + "x86_64-mlnx_msn3800-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "2" + ], + "dps460-i2c-*-59": [ + "1" + ] + } + } + }, + "x86_64-mlnx_msn4410-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "1", + "L" + ], + "dps460-i2c-*-59": [ + "2", + "R" + ] + } + }, + "a1": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "1", + "L" + ], + "dps460-i2c-*-59": [ + "2", + "R" + ] + } + } + }, + "x86_64-mlnx_msn4600-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-59": [ + "1", + "L" + ], + "dps460-i2c-*-58": [ + "2", + "R" + ] + } + } + }, + "x86_64-mlnx_msn4600c-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-59": [ + "1", + "L" + ], + "dps460-i2c-*-58": [ + "2", + "R" + ] + } + }, + "a1": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "1", + "L" + ], + "dps460-i2c-*-59": [ + "2", + "R" + ] + } + }, + "respined": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-59": [ + "1", + "L" + ], + "dps460-i2c-*-58": [ + "2", + "R" + ] + } + }, + "a1-respined": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "1", + "L" + ], + "dps460-i2c-*-59": [ + "2", + "R" + ] + } + } + }, + "x86_64-mlnx_msn4700-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "1", + "L" + ], + "dps460-i2c-*-59": [ + "2", + "R" + ] + } + }, + "a1": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-58": [ + "1", + "L" + ], + "dps460-i2c-*-59": [ + "2", + "R" + ] + } + } + }, + "x86_64-nvidia_sn2201-r0": { + "default": { + "bus": [ + "i2c-3", + "i2c-1-mux (chan_id 1)" + ], + "chip": { + "pmbus-i2c-3-58": [ + "1" + ], + "pmbus-i2c-4-58": [ + "2" + ] + } + } + }, + "x86_64-nvidia_sn4800-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-59": [ + "1", + "R" + ], + "dps460-i2c-*-58": [ + "2", + "R" + ], + "dps460-i2c-*-5b": [ + "3", + "L" + ], + "dps460-i2c-*-5a": [ + "4", + "L" + ] + } + } + }, + "x86_64-nvidia_sn5400-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-59": [ + "1", + "L" + ], + "dps460-i2c-*-5a": [ + "2", + "R" + ] + } + } + }, + "x86_64-nvidia_sn5600-r0": { + "default": { + "bus": [ + "i2c-4", + "i2c-1-mux (chan_id 3)" + ], + "chip": { + "dps460-i2c-*-59": [ + "1", + "L" + ], + "dps460-i2c-*-5a": [ + "2", + "R" + ] + } + } + } +} diff --git a/tests/platform_tests/sfp/test_sfputil.py b/tests/platform_tests/sfp/test_sfputil.py index 49967f591f..472394df31 100644 --- a/tests/platform_tests/sfp/test_sfputil.py +++ b/tests/platform_tests/sfp/test_sfputil.py @@ -13,7 +13,7 @@ from .util import parse_eeprom from .util import parse_output from .util import get_dev_conn -from tests.common.utilities import skip_release +from tests.common.utilities import skip_release, wait_until from tests.common.fixtures.duthost_utils import shutdown_ebgp # noqa F401 cmd_sfp_presence = "sudo sfputil show presence" @@ -30,6 +30,16 @@ ] +def check_interfaces_up(duthost, namespace, up_ports): + logging.info("Checking interface status") + intf_facts = duthost.interface_facts(namespace=namespace, up_ports=up_ports)["ansible_facts"] + if len(intf_facts["ansible_interface_link_down_ports"]) == 0: + return True + else: + logging.info("Some interfaces are down: {}".format(intf_facts["ansible_interface_link_down_ports"])) + return False + + def test_check_sfputil_presence(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_frontend_asic_index, conn_graph_facts, xcvr_skip_list): """ @@ -41,7 +51,13 @@ def test_check_sfputil_presence(duthosts, enum_rand_one_per_hwsku_frontend_hostn portmap, dev_conn = get_dev_conn(duthost, conn_graph_facts, enum_frontend_asic_index) logging.info("Check output of '{}'".format(cmd_sfp_presence)) - sfp_presence = duthost.command(cmd_sfp_presence) + sfp_presence = duthost.command(cmd_sfp_presence, module_ignore_errors=True) + + # For vs testbed, we will get expected Error code `ERROR_CHASSIS_LOAD = 2` here. + if duthost.facts["asic_type"] == "vs" and sfp_presence['rc'] == 2: + return + assert sfp_presence['rc'] == 0, "Run command '{}' failed".format(cmd_sfp_presence) + parsed_presence = parse_output(sfp_presence["stdout_lines"][2:]) for intf in dev_conn: if intf not in xcvr_skip_list[duthost.hostname]: @@ -90,7 +106,13 @@ def test_check_sfputil_eeprom(duthosts, enum_rand_one_per_hwsku_frontend_hostnam portmap, dev_conn = get_dev_conn(duthost, conn_graph_facts, enum_frontend_asic_index) logging.info("Check output of '{}'".format(cmd_sfp_eeprom)) - sfp_eeprom = duthost.command(cmd_sfp_eeprom) + sfp_eeprom = duthost.command(cmd_sfp_eeprom, module_ignore_errors=True) + + # For vs testbed, we will get expected Error code `ERROR_CHASSIS_LOAD = 2` here. + if duthost.facts["asic_type"] == "vs" and sfp_eeprom['rc'] == 2: + return + assert sfp_eeprom['rc'] == 0, "Run command '{}' failed".format(cmd_sfp_presence) + parsed_eeprom = parse_eeprom(sfp_eeprom["stdout_lines"]) for intf in dev_conn: if intf not in xcvr_skip_list[duthost.hostname]: @@ -129,7 +151,14 @@ def test_check_sfputil_reset(duthosts, enum_rand_one_per_hwsku_frontend_hostname time.sleep(sleep_time) logging.info("Check sfp presence again after reset") - sfp_presence = duthost.command(cmd_sfp_presence) + sfp_presence = duthost.command(cmd_sfp_presence, module_ignore_errors=True) + + # For vs testbed, we will get expected Error code `ERROR_CHASSIS_LOAD = 2` here. + if duthost.facts["asic_type"] == "vs" and sfp_presence['rc'] == 2: + pass + else: + assert sfp_presence['rc'] == 0, "Run command '{}' failed".format(cmd_sfp_presence) + parsed_presence = parse_output(sfp_presence["stdout_lines"][2:]) for intf in dev_conn: if intf not in xcvr_skip_list[duthost.hostname]: @@ -162,7 +191,14 @@ def test_check_sfputil_low_power_mode(duthosts, enum_rand_one_per_hwsku_frontend global ans_host ans_host = duthost logging.info("Check output of '{}'".format(cmd_sfp_show_lpmode)) - lpmode_show = duthost.command(cmd_sfp_show_lpmode) + lpmode_show = duthost.command(cmd_sfp_show_lpmode, module_ignore_errors=True) + + # For vs testbed, we will get expected Error code `ERROR_CHASSIS_LOAD = 2` here. + if duthost.facts["asic_type"] == "vs" and lpmode_show['rc'] == 2: + pass + else: + assert lpmode_show['rc'] == 0, "Run command '{}' failed".format(cmd_sfp_presence) + parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:]) original_lpmode = copy.deepcopy(parsed_lpmode) for intf in dev_conn: @@ -264,6 +300,7 @@ def test_check_sfputil_low_power_mode(duthosts, enum_rand_one_per_hwsku_frontend if enum_frontend_asic_index is not None: # Check if the interfaces of this AISC is present in conn_graph_facts up_ports = {k: v for k, v in list(portmap.items()) if k in mg_facts["minigraph_ports"]} - intf_facts = duthost.interface_facts(namespace=namespace, up_ports=up_ports)["ansible_facts"] - assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \ - "Some interfaces are down: {}".format(intf_facts["ansible_interface_link_down_ports"]) + all_intf_up = wait_until(100, 10, 0, check_interfaces_up, duthost, namespace, up_ports) + if not all_intf_up: + intf_facts = duthost.interface_facts(namespace=namespace, up_ports=up_ports)["ansible_facts"] + assert all_intf_up, "Some interfaces are down: {}".format(intf_facts["ansible_interface_link_down_ports"]) diff --git a/tests/platform_tests/sfp/test_show_intf_xcvr.py b/tests/platform_tests/sfp/test_show_intf_xcvr.py index f40b40b965..928720e0a3 100644 --- a/tests/platform_tests/sfp/test_show_intf_xcvr.py +++ b/tests/platform_tests/sfp/test_show_intf_xcvr.py @@ -76,8 +76,14 @@ def test_check_show_lpmode(duthosts, enum_rand_one_per_hwsku_frontend_hostname, duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] portmap, dev_conn = get_dev_conn( duthost, conn_graph_facts, enum_frontend_asic_index) - sfp_lpmode = duthost.command(cmd_sfp_lpmode) + sfp_lpmode = duthost.command(cmd_sfp_lpmode, module_ignore_errors=True) + + # For vs testbed, we will get expected Error code `ERROR_CHASSIS_LOAD = 2` here. + if duthost.facts["asic_type"] == "vs" and sfp_lpmode['rc'] == 2: + return + assert sfp_lpmode['rc'] == 0, "Run command '{}' failed".format(cmd_sfp_presence) + for intf in dev_conn: if intf not in xcvr_skip_list[duthost.hostname]: assert validate_transceiver_lpmode( - sfp_lpmode), "Interface mode incorrect in 'show interface transceiver lpmode'" + sfp_lpmode['stdout']), "Interface mode incorrect in 'show interface transceiver lpmode'" diff --git a/tests/platform_tests/sfp/util.py b/tests/platform_tests/sfp/util.py index fcd3d1dd18..50065ceb42 100644 --- a/tests/platform_tests/sfp/util.py +++ b/tests/platform_tests/sfp/util.py @@ -50,12 +50,12 @@ def get_dev_conn(duthost, conn_graph_facts, asic_index): def validate_transceiver_lpmode(output): lines = output.strip().split('\n') # Check if the header is present - if lines[0].strip() != "Port Low-power Mode": - print("Invalid output format: Header missing") + if lines[0].replace(" ", "") != "Port Low-power Mode".replace(" ", ""): + logging.error("Invalid output format: Header missing") return False for line in lines[2:]: port, lpmode = line.strip().split() if lpmode not in ["Off", "On"]: - print(f"Invalid low-power mode '{lpmode}' for port '{port}'") + logging.error("Invalid low-power mode {} for port {}".format(lpmode, port)) return False return True diff --git a/tests/platform_tests/test_advanced_reboot.py b/tests/platform_tests/test_advanced_reboot.py index c1bd133a62..5fe16dc0d6 100644 --- a/tests/platform_tests/test_advanced_reboot.py +++ b/tests/platform_tests/test_advanced_reboot.py @@ -6,14 +6,13 @@ from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 from tests.common.fixtures.duthost_utils import backup_and_restore_config_db # noqa F401 from tests.common.fixtures.advanced_reboot import get_advanced_reboot # noqa F401 -from tests.platform_tests.verify_dut_health import verify_dut_health # noqa F401 from tests.platform_tests.verify_dut_health import add_fail_step_to_reboot # noqa F401 -from tests.platform_tests.warmboot_sad_cases import get_sad_case_list, SAD_CASE_LIST +from tests.common.platform.warmboot_sad_cases import get_sad_case_list, SAD_CASE_LIST +from tests.common.platform.device_utils import advanceboot_loganalyzer, verify_dut_health # noqa F401 from tests.common.fixtures.ptfhost_utils import run_icmp_responder, run_garp_service # noqa F401 from tests.common.dualtor.dual_tor_utils import mux_cable_server_ip, show_muxcable_status -from tests.common.dualtor.mux_simulator_control import get_mux_status, check_mux_status, validate_check_result,\ - toggle_all_simulator_ports, toggle_simulator_port_to_upper_tor # noqa F401 +from tests.common.dualtor.mux_simulator_control import get_mux_status, check_mux_status, validate_check_result # noqa F401 from tests.common.dualtor.constants import LOWER_TOR from tests.common.utilities import wait_until @@ -29,9 +28,28 @@ logger = logging.getLogger() +def check_if_ssd(duthost): + try: + output = duthost.command("lsblk -d -o NAME,ROTA") + lines = output['stdout'].strip().split('\n') + for line in lines[1:]: + name, rota = line.split() + if name.startswith('sd') and int(rota) == 0: + return True + return False + except Exception as e: + logger.error(f"Error while checking SSD: {e}") + return False + + @pytest.fixture(scope="module", params=[SINGLE_TOR_MODE, DUAL_TOR_MODE]) -def testing_config(request, tbinfo): +def testing_config(request, duthosts, rand_one_dut_hostname, tbinfo): testing_mode = request.param + duthost = duthosts[rand_one_dut_hostname] + is_ssd = check_if_ssd(duthost) + neighbor_type = request.config.getoption("--neighbor_type") + if duthost.facts['platform'] == 'x86_64-arista_7050cx3_32s' and not is_ssd and neighbor_type == 'eos': + pytest.skip("skip advanced reboot tests on 7050 devices without SSD") if 'dualtor' in tbinfo['topo']['name']: if testing_mode == SINGLE_TOR_MODE: pytest.skip("skip SINGLE_TOR_MODE tests on Dual ToR testbeds") @@ -60,7 +78,8 @@ def pytest_generate_tests(metafunc): # Tetcases to verify normal reboot procedure ### def test_fast_reboot(request, get_advanced_reboot, verify_dut_health, # noqa F811 - advanceboot_loganalyzer, capture_interface_counters): + advanceboot_loganalyzer, # noqa F811 + capture_interface_counters): ''' Fast reboot test case is run using advanced reboot test fixture @@ -74,7 +93,8 @@ def test_fast_reboot(request, get_advanced_reboot, verify_dut_health, def test_fast_reboot_from_other_vendor(duthosts, rand_one_dut_hostname, request, get_advanced_reboot, verify_dut_health, # noqa F811 - advanceboot_loganalyzer, capture_interface_counters): + advanceboot_loganalyzer, # noqa F811 + capture_interface_counters): ''' Fast reboot test from other vendor case is run using advanced reboot test fixture @@ -91,7 +111,8 @@ def test_fast_reboot_from_other_vendor(duthosts, rand_one_dut_hostname, request @pytest.mark.device_type('vs') def test_warm_reboot(request, testing_config, get_advanced_reboot, verify_dut_health, # noqa F811 - duthosts, advanceboot_loganalyzer, capture_interface_counters, + duthosts, advanceboot_loganalyzer, # noqa F811 + capture_interface_counters, toggle_all_simulator_ports, enum_rand_one_per_hwsku_frontend_hostname, # noqa F811 toggle_simulator_port_to_upper_tor): # noqa F811 ''' @@ -114,12 +135,14 @@ def test_warm_reboot(request, testing_config, get_advanced_reboot, verify_dut_he toggle_simulator_port_to_upper_tor(itfs) advancedReboot = get_advanced_reboot(rebootType='warm-reboot', - advanceboot_loganalyzer=advanceboot_loganalyzer) + advanceboot_loganalyzer=advanceboot_loganalyzer # noqa F811 + ) advancedReboot.runRebootTestcase() def test_warm_reboot_mac_jump(request, get_advanced_reboot, verify_dut_health, # noqa F811 - advanceboot_loganalyzer, capture_interface_counters): + advanceboot_loganalyzer, # noqa F811 + capture_interface_counters): ''' Warm reboot testcase with one MAC address (00-06-07-08-09-0A) jumping from all VLAN ports. diff --git a/tests/platform_tests/test_cont_warm_reboot.py b/tests/platform_tests/test_cont_warm_reboot.py index b4341f91a7..c08d85c0a9 100644 --- a/tests/platform_tests/test_cont_warm_reboot.py +++ b/tests/platform_tests/test_cont_warm_reboot.py @@ -14,9 +14,10 @@ from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 -from tests.platform_tests.verify_dut_health import RebootHealthError,\ +from tests.common.platform.device_utils import RebootHealthError,\ check_services, check_interfaces_and_transceivers, check_neighbors,\ - verify_no_coredumps, handle_test_error, wait_until_uptime, get_test_report + verify_no_coredumps, handle_test_error +from tests.platform_tests.verify_dut_health import wait_until_uptime, get_test_report pytestmark = [ pytest.mark.disable_loganalyzer, diff --git a/tests/platform_tests/test_intf_fec.py b/tests/platform_tests/test_intf_fec.py index 15b386a9cd..4253d3c428 100644 --- a/tests/platform_tests/test_intf_fec.py +++ b/tests/platform_tests/test_intf_fec.py @@ -19,6 +19,14 @@ ] +@pytest.fixture(autouse=True) +def is_supported_platform(duthost): + if any(platform in duthost.facts['platform'] for platform in SUPPORTED_PLATFORMS): + skip_release(duthost, ["201811", "201911", "202012", "202205", "202211", "202305"]) + else: + pytest.skip("DUT has platform {}, test is not supported".format(duthost.facts['platform'])) + + def test_verify_fec_oper_mode(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_frontend_asic_index, conn_graph_facts): """ @@ -27,12 +35,6 @@ def test_verify_fec_oper_mode(duthosts, enum_rand_one_per_hwsku_frontend_hostnam """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - if any(platform in duthost.facts['platform'] for platform in SUPPORTED_PLATFORMS): - # Not supported on 202305 and older releases - skip_release(duthost, ["201811", "201911", "202012", "202205", "202211", "202305"]) - else: - pytest.skip("DUT has platform {}, test is not supported".format(duthost.facts['platform'])) - logging.info("Get output of '{}'".format("show interface status")) intf_status = duthost.show_and_parse("show interface status") @@ -57,16 +59,10 @@ def test_config_fec_oper_mode(duthosts, enum_rand_one_per_hwsku_frontend_hostnam enum_frontend_asic_index, conn_graph_facts): """ @Summary: Configure the FEC operational mode for all the interfaces, then check - FEC operational mode is retored to default FEC mode + FEC operational mode is restored to 'rs' FEC mode """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - if any(platform in duthost.facts['platform'] for platform in SUPPORTED_PLATFORMS): - # Not supported on 202305 and older releases - skip_release(duthost, ["201811", "201911", "202012", "202205", "202211", "202305"]) - else: - pytest.skip("DUT has platform {}, test is not supported".format(duthost.facts['platform'])) - logging.info("Get output of '{}'".format("show interface status")) intf_status = duthost.show_and_parse("show interface status") @@ -76,8 +72,9 @@ def test_config_fec_oper_mode(duthosts, enum_rand_one_per_hwsku_frontend_hostnam if sfp_presence: presence = sfp_presence[0].get('presence', '').lower() oper = intf.get('oper', '').lower() + speed = intf.get('speed', '') - if presence == "not present" or oper != "up": + if presence == "not present" or oper != "up" or speed not in SUPPORTED_SPEEDS: continue config_status = duthost.command("sudo config interface fec {} rs" @@ -91,3 +88,62 @@ def test_config_fec_oper_mode(duthosts, enum_rand_one_per_hwsku_frontend_hostnam if not (fec == "rs"): pytest.fail("FEC status is not restored for interface {}".format(intf['interface'])) + + +def get_interface_speed(duthost, interface_name): + """ + Get the speed of a specific interface on the DUT. + + :param duthost: The DUT host object. + :param interface_name: The name of the interface. + :return: The speed of the interface as a string. + """ + logging.info(f"Getting speed for interface {interface_name}") + intf_status = duthost.show_and_parse("show interfaces status {}".format(interface_name)) + + speed = intf_status[0].get('speed') + logging.info(f"Interface {interface_name} has speed {speed}") + return speed + + pytest.fail(f"Interface {interface_name} not found") + + +def test_verify_fec_stats_counters(duthosts, enum_rand_one_per_hwsku_frontend_hostname, + enum_frontend_asic_index, conn_graph_facts): + """ + @Summary: Verify the FEC stats counters are valid + Also, check for any uncorrectable FEC errors + """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + + logging.info("Get output of 'show interfaces counters fec-stats'") + intf_status = duthost.show_and_parse("show interfaces counters fec-stats") + + for intf in intf_status: + intf_name = intf['iface'] + speed = get_interface_speed(duthost, intf_name) + if speed not in SUPPORTED_SPEEDS: + continue + + fec_corr = intf.get('fec_corr', '').lower() + fec_uncorr = intf.get('fec_uncorr', '').lower() + fec_symbol_err = intf.get('fec_symbol_err', '').lower() + # Check if fec_corr, fec_uncorr, and fec_symbol_err are valid integers + try: + fec_corr_int = int(fec_corr) + fec_uncorr_int = int(fec_uncorr) + fec_symbol_err_int = int(fec_symbol_err) + except ValueError: + pytest.fail("FEC stat counters are not valid integers for interface {}, \ + fec_corr: {} fec_uncorr: {} fec_symbol_err: {}" + .format(intf_name, fec_corr, fec_uncorr, fec_symbol_err)) + + # Check for non-zero FEC uncorrectable errors + if fec_uncorr_int > 0: + pytest.fail("FEC uncorrectable errors are non-zero for interface {}: {}" + .format(intf_name, fec_uncorr_int)) + + # Check for valid FEC correctable codeword errors > FEC symbol errors + if fec_symbol_err_int > fec_corr_int: + pytest.fail("FEC symbol errors:{} are higher than FEC correctable errors:{} for interface {}" + .format(intf_name, fec_symbol_err_int, fec_corr_int)) diff --git a/tests/platform_tests/test_kdump.py b/tests/platform_tests/test_kdump.py index 3463944631..dc0b8291c5 100644 --- a/tests/platform_tests/test_kdump.py +++ b/tests/platform_tests/test_kdump.py @@ -64,10 +64,8 @@ def test_kernel_panic(self, duthosts, enum_rand_one_per_hwsku_hostname, localhos if "Enabled" not in out["stdout"]: pytest.skip('DUT {}: Skip test since kdump is not enabled'.format(hostname)) - reboot(duthost, localhost, reboot_type=REBOOT_TYPE_KERNEL_PANIC) + reboot(duthost, localhost, reboot_type=REBOOT_TYPE_KERNEL_PANIC, safe_reboot=True) - # Wait until all critical processes are healthy. - wait_critical_processes(duthost) check_interfaces_and_services(duthost, conn_graph_facts["device_conn"][hostname], xcvr_skip_list, reboot_type=REBOOT_TYPE_KERNEL_PANIC) self.wait_lc_healthy_if_sup(duthost, duthosts, localhost, conn_graph_facts, xcvr_skip_list) diff --git a/tests/platform_tests/test_link_down.py b/tests/platform_tests/test_link_down.py index dca759ee7f..75426c219e 100644 --- a/tests/platform_tests/test_link_down.py +++ b/tests/platform_tests/test_link_down.py @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) pytestmark = [ - pytest.mark.topology('t2'), + pytest.mark.topology('t0', 't1', 't2'), pytest.mark.disable_loganalyzer, ] @@ -210,7 +210,7 @@ def test_link_status_on_host_reboot(duthosts, localhost, enum_rand_one_per_hwsku # Before test, check all interfaces and services are up check_interfaces_and_services( - duthost, conn_graph_facts["device_conn"][hostname], xcvr_skip_list) + duthost, conn_graph_facts.get("device_conn", {}).get("hostname", {}), xcvr_skip_list) dut_ports = single_dut_and_ports(duthost) fanouts_and_ports = fanout_hosts_and_ports(fanouthosts, dut_ports) @@ -238,7 +238,7 @@ def test_link_status_on_host_reboot(duthosts, localhost, enum_rand_one_per_hwsku # After test, check all interfaces and services are up check_interfaces_and_services( - duthost, conn_graph_facts["device_conn"][hostname], xcvr_skip_list) + duthost, conn_graph_facts.get("device_conn", {}).get("hostname", {}), xcvr_skip_list) # Also make sure fanout hosts' links are up link_status_on_host(fanouts_and_ports) diff --git a/tests/platform_tests/test_platform_info.py b/tests/platform_tests/test_platform_info.py index 48a867c041..c4a8c23e5f 100644 --- a/tests/platform_tests/test_platform_info.py +++ b/tests/platform_tests/test_platform_info.py @@ -14,8 +14,8 @@ from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer from tests.common.utilities import wait_until, get_sup_node_or_random_node from tests.common.platform.device_utils import get_dut_psu_line_pattern -from .thermal_control_test_helper import ThermalPolicyFileContext,\ - check_cli_output_with_mocker, restart_thermal_control_daemon, check_thermal_algorithm_status,\ +from tests.common.helpers.thermal_control_test_helper import ThermalPolicyFileContext,\ + check_cli_output_with_mocker, restart_thermal_control_daemon, check_thermal_algorithm_status, \ mocker_factory, disable_thermal_policy # noqa F401 pytestmark = [ @@ -27,6 +27,8 @@ CMD_PLATFORM_FANSTATUS = "show platform fan" CMD_PLATFORM_TEMPER = "show platform temperature" +PDU_WAIT_TIME = 20 + THERMAL_CONTROL_TEST_WAIT_TIME = 65 THERMAL_CONTROL_TEST_CHECK_INTERVAL = 5 @@ -173,7 +175,11 @@ def get_healthy_psu_num(duthost): """ PSUUTIL_CMD = "sudo psuutil status" healthy_psus = 0 - psuutil_status_output = duthost.command(PSUUTIL_CMD) + psuutil_status_output = duthost.command(PSUUTIL_CMD, module_ignore_errors=True) + # For kvm testbed, we will get expected Error code `ERROR_CHASSIS_LOAD = 2` here. + if duthost.facts["asic_type"] == "vs" and psuutil_status_output['rc'] == 2: + return + assert psuutil_status_output["rc"] == 0, "Run command '{}' failed".format(PSUUTIL_CMD) psus_status = psuutil_status_output["stdout_lines"][2:] for iter in psus_status: @@ -253,8 +259,11 @@ def test_turn_on_off_psu_and_check_psustatus(duthosts, psu_line_pattern = get_dut_psu_line_pattern(duthost) psu_num = get_healthy_psu_num(duthost) - pytest_require( - psu_num >= 2, "At least 2 PSUs required for rest of the testing in this case") + # For kvm testbed, psu_num will return None + # Only physical testbeds need to check the psu number + if psu_num: + pytest_require( + psu_num >= 2, "At least 2 PSUs required for rest of the testing in this case") logging.info("Create PSU controller for testing") pdu_ctrl = get_pdu_controller(duthost) @@ -289,7 +298,7 @@ def test_turn_on_off_psu_and_check_psustatus(duthosts, logging.info("Turn off outlet {}".format(outlet)) pdu_ctrl.turn_off_outlet(outlet) - time.sleep(10) + time.sleep(PDU_WAIT_TIME) cli_psu_status = duthost.command(CMD_PLATFORM_PSUSTATUS) for line in cli_psu_status["stdout_lines"][2:]: @@ -303,7 +312,7 @@ def test_turn_on_off_psu_and_check_psustatus(duthosts, logging.info("Turn on outlet {}".format(outlet)) pdu_ctrl.turn_on_outlet(outlet) - time.sleep(10) + time.sleep(PDU_WAIT_TIME) cli_psu_status = duthost.command(CMD_PLATFORM_PSUSTATUS) for line in cli_psu_status["stdout_lines"][2:]: @@ -398,6 +407,10 @@ def check_thermal_control_load_invalid_file(duthost, file_name): loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='thermal_control') loganalyzer.expect_regex = [LOG_EXPECT_POLICY_FILE_INVALID] + # For kvm testbed, we will not restart the deamon `thermal` + # So we will not get the syslog as expected. + if duthost.facts["asic_type"] == "vs": + return with loganalyzer: with ThermalPolicyFileContext(duthost, file_name): restart_thermal_control_daemon(duthost) diff --git a/tests/platform_tests/test_reboot.py b/tests/platform_tests/test_reboot.py index 55b7f548cf..a61ef769c9 100644 --- a/tests/platform_tests/test_reboot.py +++ b/tests/platform_tests/test_reboot.py @@ -54,6 +54,8 @@ def teardown_module(duthosts, enum_rand_one_per_hwsku_hostname, "Tearing down: to make sure all the critical services, interfaces and transceivers are good") interfaces = conn_graph_facts.get("device_conn", {}).get(duthost.hostname, {}) wait_for_startup(duthost, localhost, delay=10, timeout=300) + if duthost.facts['hwsku'] in {"Nokia-M0-7215", "Nokia-7215"}: + wait_critical_processes(duthost) check_critical_processes(duthost, watch_secs=10) check_interfaces_and_services(duthost, interfaces, xcvr_skip_list) if duthost.is_supervisor_node(): diff --git a/tests/platform_tests/test_reload_config.py b/tests/platform_tests/test_reload_config.py index be497e6cc0..1beed69b2d 100644 --- a/tests/platform_tests/test_reload_config.py +++ b/tests/platform_tests/test_reload_config.py @@ -5,7 +5,7 @@ https://github.com/sonic-net/SONiC/blob/master/doc/pmon/sonic_platform_test_plan.md """ import logging - +import time import pytest from tests.common.fixtures.conn_graph_facts import conn_graph_facts # noqa F401 @@ -110,12 +110,50 @@ def check_database_status(duthost): return True +def execute_config_reload_cmd(duthost, timeout=120, check_interval=5): + start_time = time.time() + _, res = duthost.shell("sudo config reload -y", + executable="/bin/bash", + module_ignore_errors=True, + module_async=True) + + while not res.ready(): + elapsed_time = time.time() - start_time + if elapsed_time > timeout: + logging.info("Config reload command did not complete within {} seconds".format(timeout)) + return False, None + + logging.debug("Waiting for config reload command to complete. Elapsed time: {} seconds.".format(elapsed_time)) + time.sleep(check_interval) + + if res.successful(): + result = res.get() + logging.debug("Config reload command result: {}".format(result)) + return True, result + else: + logging.info("Config reload command execution failed: {}".format(res)) + return False, None + + +def check_docker_status(duthost): + containers = duthost.get_all_containers() + for container in containers: + if not duthost.is_service_fully_started(container): + return False + return True + + def test_reload_configuration_checks(duthosts, enum_rand_one_per_hwsku_hostname, delayed_services, localhost, conn_graph_facts, xcvr_skip_list): # noqa F811 """ @summary: This test case is to test various system checks in config reload """ duthost = duthosts[enum_rand_one_per_hwsku_hostname] + hwsku = duthost.facts["hwsku"] + + config_reload_timeout = 120 + if hwsku in ["Nokia-M0-7215", "Nokia-7215"]: + config_reload_timeout = 180 if not config_force_option_supported(duthost): return @@ -129,27 +167,30 @@ def test_reload_configuration_checks(duthosts, enum_rand_one_per_hwsku_hostname, wait_until(360, 1, 0, check_database_status, duthost) logging.info("Reload configuration check") - out = duthost.shell("sudo config reload -y", - executable="/bin/bash", module_ignore_errors=True) + result, out = execute_config_reload_cmd(duthost, config_reload_timeout) # config reload command shouldn't work immediately after system reboot - assert "Retry later" in out['stdout'] - + assert result and "Retry later" in out['stdout'] assert wait_until(300, 20, 0, config_system_checks_passed, duthost, delayed_services) + # Check if all containers have started + assert wait_until(300, 10, 0, check_docker_status, duthost) + + # To ensure the system is stable enough, wait for another 30s + time.sleep(30) + # After the system checks succeed the config reload command should not throw error - out = duthost.shell("sudo config reload -y", - executable="/bin/bash", module_ignore_errors=True) - assert "Retry later" not in out['stdout'] + result, out = execute_config_reload_cmd(duthost, config_reload_timeout) + assert result and "Retry later" not in out['stdout'] # Immediately after one config reload command, another shouldn't execute and wait for system checks logging.info("Checking config reload after system is up") # Check if all database containers have started wait_until(60, 1, 0, check_database_status, duthost) - out = duthost.shell("sudo config reload -y", - executable="/bin/bash", module_ignore_errors=True) - assert "Retry later" in out['stdout'] + result, out = execute_config_reload_cmd(duthost, config_reload_timeout) + assert result and "Retry later" in out['stdout'] assert wait_until(300, 20, 0, config_system_checks_passed, duthost, delayed_services) - + # Wait untill all critical processes come up so that it doesnt interfere with swss stop job + wait_critical_processes(duthost) logging.info("Stopping swss docker and checking config reload") if duthost.is_multi_asic: for asic in duthost.asics: @@ -158,9 +199,8 @@ def test_reload_configuration_checks(duthosts, enum_rand_one_per_hwsku_hostname, duthost.shell("sudo service swss stop") # Without swss running config reload option should not proceed - out = duthost.shell("sudo config reload -y", - executable="/bin/bash", module_ignore_errors=True) - assert "Retry later" in out['stdout'] + result, out = execute_config_reload_cmd(duthost, config_reload_timeout) + assert result and "Retry later" in out['stdout'] # However with force option config reload should proceed logging.info("Performing force config reload") diff --git a/tests/platform_tests/test_secure_upgrade.py b/tests/platform_tests/test_secure_upgrade.py index fe91fa4b50..0a9c72eb69 100644 --- a/tests/platform_tests/test_secure_upgrade.py +++ b/tests/platform_tests/test_secure_upgrade.py @@ -16,7 +16,7 @@ import re from tests.common.errors import RunAnsibleModuleFail from tests.common.helpers.assertions import pytest_assert -from tests.upgrade_path.upgrade_helpers import install_sonic +from tests.common.helpers.upgrade_helpers import install_sonic pytestmark = [ pytest.mark.topology('any'), @@ -48,6 +48,10 @@ def non_secure_image_path(request): :return: given non secure image path ''' non_secure_img_path = request.config.getoption('target_image_list') + + if not non_secure_img_path: + pytest.skip("Skip test case since parameter '--target_image_list' is not specified") + return str(non_secure_img_path) diff --git a/tests/platform_tests/test_sensors.py b/tests/platform_tests/test_sensors.py index d320d6631a..408b1cfd16 100644 --- a/tests/platform_tests/test_sensors.py +++ b/tests/platform_tests/test_sensors.py @@ -3,10 +3,11 @@ import os import pytest import yaml - from tests.common.helpers.assertions import pytest_assert from tests.common import mellanox_data +from tests.platform_tests.sensors_utils.psu_sensor_utils import SensorHelper + pytestmark = [ pytest.mark.topology('any') ] @@ -18,17 +19,56 @@ def to_json(obj): return json.dumps(obj, indent=4) +def to_yaml(obj): + return yaml.dump(obj, indent=4) + + @pytest.fixture(scope='module') def sensors_data(): - sensors_data_file_fullpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), SENSORS_DATA_FILE) - return yaml.safe_load(open(sensors_data_file_fullpath).read()) + """ + Parses SENSORS_DATA_FILE yaml. + """ + sensors_data_file_full_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), SENSORS_DATA_FILE) + return yaml.safe_load(open(sensors_data_file_full_path).read()) + + +def update_sensors_checks(duthost, sensors_checks, hardware_version): + """ + This function will update the update_sensors_checks dynamically according to the sensors of the PSUs installed on + the dut, specified in psu-sensors-data.yml. In order to do that, we first remove all PSU related sensors from + sensors_checks variable and then add the correct ones from psu-sensors-data.yml. + @param duthost: duthost fixture + @param sensors_checks: the sensors data fetched from sensors-sku-data.yml file + @param hardware_version: hardware version as retrieved from mellanox_data.get_hardware_version + """ + sensor_helper = SensorHelper(duthost) + if sensor_helper.platform_supports_dynamic_psu(): + # Remove sensors of PSUs that aren't installed on the dut + missing_psu_indexes = sensor_helper.get_missing_psus() + if missing_psu_indexes: + sensor_helper.remove_psu_checks(sensors_checks, missing_psu_indexes) + + psu_models_to_replace = sensor_helper.get_psu_index_model_dict() + # We only replace psus that are covered by psu-sensors-data.yml + if psu_models_to_replace: + logging.info(f"Fetching PSU sensors for PSUS: {psu_models_to_replace}\n") + if sensor_helper.get_uncovered_psus(): + logging.warning(f"Unsupported PSUs (regardless of fan direction) in psu_data.yml: " + f"{sensor_helper.get_uncovered_psus()}\n") + + sensor_helper.remove_psu_checks(sensors_checks, set(psu_models_to_replace.keys())) + + sensor_helper.update_psu_sensors(sensors_checks, psu_models_to_replace, hardware_version) + else: + logging.warning(f"PSU sensors not covered by psu_data.yml. Unsupported PSUs" + f" (regardless of fan direction): {sensor_helper.get_uncovered_psus()}\n") def test_sensors(duthosts, rand_one_dut_hostname, sensors_data): duthost = duthosts[rand_one_dut_hostname] # Get platform name platform = duthost.facts['platform'] - + hardware_version = "" if mellanox_data.is_mellanox_device(duthost): hardware_version = mellanox_data.get_hardware_version(duthost, platform) if hardware_version: @@ -40,6 +80,7 @@ def test_sensors(duthosts, rand_one_dut_hostname, sensors_data): if platform not in list(sensors_checks.keys()): pytest.skip("Skip test due to not support check sensors for current platform({})".format(platform)) + update_sensors_checks(duthost, sensors_checks[platform], hardware_version) logging.info("Sensor checks:\n{}".format(to_json(sensors_checks[platform]))) # Gather sensor facts diff --git a/tests/platform_tests/test_service_warm_restart.py b/tests/platform_tests/test_service_warm_restart.py index f3d7c3547e..e245ea77ea 100644 --- a/tests/platform_tests/test_service_warm_restart.py +++ b/tests/platform_tests/test_service_warm_restart.py @@ -4,8 +4,9 @@ from tests.common.fixtures.advanced_reboot import get_advanced_reboot # noqa F401 from tests.common.helpers.assertions import pytest_require from tests.common.utilities import skip_release -from tests.platform_tests.verify_dut_health import verify_dut_health # noqa F401 +from tests.common.platform.device_utils import verify_dut_health # noqa F401 from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.common.platform.device_utils import advanceboot_loganalyzer # noqa F401 pytestmark = [ pytest.mark.disable_loganalyzer, @@ -93,7 +94,8 @@ def passes_all_checks(service): def test_service_warm_restart(request, duthosts, rand_one_dut_hostname, verify_dut_health, get_advanced_reboot, # noqa F811 - advanceboot_loganalyzer, capture_interface_counters): + advanceboot_loganalyzer, # noqa F811 + capture_interface_counters): duthost = duthosts[rand_one_dut_hostname] candidate_service_list = select_services_to_warmrestart(duthost, request) diff --git a/tests/platform_tests/test_thermal_state_db.py b/tests/platform_tests/test_thermal_state_db.py index 32bf079d83..d8123b3508 100644 --- a/tests/platform_tests/test_thermal_state_db.py +++ b/tests/platform_tests/test_thermal_state_db.py @@ -65,7 +65,7 @@ def test_thermal_state_db(duthosts, enum_rand_one_per_hwsku_hostname, tbinfo): This test case will verify thermal local state db data on each hwsku type in chassis """ duthost = duthosts[enum_rand_one_per_hwsku_hostname] - if duthost.facts['modular_chassis'] == "False": + if not duthost.facts['modular_chassis']: pytest.skip("Test skipped applicable to modular chassis only") num_thermals = get_expected_num_thermals(duthosts, enum_rand_one_per_hwsku_hostname) thermal_out = duthost.command("redis-dump -d 6 -y -k \"TEMP*\"") @@ -84,7 +84,7 @@ def test_thermal_global_state_db(duthosts, enum_supervisor_dut_hostname, tbinfo) Verify data for all sensors from line cards and fabric cards present in global state db """ duthost = duthosts[enum_supervisor_dut_hostname] - if duthost.facts['modular_chassis'] == "False": + if not duthost.facts['modular_chassis']: pytest.skip("Test skipped applicable to modular chassis only") if not duthost.is_supervisor_node(): pytest.skip("Test skipped applicable to supervisor only") diff --git a/tests/platform_tests/verify_dut_health.py b/tests/platform_tests/verify_dut_health.py index 1fd938f2f9..366a927f10 100644 --- a/tests/platform_tests/verify_dut_health.py +++ b/tests/platform_tests/verify_dut_health.py @@ -2,130 +2,11 @@ import pytest import logging import time -import traceback -from tests.common.helpers.assertions import pytest_assert -from tests.common.utilities import wait_until -from tests.common.platform.transceiver_utils import parse_transceiver_info from tests.common.reboot import reboot, REBOOT_TYPE_COLD test_report = dict() -def handle_test_error(health_check): - def _wrapper(*args, **kwargs): - try: - health_check(*args, **kwargs) - except RebootHealthError as err: - # set result to fail - logging.error("Health check {} failed with {}".format( - health_check.__name__, err.message)) - test_report[health_check.__name__] = err.message - return - except Exception as err: - traceback.print_exc() - logging.error("Health check {} failed with unknown error: {}".format( - health_check.__name__, str(err))) - test_report[health_check.__name__] = "Unkown error" - return - # set result to pass - test_report[health_check.__name__] = True - return _wrapper - - -@handle_test_error -def check_services(duthost): - """ - Perform a health check of services - """ - logging.info("Wait until all critical services are fully started") - if not wait_until(330, 30, 0, duthost.critical_services_fully_started): - raise RebootHealthError("dut.critical_services_fully_started is False") - - logging.info("Check critical service status") - for service in duthost.critical_services: - status = duthost.get_service_props(service) - if status["ActiveState"] != "active": - raise RebootHealthError("ActiveState of {} is {}, expected: active".format( - service, status["ActiveState"])) - if status["SubState"] != "running": - raise RebootHealthError( - "SubState of {} is {}, expected: running".format(service, status["SubState"])) - - -@handle_test_error -def check_interfaces_and_transceivers(duthost, request): - """ - Perform a check of transceivers, LAGs and interfaces status - @param dut: The AnsibleHost object of DUT. - @param interfaces: DUT's interfaces defined by minigraph - """ - logging.info("Check if all the interfaces are operational") - check_interfaces = request.getfixturevalue("check_interfaces") - conn_graph_facts = request.getfixturevalue("conn_graph_facts") - results = check_interfaces() - failed = [ - result for result in results if "failed" in result and result["failed"]] - if failed: - raise RebootHealthError( - "Interface check failed, not all interfaces are up. Failed: {}".format(failed)) - - # Skip this step for virtual testbed - KVM testbed has transeivers marked as "Not present" - # and the DB returns an "empty array" for "keys TRANSCEIVER_INFO*" - if duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0': - return - - logging.info( - "Check whether transceiver information of all ports are in redis") - xcvr_info = duthost.command("redis-cli -n 6 keys TRANSCEIVER_INFO*") - parsed_xcvr_info = parse_transceiver_info(xcvr_info["stdout_lines"]) - interfaces = conn_graph_facts["device_conn"][duthost.hostname] - for intf in interfaces: - if intf not in parsed_xcvr_info: - raise RebootHealthError( - "TRANSCEIVER INFO of {} is not found in DB".format(intf)) - - -@handle_test_error -def check_neighbors(duthost, tbinfo): - """ - Perform a BGP neighborship check. - """ - logging.info("Check BGP neighbors status. Expected state - established") - bgp_facts = duthost.bgp_facts()['ansible_facts'] - mg_facts = duthost.get_extended_minigraph_facts(tbinfo) - - for value in list(bgp_facts['bgp_neighbors'].values()): - # Verify bgp sessions are established - if value['state'] != 'established': - raise RebootHealthError("BGP session not established") - # Verify locat ASNs in bgp sessions - if (value['local AS'] != mg_facts['minigraph_bgp_asn']): - raise RebootHealthError("Local ASNs not found in BGP session.\ - Minigraph: {}. Found {}".format(value['local AS'], mg_facts['minigraph_bgp_asn'])) - for v in mg_facts['minigraph_bgp']: - # Compare the bgp neighbors name with minigraph bgp neigbhors name - if (v['name'] != bgp_facts['bgp_neighbors'][v['addr'].lower()]['description']): - raise RebootHealthError("BGP neighbor's name does not match minigraph.\ - Minigraph: {}. Found {}".format(v['name'], - bgp_facts['bgp_neighbors'][v['addr'].lower()]['description'])) - # Compare the bgp neighbors ASN with minigraph - if (v['asn'] != bgp_facts['bgp_neighbors'][v['addr'].lower()]['remote AS']): - raise RebootHealthError("BGP neighbor's ASN does not match minigraph.\ - Minigraph: {}. Found {}".format(v['asn'], bgp_facts['bgp_neighbors'][v['addr'].lower()]['remote AS'])) - - -@handle_test_error -def verify_no_coredumps(duthost, pre_existing_cores): - if "20191130" in duthost.os_version: - coredumps_count = duthost.shell( - 'ls /var/core/ | grep -v python | wc -l')['stdout'] - else: - coredumps_count = duthost.shell('ls /var/core/ | wc -l')['stdout'] - if int(coredumps_count) > int(pre_existing_cores): - raise RebootHealthError("Core dumps found. Expected: {} Found: {}".format(pre_existing_cores, - coredumps_count)) - - def wait_until_uptime(duthost, post_reboot_delay): logging.info("Wait until DUT uptime reaches {}s".format(post_reboot_delay)) while duthost.get_uptime().total_seconds() < post_reboot_delay: @@ -139,34 +20,6 @@ def get_test_report(): return result -@pytest.fixture -def verify_dut_health(request, duthosts, rand_one_dut_hostname, tbinfo): - global test_report - test_report = {} - duthost = duthosts[rand_one_dut_hostname] - check_services(duthost) - check_interfaces_and_transceivers(duthost, request) - check_neighbors(duthost, tbinfo) - if "20191130" in duthost.os_version: - pre_existing_cores = duthost.shell( - 'ls /var/core/ | grep -v python | wc -l')['stdout'] - else: - pre_existing_cores = duthost.shell('ls /var/core/ | wc -l')['stdout'] - check_all = all([check is True for check in list(test_report.values())]) - pytest_assert(check_all, "DUT not ready for test. Health check failed before reboot: {}".format(test_report)) - - yield - - test_report = {} - check_services(duthost) - check_interfaces_and_transceivers(duthost, request) - check_neighbors(duthost, tbinfo) - verify_no_coredumps(duthost, pre_existing_cores) - check_all = all([check is True for check in list(test_report.values())]) - pytest_assert(check_all, "Health check failed after reboot: {}" - .format(test_report)) - - @pytest.fixture def add_fail_step_to_reboot(localhost, duthosts, rand_one_dut_hostname): duthost = duthosts[rand_one_dut_hostname] @@ -201,9 +54,3 @@ def add_exit_to_script(reboot_type): duthost.shell(replace_cmd) # cold reboot DUT to restore any bad state caused by negative test reboot(duthost, localhost, reboot_type=REBOOT_TYPE_COLD) - - -class RebootHealthError(Exception): - def __init__(self, message): - self.message = message - super(RebootHealthError, self).__init__(message) diff --git a/tests/process_monitoring/test_critical_process_monitoring.py b/tests/process_monitoring/test_critical_process_monitoring.py index dc7774abe7..3be0b3191d 100755 --- a/tests/process_monitoring/test_critical_process_monitoring.py +++ b/tests/process_monitoring/test_critical_process_monitoring.py @@ -486,6 +486,10 @@ def ensure_process_is_running(duthost, container_name, critical_process): Returns: None. """ + if critical_process in ["dhcp6relay", "dhcprelayd"]: + # For dhcp-relay container, the process name in supervisord started 'dhcp-relay: + the process name' + critical_process = "dhcp-relay" + ":" + critical_process + logger.info("Checking whether process '{}' in container '{}' is running..." .format(critical_process, container_name)) program_status, program_pid = get_program_info(duthost, container_name, critical_process) diff --git a/tests/ptf_runner.py b/tests/ptf_runner.py index 24c17fc80a..7fc6e5a247 100644 --- a/tests/ptf_runner.py +++ b/tests/ptf_runner.py @@ -1,9 +1,13 @@ +import ast +import pathlib import pipes import traceback import logging import allure import json from datetime import datetime +import os +import six logger = logging.getLogger(__name__) @@ -46,29 +50,101 @@ def get_dut_type(host): return "Unknown" +def get_ptf_image_type(host): + """ + The function queries the PTF image to determine + if the image is of type 'mixed' or 'py3only' + """ + pyvenv = host.stat(path="/root/env-python3/pyvenv.cfg") + if pyvenv["stat"]["exists"]: + return "mixed" + return "py3only" + + +def get_test_path(testdir, testname): + """ + Returns two values + - first: the complete path of the test based on testdir and testname. + - second: True if file is in 'py3' False otherwise + Raises FileNotFoundError if file is not found + """ + curr_path = os.path.dirname(os.path.abspath(__file__)) + base_path = pathlib.Path(curr_path).joinpath('..').joinpath('ansible/roles/test/files').joinpath(testdir) + idx = testname.find('.') + test_fname = testname + '.py' if idx == -1 else testname[:idx] + '.py' + chk_path = base_path.joinpath('py3').joinpath(test_fname) + if chk_path.exists(): + return chk_path, True + chk_path = base_path.joinpath(test_fname) + if chk_path.exists(): + return chk_path, False + raise FileNotFoundError("Testdir: {} Testname: {} File: {} not found".format(testdir, testname, chk_path)) + + +def is_py3_compat(test_fpath): + """ + Returns True if the test can be run in a Python 3 environment + False otherwise. + """ + if six.PY2: + raise Exception("must run in a Python 3 runtime") + with open(test_fpath, 'rb') as f: + code = f.read() + try: + ast.parse(code) + except SyntaxError: + return False + return True + # shouldn't get here + return False + + def ptf_runner(host, testdir, testname, platform_dir=None, params={}, platform="remote", qlen=0, relax=True, debug_level="info", socket_recv_size=None, log_file=None, device_sockets=[], timeout=0, custom_options="", - module_ignore_errors=False, is_python3=False, async_mode=False): - # Call virtual env ptf for migrated py3 scripts. - # ptf will load all scripts under ptftests, it will throw error for py2 scripts. - # So move migrated scripts to seperated py3 folder avoid impacting py2 scripts. + module_ignore_errors=False, is_python3=None, async_mode=False, pdb=False): + dut_type = get_dut_type(host) if dut_type == "kvm" and params.get("kvm_support", True) is False: logger.info("Skip test case {} for not support on KVM DUT".format(testname)) return True - if is_python3: - path_exists = host.stat(path="/root/env-python3/bin/ptf") - if path_exists["stat"]["exists"]: - cmd = "/root/env-python3/bin/ptf --test-dir {} {}".format(testdir + '/py3', testname) + cmd = "" + ptf_img_type = get_ptf_image_type(host) + logger.info('PTF image type: {}'.format(ptf_img_type)) + test_fpath, in_py3 = get_test_path(testdir, testname) + logger.info('Test file path {}, in py3: {}'.format(test_fpath, in_py3)) + is_python3 = is_py3_compat(test_fpath) + + # The logic below automatically chooses the PTF binary to execute a test script + # based on the container type "mixed" vs. "py3only". + # + # For "mixed" type PTF image the global environment has Python 2 and Python 2 compatible + # ptf binary. Python 3 is part of a virtual environment under "/root/env-python3". All + # packages and Python 3 compatible ptf binary is in the virtual environment. + # + # For "py3only" type PTF image the global environment has Python 3 only in the global + # environment. Python 2 does not exist on this image and attempt to execute any + # Python 2 PTF tests raises an exception. + + ptf_cmd = None + if ptf_img_type == "mixed": + if is_python3: + ptf_cmd = '/root/env-python3/bin/ptf' else: - error_msg = "Virtual environment for Python3 /root/env-python3/bin/ptf doesn't exist.\n" \ - "Please check and update docker-ptf image, make sure to use the correct one." - logger.error("Exception caught while executing case: {}. Error message: {}".format(testname, error_msg)) - raise Exception(error_msg) + ptf_cmd = '/usr/bin/ptf' + else: + if is_python3: + ptf_cmd = '/usr/local/bin/ptf' + else: + err_msg = 'cannot run Python 2 test in a Python 3 only {} {}'.format(testdir, testname) + raise Exception(err_msg) + + if in_py3: + tdir = pathlib.Path(testdir).joinpath('py3') + cmd = "{} --test-dir {} {}".format(ptf_cmd, tdir, testname) else: - cmd = "ptf --test-dir {} {}".format(testdir, testname) + cmd = "{} --test-dir {} {}".format(ptf_cmd, testdir, testname) if platform_dir: cmd += " --platform-dir {}".format(platform_dir) @@ -98,7 +174,7 @@ def ptf_runner(host, testdir, testname, platform_dir=None, params={}, if device_sockets: cmd += " ".join(map(" --device-socket {}".format, device_sockets)) - if timeout: + if timeout and not pdb: cmd += " --test-case-timeout {}".format(int(timeout)) if custom_options: @@ -111,6 +187,16 @@ def ptf_runner(host, testdir, testname, platform_dir=None, params={}, host.create_macsec_info() try: + if pdb: + # Write command to file. Use short test name for simpler launch in ptf container. + script_name = "/tmp/" + testname.split(".")[-1] + ".sh" + with open(script_name, 'w') as f: + f.write(cmd) + host.copy(src=script_name, dest="/root/") + print("Run command from ptf: sh {}".format(script_name)) + import pdb + pdb.set_trace() + logger.info('ptf command: {}'.format(cmd)) result = host.shell(cmd, chdir="/root", module_ignore_errors=module_ignore_errors, module_async=async_mode) if not async_mode: if log_file: diff --git a/tests/pytest.ini b/tests/pytest.ini index de9f347e4f..835191a4ee 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -23,6 +23,7 @@ markers: dynamic_config: dynamic_config marker static_config: static_config marker dependency: dependency marker + skip_traffic_test: skip_traffic_test marker log_cli_format: %(asctime)s %(funcNamewithModule)-40.40s L%(lineno)-.4d %(levelname)-7s| %(message)s log_file_format: %(asctime)s %(funcNamewithModule)-40.40s L%(lineno)-.4d %(levelname)-7s| %(message)s diff --git a/tests/qos/files/cisco/qos_param_generator.py b/tests/qos/files/cisco/qos_param_generator.py index c291d4bc49..c1e2e05a3b 100644 --- a/tests/qos/files/cisco/qos_param_generator.py +++ b/tests/qos/files/cisco/qos_param_generator.py @@ -1,4 +1,5 @@ import logging +from tests.qos.qos_sai_base import QosSaiBase logger = logging.getLogger(__name__) @@ -65,12 +66,98 @@ def __init__(self, qos_params, duthost, dutAsic, topo, bufferConfig, portSpeedCa # Tune thresholds with padding for precise testing self.pause_thr = pre_pad_pause + (8 * self.buffer_size) self.drop_thr = pre_pad_drop + (12 * self.buffer_size) + if self.is_deep_buffer: + self.reduced_pause_thr = 10 * (1024 ** 2) * (2 ** dynamic_th) + elif self.is_large_sms: + self.reduced_pause_thr = 3 * (1024 ** 2) + else: + self.reduced_pause_thr = 2.25 * (1024 ** 2) + if dutAsic == "gr": + self.reduced_pause_thr = self.gr_get_hw_thr_buffs(self.reduced_pause_thr + // self.buffer_size) * self.buffer_size self.log("Max pause thr bytes: {}".format(max_pause)) self.log("Attempted pause thr bytes: {}".format(attempted_pause)) self.log("Pre-pad pause thr bytes: {}".format(pre_pad_pause)) self.log("Pause thr bytes: {}".format(self.pause_thr)) self.log("Pre-pad drop thr bytes: {}".format(pre_pad_drop)) self.log("Drop thr bytes: {}".format(self.drop_thr)) + self.log("Reduced pause thr bytes: {}".format(self.reduced_pause_thr)) + self.config_facts = duthost.get_running_config_facts() + # DSCP value for lossy + self.dscp_queue0 = self.get_one_dscp_from_queue(0) + self.dscp_queue1 = self.get_one_dscp_from_queue(1) + # DSCP, queue, weight list + self.dscp_list, self.q_list, self.weight_list = self.get_dscp_q_weight_list() + + def get_one_dscp_from_queue(self, queue): + ''' + Get one dscp value which is mapped to given queue + ''' + dscp_to_tc_map = self.config_facts['DSCP_TO_TC_MAP']['AZURE'] + tc_to_queue_map = self.config_facts['TC_TO_QUEUE_MAP']['AZURE'] + queue = str(queue) + tc = list(tc_to_queue_map.keys())[list(tc_to_queue_map.values()).index(queue)] + dscp = list(dscp_to_tc_map.keys())[list(dscp_to_tc_map.values()).index(tc)] + return int(dscp) + + def get_scheduler_cfg(self): + ''' + Get scheduler configuration + ''' + return self.config_facts['SCHEDULER'] + + def get_queue_cfg(self): + ''' + Get queue configuration of first interface + ''' + queue_cfg = self.config_facts['QUEUE'] + interface = list(queue_cfg.keys())[0] + return queue_cfg[interface] + + def get_queues_from_scheduler(self, scheduler): + ''' + Get queue list which are mapped to given scheduler + ''' + queue_list = [] + for q, value in self.get_queue_cfg().items(): + if scheduler == value['scheduler']: + queue_list.append(int(q)) + return queue_list + + def get_scheduler_from_queue(self, queue): + ''' + Get scheduler for given queue + ''' + return self.get_queue_cfg()[str(queue)]["scheduler"] + + def get_queues_on_same_scheduler(self, queue): + ''' + Get queue list on the same scheduler for given queue + ''' + scheduler = self.get_scheduler_from_queue(queue) + return self.get_queues_from_scheduler(scheduler) + + def get_queue_dscp_weight_dict(self): + q_dscp_weight_dict = {} + scheduler_cfg = self.get_scheduler_cfg() + for q, value in self.get_queue_cfg().items(): + queue = int(q) + scheduler = value['scheduler'] + q_dscp_weight_dict[queue] = {} + q_dscp_weight_dict[queue]["dscp"] = self.get_one_dscp_from_queue(queue) + q_dscp_weight_dict[queue]["weight"] = int(scheduler_cfg[scheduler]["weight"]) + return q_dscp_weight_dict + + def get_dscp_q_weight_list(self): + dscp_list = [] + q_list = [] + weight_list = [] + q_dscp_weight_dict = self.get_queue_dscp_weight_dict() + for queue, value in q_dscp_weight_dict.items(): + q_list.append(queue) + weight_list.append(value["weight"]) + dscp_list.append(value["dscp"]) + return dscp_list, q_list, weight_list def run(self): ''' @@ -92,6 +179,8 @@ def run(self): self.__define_lossless_voq() self.__define_q_watermark_all_ports() self.__define_pg_drop() + self.__define_wrr() + self.__define_wrr_chg() return self.qos_params def gr_get_mantissa_exp(self, thr): @@ -167,7 +256,9 @@ def __define_shared_reservation_size(self): return if self.is_large_sms: if self.is_deep_buffer: - res_1 = {"dscps": [8, 8, 1, 1, 3, 4, 3, 4, 3, 4, 3], + res_1 = {"dscps": [self.dscp_queue0, self.dscp_queue0, + self.dscp_queue1, self.dscp_queue1, + 3, 4, 3, 4, 3, 4, 3], "pgs": [0, 0, 0, 0, 3, 4, 3, 4, 3, 4, 3], "queues": [0, 0, 1, 1, 3, 4, 3, 4, 3, 4, 3], "src_port_i": [0, 1, 0, 1, 0, 0, 1, 1, 2, 2, 4], @@ -246,6 +337,8 @@ def __define_pfc_xon_limit(self): "ecn": 1, "pg": dscp_pg, "pkts_num_trig_pfc": (self.pause_thr // self.buffer_size // packet_buffs) - 1, + "pkts_num_hysteresis": int(((self.pause_thr - self.reduced_pause_thr) + // self.buffer_size // packet_buffs) - 2), "pkts_num_dismiss_pfc": 2, "packet_size": packet_size} self.write_params("xon_{}".format(param_i), params) @@ -267,7 +360,7 @@ def __define_pg_shared_watermark(self): self.write_params("wm_pg_shared_lossless", lossless_params) if self.should_autogen(["wm_pg_shared_lossy"]): lossy_params = common_params.copy() - lossy_params.update({"dscp": 8, + lossy_params.update({"dscp": self.dscp_queue0, "pg": 0, "pkts_num_trig_egr_drp": self.max_depth // self.buffer_size}) self.write_params("wm_pg_shared_lossy", lossy_params) @@ -286,7 +379,7 @@ def __define_buffer_pool_watermark(self): "packet_size": packet_size} self.write_params("wm_buf_pool_lossless", lossless_params) if self.should_autogen(["wm_buf_pool_lossy"]): - lossy_params = {"dscp": 8, + lossy_params = {"dscp": self.dscp_queue0, "ecn": 1, "pg": 0, "queue": 0, @@ -307,7 +400,7 @@ def __define_q_shared_watermark(self): "cell_size": self.buffer_size} self.write_params("wm_q_shared_lossless", lossless_params) if self.should_autogen(["wm_q_shared_lossy"]): - lossy_params = {"dscp": 8, + lossy_params = {"dscp": self.dscp_queue0, "ecn": 1, "queue": 0, "pkts_num_fill_min": 0, @@ -318,7 +411,7 @@ def __define_q_shared_watermark(self): def __define_lossy_queue_voq(self): if self.should_autogen(["lossy_queue_voq_1"]): - params = {"dscp": 8, + params = {"dscp": self.dscp_queue0, "ecn": 1, "pg": 0, "flow_config": self.flow_config, @@ -328,7 +421,7 @@ def __define_lossy_queue_voq(self): "cell_size": self.buffer_size} self.write_params("lossy_queue_voq_1", params) if self.should_autogen(["lossy_queue_voq_2"]): - params = {"dscp": 8, + params = {"dscp": self.dscp_queue0, "ecn": 1, "pg": 0, "flow_config": "shared", @@ -338,18 +431,18 @@ def __define_lossy_queue_voq(self): "cell_size": self.buffer_size} self.write_params("lossy_queue_voq_2", params) if self.should_autogen(["lossy_queue_voq_3"]): - params = {"dscp": 8, + params = {"dscp": self.dscp_queue0, "ecn": 1, "pg": 0, "pkts_num_trig_egr_drp": self.max_depth // self.buffer_size, "pkts_num_margin": 4, - "packet_size": 64, + "packet_size": 1350, "cell_size": self.buffer_size} self.write_params("lossy_queue_voq_3", params) def __define_lossy_queue(self): if self.should_autogen(["lossy_queue_1"]): - params = {"dscp": 8, + params = {"dscp": self.dscp_queue0, "ecn": 1, "pg": 0, "pkts_num_trig_egr_drp": self.max_depth // self.buffer_size, @@ -418,3 +511,41 @@ def __define_pg_drop(self): "pkts_num_margin": margin, "iterations": 100} self.write_params("pg_drop", params) + + def __define_wrr(self): + q_pkt_cnt = [] + pkt_cnt_multiplier = 470/sum(self.weight_list) + for weight in self.weight_list: + q_pkt_cnt.append(int(weight * pkt_cnt_multiplier)) + if self.should_autogen(["wrr"]): + params = {"ecn": 1, + "dscp_list": self.dscp_list, + "q_list": self.q_list, + "q_pkt_cnt": q_pkt_cnt, + "limit": 80} + self.write_params("wrr", params) + + def __define_wrr_chg(self): + q_pkt_cnt = [] + weight_list = [] + lossy_queues_chg = self.get_queues_on_same_scheduler(QosSaiBase.TARGET_LOSSY_QUEUE_SCHED) + lossless_queues_chg = self.get_queues_on_same_scheduler(QosSaiBase.TARGET_LOSSLESS_QUEUE_SCHED) + for queue, weight in zip(self.q_list, self.weight_list): + if queue in lossy_queues_chg: + weight_list.append(8) + elif queue in lossless_queues_chg: + weight_list.append(30) + else: + weight_list.append(weight) + pkt_cnt_multiplier = 470/sum(weight_list) + for weight in weight_list: + q_pkt_cnt.append(int(weight * pkt_cnt_multiplier)) + if self.should_autogen(["wrr_chg"]): + params = {"ecn": 1, + "dscp_list": self.dscp_list, + "q_list": self.q_list, + "q_pkt_cnt": q_pkt_cnt, + "limit": 80, + "lossy_weight": 8, + "lossless_weight": 30} + self.write_params("wrr_chg", params) diff --git a/tests/qos/files/dynamic_buffer_param.json b/tests/qos/files/dynamic_buffer_param.json index c527442475..486a929470 100644 --- a/tests/qos/files/dynamic_buffer_param.json +++ b/tests/qos/files/dynamic_buffer_param.json @@ -51,6 +51,7 @@ "x86_64-mlnx_msn4700-r0": "400000", "x86_64-mlnx_msn4700_simx-r0": "400000", "x86_64-nvidia_sn4280_simx-r0": "400000", + "x86_64-nvidia_sn4280-r0": "400000", "x86_64-nvidia_sn4800-r0": "400000", "x86_64-nvidia_sn4800_simx-r0": "400000", "x86_64-nvidia_sn5600-r0": "800000", diff --git a/tests/qos/files/qos_params.gb.yaml b/tests/qos/files/qos_params.gb.yaml index aff59779ce..0a098ddee9 100644 --- a/tests/qos/files/qos_params.gb.yaml +++ b/tests/qos/files/qos_params.gb.yaml @@ -265,30 +265,30 @@ qos_params: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 11522 - pkts_num_trig_ingr_drp: 23044 + pkts_num_trig_pfc: 3202 + pkts_num_trig_ingr_drp: 14724 packet_size: 8156 pkts_num_margin: 20 xoff_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 11522 - pkts_num_trig_ingr_drp: 23044 + pkts_num_trig_pfc: 3202 + pkts_num_trig_ingr_drp: 14724 packet_size: 8156 pkts_num_margin: 20 xon_1: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 11521 + pkts_num_trig_pfc: 3201 pkts_num_dismiss_pfc: 2 packet_size: 8156 xon_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 11521 + pkts_num_trig_pfc: 3201 pkts_num_dismiss_pfc: 2 packet_size: 8156 lossy_queue_1: @@ -303,7 +303,7 @@ qos_params: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 23043 + pkts_num_trig_pfc: 14723 pkts_num_fill_min: 0 pkts_num_margin: 4 packet_size: 8156 @@ -323,7 +323,7 @@ qos_params: pg: 3 queue: 3 pkts_num_fill_ingr_min: 140 - pkts_num_trig_pfc: 23085 + pkts_num_trig_pfc: 14744 cell_size: 8192 packet_size: 8140 pkts_num_margin: 20 @@ -332,8 +332,8 @@ qos_params: ecn: 1 queue: 3 pkts_num_fill_min: 0 - pkts_num_trig_ingr_drp: 46087 - pkts_num_margin: 26624 + pkts_num_trig_ingr_drp: 29447 + pkts_num_margin: 24576 cell_size: 6144 packet_size: 8156 wm_q_shared_lossy: @@ -421,6 +421,7 @@ qos_params: ecn: 1 pg: 3 pkts_num_trig_pfc: 3415 + pkts_num_hysteresis: 1365 pkts_num_dismiss_pfc: 23 packet_size: 1350 xon_2: @@ -428,6 +429,7 @@ qos_params: ecn: 1 pg: 4 pkts_num_trig_pfc: 3415 + pkts_num_hysteresis: 1365 pkts_num_dismiss_pfc: 23 packet_size: 1350 lossy_queue_1: @@ -544,6 +546,7 @@ qos_params: ecn: 1 pg: 3 pkts_num_trig_pfc: 3038 + pkts_num_hysteresis: 1140 pkts_num_dismiss_pfc: 23 packet_size: 1350 xon_2: @@ -551,6 +554,7 @@ qos_params: ecn: 1 pg: 4 pkts_num_trig_pfc: 3038 + pkts_num_hysteresis: 1140 pkts_num_dismiss_pfc: 23 packet_size: 1350 lossless_voq_1: @@ -731,44 +735,34 @@ qos_params: 400000_120000m: pkts_num_leak_out: 0 pkts_num_egr_mem: 957 - pg_drop: - dscp: 3 - ecn: 1 - pg: 3 - queue: 3 - pkts_num_trig_pfc: 943719 - pkts_num_trig_ingr_drp: 1887437 - pkts_num_margin: 108000 - iterations: 30 - cell_size: 8192 xoff_1: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 11522 - pkts_num_trig_ingr_drp: 23044 + pkts_num_trig_pfc: 3202 + pkts_num_trig_ingr_drp: 14724 packet_size: 8156 pkts_num_margin: 20 xoff_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 11522 - pkts_num_trig_ingr_drp: 23044 + pkts_num_trig_pfc: 3202 + pkts_num_trig_ingr_drp: 14724 packet_size: 8156 pkts_num_margin: 20 xon_1: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 11522 + pkts_num_trig_pfc: 3202 pkts_num_dismiss_pfc: 4 packet_size: 8156 xon_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 11522 + pkts_num_trig_pfc: 3202 pkts_num_dismiss_pfc: 4 packet_size: 8156 lossy_queue_1: @@ -801,7 +795,7 @@ qos_params: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 23043 + pkts_num_trig_pfc: 14723 pkts_num_fill_min: 0 pkts_num_margin: 4 packet_size: 8156 diff --git a/tests/qos/files/qos_params.gr.yaml b/tests/qos/files/qos_params.gr.yaml index cd51ed7aa7..68fe295ba1 100644 --- a/tests/qos/files/qos_params.gr.yaml +++ b/tests/qos/files/qos_params.gr.yaml @@ -1,24 +1,6 @@ qos_params: gr: topo-any: - wm_pg_shared_lossy: - dscp: 8 - ecn: 1 - pg: 0 - pkts_num_trig_egr_drp: 64000 - pkts_num_fill_min: 0 - pkts_num_margin: 4 - packet_size: 1350 - cell_size: 384 - wm_buf_pool_lossy: - dscp: 8 - ecn: 1 - pg: 0 - queue: 0 - pkts_num_trig_egr_drp: 16000 - pkts_num_fill_egr_min: 0 - cell_size: 384 - packet_size: 1350 shared_res_size_1: ecn: 1 packet_size: 1350 @@ -29,27 +11,5 @@ qos_params: packet_size: 1350 cell_size: 384 pkts_num_margin: 1 - wrr_chg: - ecn: 1 - q0_num_of_pkts: 40 - q1_num_of_pkts: 40 - q2_num_of_pkts: 40 - q3_num_of_pkts: 150 - q4_num_of_pkts: 150 - q5_num_of_pkts: 40 - q6_num_of_pkts: 40 - limit: 80 - lossy_weight: 8 - lossless_weight: 30 - wrr: - ecn: 1 - q0_num_of_pkts: 70 - q1_num_of_pkts: 70 - q2_num_of_pkts: 70 - q3_num_of_pkts: 75 - q4_num_of_pkts: 75 - q5_num_of_pkts: 70 - q6_num_of_pkts: 70 - limit: 80 hdrm_pool_wm_multiplier: 1 cell_size: 384 diff --git a/tests/qos/files/qos_params.j2c.yaml b/tests/qos/files/qos_params.j2c.yaml index febd4f1cf4..aa88cee420 100644 --- a/tests/qos/files/qos_params.j2c.yaml +++ b/tests/qos/files/qos_params.j2c.yaml @@ -220,15 +220,15 @@ qos_params: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 35108 - pkts_num_trig_ingr_drp: 38619 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 418781 pkts_num_margin: 100 xoff_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 35108 - pkts_num_trig_ingr_drp: 38619 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 418781 pkts_num_margin: 100 hdrm_pool_size: dscps: [ 3, 4 ] @@ -237,43 +237,44 @@ qos_params: src_port_ids: [ 0, 2, 4, 6, 8, 10, 12, 14, 16 ] dst_port_id: 18 pgs_num: 18 - pkts_num_trig_pfc: 35208 - pkts_num_hdrm_full: 362 - pkts_num_hdrm_partial: 182 + pkts_num_trig_pfc: 415271 + pkts_num_hdrm_full: 3510 + pkts_num_hdrm_partial: 3500 + margin: 5 wm_pg_headroom: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 35108 - pkts_num_trig_ingr_drp: 38619 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 418781 cell_size: 4096 pkts_num_margin: 30 xon_1: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 35108 - pkts_num_dismiss_pfc: 200 + pkts_num_trig_pfc: 415271 + pkts_num_dismiss_pfc: 3243 pkts_num_margin: 150 xon_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 35108 - pkts_num_dismiss_pfc: 200 + pkts_num_trig_pfc: 415271 + pkts_num_dismiss_pfc: 3243 pkts_num_margin: 150 lossy_queue_1: - dscp: 7 + dscp: 8 ecn: 1 - pg: 1 - pkts_num_trig_egr_drp: 2396745 + pg: 0 + pkts_num_trig_egr_drp: 2179900 pkts_num_margin: 200 wm_pg_shared_lossless: dscp: 3 ecn: 1 pg: 3 - pkts_num_fill_min: 51 - pkts_num_trig_pfc: 9874 + pkts_num_fill_min: 0 + pkts_num_trig_pfc: 415271 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -281,8 +282,8 @@ qos_params: dscp: 8 ecn: 1 pg: 0 - pkts_num_fill_min: 51 - pkts_num_trig_egr_drp: 2396745 + pkts_num_fill_min: 0 + pkts_num_trig_egr_drp: 2179900 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -291,7 +292,7 @@ qos_params: ecn: 1 queue: 3 pkts_num_fill_min: 0 - pkts_num_trig_ingr_drp: 38619 + pkts_num_trig_ingr_drp: 418781 cell_size: 4096 wm_buf_pool_lossless: dscp: 3 @@ -308,7 +309,7 @@ qos_params: ecn: 1 queue: 0 pkts_num_fill_min: 0 - pkts_num_trig_egr_drp: 2396745 + pkts_num_trig_egr_drp: 2179900 cell_size: 4096 wm_buf_pool_lossy: dscp: 8 @@ -326,15 +327,15 @@ qos_params: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 37449 - pkts_num_trig_ingr_drp: 216064 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 593885 pkts_num_margin: 100 xoff_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 37449 - pkts_num_trig_ingr_drp: 216064 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 593885 pkts_num_margin: 100 hdrm_pool_size: dscps: [ 3, 4 ] @@ -343,43 +344,44 @@ qos_params: src_port_ids: [ 0, 2, 4, 6, 8, 10, 12, 14, 16 ] dst_port_id: 18 pgs_num: 18 - pkts_num_trig_pfc: 37549 - pkts_num_hdrm_full: 362 - pkts_num_hdrm_partial: 182 + pkts_num_trig_pfc: 415271 + pkts_num_hdrm_full: 176849 + pkts_num_hdrm_partial: 176749 + margin: 100 wm_pg_headroom: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 37449 - pkts_num_trig_ingr_drp: 216064 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 593885 cell_size: 4096 pkts_num_margin: 30 xon_1: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 37449 - pkts_num_dismiss_pfc: 200 + pkts_num_trig_pfc: 415271 + pkts_num_dismiss_pfc: 3245 pkts_num_margin: 150 xon_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 37449 - pkts_num_dismiss_pfc: 200 + pkts_num_trig_pfc: 415271 + pkts_num_dismiss_pfc: 3245 pkts_num_margin: 150 lossy_queue_1: - dscp: 7 + dscp: 8 ecn: 1 - pg: 1 - pkts_num_trig_egr_drp: 2396745 + pg: 0 + pkts_num_trig_egr_drp: 2179900 pkts_num_margin: 200 wm_pg_shared_lossless: dscp: 3 ecn: 1 pg: 3 - pkts_num_fill_min: 51 - pkts_num_trig_pfc: 37449 + pkts_num_fill_min: 0 + pkts_num_trig_pfc: 415271 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -387,8 +389,8 @@ qos_params: dscp: 8 ecn: 1 pg: 0 - pkts_num_fill_min: 51 - pkts_num_trig_egr_drp: 2396745 + pkts_num_fill_min: 0 + pkts_num_trig_egr_drp: 2179900 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -397,7 +399,7 @@ qos_params: ecn: 1 queue: 3 pkts_num_fill_min: 0 - pkts_num_trig_ingr_drp: 216064 + pkts_num_trig_ingr_drp: 593885 cell_size: 4096 wm_buf_pool_lossless: dscp: 3 @@ -414,7 +416,7 @@ qos_params: ecn: 1 queue: 0 pkts_num_fill_min: 0 - pkts_num_trig_egr_drp: 2396745 + pkts_num_trig_egr_drp: 2179900 cell_size: 4096 wm_buf_pool_lossy: dscp: 8 @@ -432,15 +434,15 @@ qos_params: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 37449 - pkts_num_trig_ingr_drp: 750848 + pkts_num_trig_pfc: 657523 + pkts_num_trig_ingr_drp: 1370921 pkts_num_margin: 100 xoff_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 37449 - pkts_num_trig_ingr_drp: 750848 + pkts_num_trig_pfc: 657523 + pkts_num_trig_ingr_drp: 1370921 pkts_num_margin: 100 hdrm_pool_size: dscps: [ 3, 4 ] @@ -449,44 +451,44 @@ qos_params: src_port_ids: [ 0, 2, 4, 6, 8, 10, 12, 14, 16 ] dst_port_id: 18 pgs_num: 18 - pkts_num_trig_pfc: 37549 - pkts_num_hdrm_full: 362 - pkts_num_hdrm_partial: 182 - margin: 1 + pkts_num_trig_pfc: 657523 + pkts_num_hdrm_full: 622850 + pkts_num_hdrm_partial: 622750 + margin: 300 wm_pg_headroom: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 37449 - pkts_num_trig_ingr_drp: 750848 + pkts_num_trig_pfc: 657523 + pkts_num_trig_ingr_drp: 1370921 cell_size: 4096 pkts_num_margin: 30 xon_1: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 37449 - pkts_num_dismiss_pfc: 200 + pkts_num_trig_pfc: 657523 + pkts_num_dismiss_pfc: 12985 pkts_num_margin: 150 xon_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 37449 - pkts_num_dismiss_pfc: 200 + pkts_num_trig_pfc: 657523 + pkts_num_dismiss_pfc: 12985 pkts_num_margin: 150 lossy_queue_1: - dscp: 7 + dscp: 8 ecn: 1 - pg: 1 - pkts_num_trig_egr_drp: 2396745 - pkts_num_margin: 3500 + pg: 0 + pkts_num_trig_egr_drp: 2179900 + pkts_num_margin: 100 wm_pg_shared_lossless: dscp: 3 ecn: 1 pg: 3 - pkts_num_fill_min: 71 - pkts_num_trig_pfc: 37449 + pkts_num_fill_min: 0 + pkts_num_trig_pfc: 657523 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -494,8 +496,8 @@ qos_params: dscp: 8 ecn: 1 pg: 0 - pkts_num_fill_min: 71 - pkts_num_trig_egr_drp: 2396745 + pkts_num_fill_min: 0 + pkts_num_trig_egr_drp: 2179900 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -504,7 +506,7 @@ qos_params: ecn: 1 queue: 3 pkts_num_fill_min: 0 - pkts_num_trig_ingr_drp: 750848 + pkts_num_trig_ingr_drp: 1370921 cell_size: 4096 wm_buf_pool_lossless: dscp: 3 @@ -521,7 +523,7 @@ qos_params: ecn: 1 queue: 0 pkts_num_fill_min: 0 - pkts_num_trig_egr_drp: 2396745 + pkts_num_trig_egr_drp: 2179900 cell_size: 4096 wm_buf_pool_lossy: dscp: 8 diff --git a/tests/qos/files/qos_params.jr2.yaml b/tests/qos/files/qos_params.jr2.yaml index eaf1b70cfd..d9e144b47d 100644 --- a/tests/qos/files/qos_params.jr2.yaml +++ b/tests/qos/files/qos_params.jr2.yaml @@ -114,15 +114,15 @@ qos_params: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 35108 - pkts_num_trig_ingr_drp: 38619 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 418781 pkts_num_margin: 100 xoff_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 35108 - pkts_num_trig_ingr_drp: 38619 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 418781 pkts_num_margin: 100 hdrm_pool_size: dscps: [ 3, 4 ] @@ -131,43 +131,43 @@ qos_params: src_port_ids: [ 0, 2, 4, 6, 8, 10, 12, 14, 16 ] dst_port_id: 18 pgs_num: 18 - pkts_num_trig_pfc: 35208 + pkts_num_trig_pfc: 415271 pkts_num_hdrm_full: 362 pkts_num_hdrm_partial: 182 wm_pg_headroom: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 35108 - pkts_num_trig_ingr_drp: 38619 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 418781 cell_size: 4096 pkts_num_margin: 30 xon_1: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 35108 - pkts_num_dismiss_pfc: 200 + pkts_num_trig_pfc: 415271 + pkts_num_dismiss_pfc: 3243 pkts_num_margin: 150 xon_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 35108 - pkts_num_dismiss_pfc: 200 + pkts_num_trig_pfc: 415271 + pkts_num_dismiss_pfc: 3243 pkts_num_margin: 150 lossy_queue_1: - dscp: 7 + dscp: 8 ecn: 1 - pg: 1 - pkts_num_trig_egr_drp: 2396745 + pg: 0 + pkts_num_trig_egr_drp: 2179900 pkts_num_margin: 200 wm_pg_shared_lossless: dscp: 3 ecn: 1 pg: 3 - pkts_num_fill_min: 51 - pkts_num_trig_pfc: 9874 + pkts_num_fill_min: 0 + pkts_num_trig_pfc: 415271 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -175,8 +175,8 @@ qos_params: dscp: 8 ecn: 1 pg: 0 - pkts_num_fill_min: 51 - pkts_num_trig_egr_drp: 2396745 + pkts_num_fill_min: 0 + pkts_num_trig_egr_drp: 2179900 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -185,7 +185,7 @@ qos_params: ecn: 1 queue: 3 pkts_num_fill_min: 0 - pkts_num_trig_ingr_drp: 38619 + pkts_num_trig_ingr_drp: 418781 cell_size: 4096 wm_buf_pool_lossless: dscp: 3 @@ -202,7 +202,7 @@ qos_params: ecn: 1 queue: 0 pkts_num_fill_min: 0 - pkts_num_trig_egr_drp: 2396745 + pkts_num_trig_egr_drp: 2179900 cell_size: 4096 wm_buf_pool_lossy: dscp: 8 @@ -220,15 +220,15 @@ qos_params: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 37449 - pkts_num_trig_ingr_drp: 216064 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 593885 pkts_num_margin: 100 xoff_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 37449 - pkts_num_trig_ingr_drp: 216064 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 593885 pkts_num_margin: 100 hdrm_pool_size: dscps: [ 3, 4 ] @@ -237,43 +237,43 @@ qos_params: src_port_ids: [ 0, 2, 4, 6, 8, 10, 12, 14, 16 ] dst_port_id: 18 pgs_num: 18 - pkts_num_trig_pfc: 37549 + pkts_num_trig_pfc: 415271 pkts_num_hdrm_full: 362 pkts_num_hdrm_partial: 182 wm_pg_headroom: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 37449 - pkts_num_trig_ingr_drp: 216064 + pkts_num_trig_pfc: 415271 + pkts_num_trig_ingr_drp: 593885 cell_size: 4096 pkts_num_margin: 30 xon_1: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 37449 - pkts_num_dismiss_pfc: 200 + pkts_num_trig_pfc: 415271 + pkts_num_dismiss_pfc: 3245 pkts_num_margin: 150 xon_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 37449 - pkts_num_dismiss_pfc: 200 + pkts_num_trig_pfc: 415271 + pkts_num_dismiss_pfc: 3245 pkts_num_margin: 150 lossy_queue_1: - dscp: 7 + dscp: 8 ecn: 1 - pg: 1 - pkts_num_trig_egr_drp: 2396745 + pg: 0 + pkts_num_trig_egr_drp: 2179900 pkts_num_margin: 200 wm_pg_shared_lossless: dscp: 3 ecn: 1 pg: 3 - pkts_num_fill_min: 51 - pkts_num_trig_pfc: 37449 + pkts_num_fill_min: 0 + pkts_num_trig_pfc: 415271 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -281,8 +281,8 @@ qos_params: dscp: 8 ecn: 1 pg: 0 - pkts_num_fill_min: 51 - pkts_num_trig_egr_drp: 2396745 + pkts_num_fill_min: 0 + pkts_num_trig_egr_drp: 2179900 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -291,7 +291,7 @@ qos_params: ecn: 1 queue: 3 pkts_num_fill_min: 0 - pkts_num_trig_ingr_drp: 216064 + pkts_num_trig_ingr_drp: 593885 cell_size: 4096 wm_buf_pool_lossless: dscp: 3 @@ -308,7 +308,7 @@ qos_params: ecn: 1 queue: 0 pkts_num_fill_min: 0 - pkts_num_trig_egr_drp: 2396745 + pkts_num_trig_egr_drp: 2179900 cell_size: 4096 wm_buf_pool_lossy: dscp: 8 diff --git a/tests/qos/files/qos_params.pac.yaml b/tests/qos/files/qos_params.pac.yaml index ec721b403e..a9c4fa163a 100644 --- a/tests/qos/files/qos_params.pac.yaml +++ b/tests/qos/files/qos_params.pac.yaml @@ -93,6 +93,7 @@ qos_params: ecn: 1 pg: 3 pkts_num_trig_pfc: 2561 + pkts_num_hysteresis: 1024 pkts_num_dismiss_pfc: 2 packet_size: 1350 xon_2: @@ -100,6 +101,7 @@ qos_params: ecn: 1 pg: 4 pkts_num_trig_pfc: 2561 + pkts_num_hysteresis: 1024 pkts_num_dismiss_pfc: 2 packet_size: 1350 lossless_voq_1: @@ -226,6 +228,7 @@ qos_params: ecn: 1 pg: 3 pkts_num_trig_pfc: 2562 + pkts_num_hysteresis: 1024 pkts_num_dismiss_pfc: 21 packet_size: 1350 xon_2: @@ -233,6 +236,7 @@ qos_params: ecn: 1 pg: 4 pkts_num_trig_pfc: 2562 + pkts_num_hysteresis: 1024 pkts_num_dismiss_pfc: 21 packet_size: 1350 lossless_voq_1: diff --git a/tests/qos/files/qos_params.th2.yaml b/tests/qos/files/qos_params.th2.yaml index 0b99ccee64..c5726a9f51 100644 --- a/tests/qos/files/qos_params.th2.yaml +++ b/tests/qos/files/qos_params.th2.yaml @@ -184,6 +184,34 @@ qos_params: cell_size: 208 hdrm_pool_wm_multiplier: 4 cell_size: 208 + topo-dualtor-aa-56: + 50000_40m: + pcbb_xoff_1: + dscp: 3 + ecn: 1 + pg: 3 + pkts_num_trig_pfc: 19939 + pkts_num_margin: 4 + pcbb_xoff_2: + dscp: 4 + ecn: 1 + pg: 4 + pkts_num_trig_pfc: 19939 + pkts_num_margin: 4 + pcbb_xoff_3: + outer_dscp: 2 + dscp: 3 + ecn: 1 + pg: 2 + pkts_num_trig_pfc: 19939 + pkts_num_margin: 4 + pcbb_xoff_4: + outer_dscp: 6 + dscp: 4 + ecn: 1 + pg: 6 + pkts_num_trig_pfc: 19939 + pkts_num_margin: 4 topo-any: 40000_300m: pkts_num_leak_out: 0 diff --git a/tests/qos/qos_sai_base.py b/tests/qos/qos_sai_base.py index 6274e6a27f..7e599f8c6e 100644 --- a/tests/qos/qos_sai_base.py +++ b/tests/qos/qos_sai_base.py @@ -36,12 +36,12 @@ class QosBase: Common APIs """ SUPPORTED_T0_TOPOS = ["t0", "t0-56-po2vlan", "t0-64", "t0-116", "t0-35", "dualtor-56", "dualtor-64", "dualtor-120", - "dualtor", "t0-120", "t0-80", "t0-backend", "t0-56-o8v48", "t0-8-lag", "t0-standalone-32", - "t0-standalone-64", "t0-standalone-128", "t0-standalone-256"] - SUPPORTED_T1_TOPOS = ["t1-lag", "t1-64-lag", "t1-56-lag", "t1-backend", "t1-28-lag"] + "dualtor", "dualtor-64-breakout", "t0-120", "t0-80", "t0-backend", "t0-56-o8v48", "t0-8-lag", + "t0-standalone-32", "t0-standalone-64", "t0-standalone-128", "t0-standalone-256", "t0-28"] + SUPPORTED_T1_TOPOS = ["t1-lag", "t1-64-lag", "t1-56-lag", "t1-backend", "t1-28-lag", "t1-32-lag"] SUPPORTED_PTF_TOPOS = ['ptf32', 'ptf64'] - SUPPORTED_ASIC_LIST = ["pac", "gr", "gb", "td2", "th", "th2", "spc1", "spc2", "spc3", "spc4", "td3", "th3", - "j2c+", "jr2"] + SUPPORTED_ASIC_LIST = ["pac", "gr", "gr2", "gb", "td2", "th", "th2", "spc1", "spc2", "spc3", "spc4", "td3", "th3", + "j2c+", "jr2", "th5"] BREAKOUT_SKUS = ['Arista-7050-QX-32S'] @@ -114,7 +114,7 @@ def dutTestParams(self, duthosts, dut_test_params_qos, tbinfo, get_src_dst_asic_ yield dut_test_params_qos - def runPtfTest(self, ptfhost, testCase='', testParams={}, relax=False): + def runPtfTest(self, ptfhost, testCase='', testParams={}, relax=False, pdb=False): """ Runs QoS SAI test case on PTF host @@ -132,19 +132,24 @@ def runPtfTest(self, ptfhost, testCase='', testParams={}, relax=False): """ custom_options = " --disable-ipv6 --disable-vxlan --disable-geneve" \ " --disable-erspan --disable-mpls --disable-nvgre" + # Append a suffix to the logfile name if log_suffix is present in testParams + log_suffix = testParams.get("log_suffix", "") + logfile_suffix = "_{0}".format(log_suffix) if log_suffix else "" + ptf_runner( ptfhost, "saitests", testCase, platform_dir="ptftests", params=testParams, - log_file="/tmp/{0}.log".format(testCase), + log_file="/tmp/{0}{1}.log".format(testCase, logfile_suffix), # Include suffix in the logfile name, qlen=10000, is_python3=True, relax=relax, timeout=1200, socket_recv_size=16384, - custom_options=custom_options + custom_options=custom_options, + pdb=pdb ) @@ -811,9 +816,32 @@ def __buildPortSpeeds(self, config_facts): port_speeds[attr['speed']].append(etp) return port_speeds + @pytest.fixture(scope='class', autouse=False) + def configure_ip_on_ptf_intfs(self, ptfhost, get_src_dst_asic_and_duts, tbinfo): + src_dut = get_src_dst_asic_and_duts['src_dut'] + src_mgFacts = src_dut.get_extended_minigraph_facts(tbinfo) + topo = tbinfo["topo"]["name"] + + # if PTF64 and is Cisco, set ip IP address on eth interfaces of the ptf" + if topo == 'ptf64' and is_cisco_device(src_dut): + minigraph_ip_interfaces = src_mgFacts['minigraph_interfaces'] + for entry in minigraph_ip_interfaces: + ptfhost.shell("ip addr add {}/31 dev eth{}".format( + entry['peer_addr'], src_mgFacts["minigraph_ptf_indices"][entry['attachto']]) + ) + yield + for entry in minigraph_ip_interfaces: + ptfhost.shell("ip addr del {}/31 dev eth{}".format( + entry['peer_addr'], src_mgFacts["minigraph_ptf_indices"][entry['attachto']]) + ) + return + else: + yield + return + @pytest.fixture(scope='class', autouse=True) def dutConfig( - self, request, duthosts, get_src_dst_asic_and_duts, + self, request, duthosts, configure_ip_on_ptf_intfs, get_src_dst_asic_and_duts, lower_tor_host, tbinfo, dualtor_ports_for_duts, dut_qos_maps): # noqa F811 """ Build DUT host config pertaining to QoS SAI tests @@ -933,7 +961,7 @@ def dutConfig( testPortIds[src_dut_index][src_asic_index] = sorted( list(testPortIps[src_dut_index][src_asic_index].keys())) - elif topo in self.SUPPORTED_T1_TOPOS: + elif topo in self.SUPPORTED_T1_TOPOS or (topo in self.SUPPORTED_PTF_TOPOS and is_cisco_device(src_dut)): # T1 is supported only for 'single_asic' or 'single_dut_multi_asic'. # So use src_dut as the dut use_separated_upkink_dscp_tc_map = separated_dscp_to_tc_map_on_uplink(dut_qos_maps) @@ -1016,7 +1044,7 @@ def dutConfig( # Map port IDs to system port for dnx chassis if 'platform_asic' in get_src_dst_asic_and_duts["src_dut"].facts and \ get_src_dst_asic_and_duts["src_dut"].facts['platform_asic'] == 'broadcom-dnx': - sys_key = src_asic.namespace + '|' + iface + sys_key = src_asic.namespace + '|' + iface if src_asic.namespace else iface if sys_key in src_system_port: system_port = src_system_port[sys_key]['system_port_id'] sysPort = {'port': iface, 'system_port': system_port, 'port_type': iface} @@ -1033,7 +1061,7 @@ def dutConfig( if 'platform_asic' in get_src_dst_asic_and_duts["src_dut"].facts and \ get_src_dst_asic_and_duts["src_dut"].facts['platform_asic'] == 'broadcom-dnx': for portName in src_mgFacts["minigraph_portchannels"][iface]["members"]: - sys_key = src_asic.namespace + '|' + portName + sys_key = src_asic.namespace + '|' + portName if src_asic.namespace else portName port_Index = src_mgFacts["minigraph_ptf_indices"][portName] if sys_key in src_system_port: system_port = src_system_port[sys_key]['system_port_id'] @@ -1069,7 +1097,7 @@ def dutConfig( # Map port IDs to system port IDs if 'platform_asic' in get_src_dst_asic_and_duts["src_dut"].facts and \ get_src_dst_asic_and_duts["src_dut"].facts['platform_asic'] == 'broadcom-dnx': - sys_key = dst_asic.namespace + '|' + iface + sys_key = dst_asic.namespace + '|' + iface if dst_asic.namespace else iface if sys_key in dst_system_port: system_port = dst_system_port[sys_key]['system_port_id'] sysPort = {'port': iface, 'system_port': system_port, 'port_type': iface} @@ -1086,7 +1114,7 @@ def dutConfig( if 'platform_asic' in get_src_dst_asic_and_duts["src_dut"].facts and \ get_src_dst_asic_and_duts["src_dut"].facts['platform_asic'] == 'broadcom-dnx': for portName in dst_mgFacts["minigraph_portchannels"][iface]["members"]: - sys_key = dst_asic.namespace + '|' + portName + sys_key = dst_asic.namespace + '|' + portName if dst_asic.namespace else portName port_Index = dst_mgFacts["minigraph_ptf_indices"][portName] if sys_key in dst_system_port: system_port = dst_system_port[sys_key]['system_port_id'] @@ -1248,7 +1276,7 @@ def updateIptables(self, duthosts, get_src_dst_asic_and_duts, swapSyncd_on_selec @pytest.fixture(scope='class') def stopServices( - self, duthosts, get_src_dst_asic_and_duts, + self, duthosts, get_src_dst_asic_and_duts, dut_disable_ipv6, swapSyncd_on_selected_duts, enable_container_autorestart, disable_container_autorestart, get_mux_status, # noqa F811 tbinfo, upper_tor_host, lower_tor_host, toggle_all_simulator_ports): # noqa F811 """ @@ -2043,14 +2071,21 @@ def updateRedisSchedParam(schedParam): ] ) + portSpeedCableLength = dutQosConfig["portSpeedCableLength"] + qosConfig = dutQosConfig["param"] + if "wrr_chg" in qosConfig[portSpeedCableLength]: + qosConfigWrrChg = qosConfig[portSpeedCableLength]["wrr_chg"] + else: + qosConfigWrrChg = qosConfig["wrr_chg"] + wrrSchedParams = [ { "profile": lossySchedProfile["schedProfile"], - "qosConfig": dutQosConfig["param"]["wrr_chg"]["lossy_weight"] + "qosConfig": qosConfigWrrChg["lossy_weight"] }, { "profile": losslessSchedProfile["schedProfile"], - "qosConfig": dutQosConfig["param"]["wrr_chg"]["lossless_weight"] + "qosConfig": qosConfigWrrChg["lossless_weight"] }, ] @@ -2261,19 +2296,20 @@ def get_interface_ip(self, dut_asic): return mapping @pytest.fixture(autouse=False) - def _check_ingress_speed_gte_400g( + def skip_400g_longlink( self, get_src_dst_asic_and_duts, dutQosConfig): portSpeedCableLength = dutQosConfig["portSpeedCableLength"] - m = re.search("([0-9]+)_([0-9]+m)", portSpeedCableLength) + m = re.search("([0-9]+)_([0-9]+)m", portSpeedCableLength) if not m: raise RuntimeError( "Format error in portSpeedCableLength:{}". format(portSpeedCableLength)) speed = int(m.group(1)) - if speed >= 400000: - pytest.skip("PGDrop test is not supported for 400G port speed.") + cable_length = int(m.group(2)) + if speed >= 400000 and cable_length >= 120000: + pytest.skip("PGDrop test is not supported for 400G longlink.") def select_port_ids_for_mellnaox_device(self, duthost, mgFacts, testPortIds, dualtor_dut_ports=None): """ @@ -2379,6 +2415,67 @@ def populate_arp_entries( ptfhost, testCase=saiQosTest, testParams=testParams ) + @pytest.fixture(scope="function", autouse=False) + def set_static_route_ptf64(self, dutConfig, get_src_dst_asic_and_duts, dutTestParams, enum_frontend_asic_index): + def generate_ip_address(base_ip, new_first_octet): + octets = base_ip.split('.') + if len(octets) != 4: + raise ValueError("Invalid IP address format") + octets[0] = str(new_first_octet) + octets[2] = octets[3] + octets[3] = '1' + return '.'.join(octets) + + def combine_ips(src_ips, dst_ips, new_first_octet): + combined_ips_map = {} + + for key, src_info in src_ips.items(): + src_ip = src_info['peer_addr'] + new_ip = generate_ip_address(src_ip, new_first_octet) + combined_ips_map[key] = {'original_ip': src_ip, 'generated_ip': new_ip} + + for key, dst_info in dst_ips.items(): + dst_ip = dst_info['peer_addr'] + new_ip = generate_ip_address(dst_ip, new_first_octet) + combined_ips_map[key] = {'original_ip': dst_ip, 'generated_ip': new_ip} + + return combined_ips_map + + def configRoutePrefix(add_route): + action = "add" if add_route else "del" + for port, entry in combined_ips_map.items(): + if enum_frontend_asic_index is None: + src_asic.shell("config route {} prefix {}.0/24 nexthop {}".format( + action, '.'.join(entry['generated_ip'].split('.')[:3]), entry['original_ip'])) + else: + src_asic.shell("ip netns exec asic{} config route {} prefix {}.0/24 nexthop {}".format( + enum_frontend_asic_index, + action, '.'.join(entry['generated_ip'].split('.')[:3]), + entry['original_ip']) + ) + + if dutTestParams["basicParams"]["sonic_asic_type"] != "cisco-8000": + pytest.skip("Traffic sanity test is not supported") + + if dutTestParams["topo"] != "ptf64": + pytest.skip("Test not supported in {} topology. Use ptf64 topo".format(dutTestParams["topo"])) + + src_dut_index = get_src_dst_asic_and_duts['src_dut_index'] + dst_dut_index = get_src_dst_asic_and_duts['dst_dut_index'] + src_asic_index = get_src_dst_asic_and_duts['src_asic_index'] + dst_asic_index = get_src_dst_asic_and_duts['dst_asic_index'] + src_asic = get_src_dst_asic_and_duts['src_asic'] + + src_testPortIps = dutConfig["testPortIps"][src_dut_index][src_asic_index] + dst_testPortIps = dutConfig["testPortIps"][dst_dut_index][dst_asic_index] + + new_first_octet = 100 + combined_ips_map = combine_ips(src_testPortIps, dst_testPortIps, new_first_octet) + + configRoutePrefix(True) + yield combined_ips_map + configRoutePrefix(False) + @pytest.fixture(scope="function", autouse=False) def skip_longlink(self, dutQosConfig): portSpeedCableLength = dutQosConfig["portSpeedCableLength"] @@ -2388,3 +2485,15 @@ def skip_longlink(self, dutQosConfig): "This test is skipped for longlink.") yield return + + @pytest.fixture(scope="class", autouse=False) + def tc_to_dscp_count(self, get_src_dst_asic_and_duts): + duthost = get_src_dst_asic_and_duts['src_dut'] + tc_to_dscp_count_map = {} + for tc in range(8): + tc_to_dscp_count_map[tc] = 0 + config_facts = duthost.get_running_config_facts() + dscp_to_tc_map = config_facts['DSCP_TO_TC_MAP']['AZURE'] + for dscp, tc in dscp_to_tc_map.items(): + tc_to_dscp_count_map[int(tc)] += 1 + yield tc_to_dscp_count_map diff --git a/tests/qos/test_ecn_config.py b/tests/qos/test_ecn_config.py index dedb3a66f7..7e6b2f12d4 100644 --- a/tests/qos/test_ecn_config.py +++ b/tests/qos/test_ecn_config.py @@ -33,9 +33,13 @@ def enable_serviceability_cli(duthost, show_cmd): logging.info("Enabling dshell client") for asic in asics: - cmd = "docker exec -i syncd{} supervisorctl start dshell_client".format(asic) + cmd = "docker exec syncd{} supervisorctl start dshell_client".format(asic) result = duthost.command(cmd) verify_command_result(result, cmd) + if "already started" in result["stdout"]: + cmd = "docker exec syncd{} supervisorctl restart dshell_client".format(asic) + result = duthost.command(cmd) + verify_command_result(result, cmd) time.sleep(20) @@ -97,8 +101,7 @@ def verify_command_result(result, cmd): assert not traceback_found, "Traceback found in {}".format(cmd) -@pytest.mark.parametrize("pg_to_test", [3, 4]) -def test_verify_ecn_marking_config(duthosts, rand_one_dut_hostname, pg_to_test, request): +def test_verify_ecn_marking_config(duthosts, rand_one_dut_hostname, request): """ @summary: Verify output of `show platform npu voq cgm_profile with wred_profile drop probability` """ @@ -153,75 +156,96 @@ def test_verify_ecn_marking_config(duthosts, rand_one_dut_hostname, pg_to_test, else: pytest.skip("No ports available") + port_qos_map_command = "sonic-cfggen -d{} --var-json PORT_QOS_MAP" + logging.info("Fetching PORT_QOS_MAP for asic: {}".format(asic)) + cmd = port_qos_map_command.format(asic_namespace_string) + result = duthost.command(cmd) + verify_command_result(result, cmd) + + json_str = result["stdout"].strip() + try: + port_qos_map_data = json.loads(json_str) + except Exception as e: + logging.info("JSon load error: {}".format(e)) + continue + show_command = "sudo show platform npu voq cgm_profile -i {} -t {}{} -d" for port in all_ports: - logging.info("Checking Port: {}".format(port)) - cmd = show_command.format(port, pg_to_test, asic_namespace_string) - result = duthost.command(cmd) - verify_command_result(result, cmd) - json_str = result["stdout"].strip() - try: - data = json.loads(json_str) - except Exception as e: - logging.info("JSon load error: {}".format(e)) - continue - voq_mark_data = None - if "voq_mark_prob_g" in data: - voq_mark_data = data["voq_mark_prob_g"] - if voq_mark_data: - sms_quant_len = len(voq_mark_data) - voq_quant_len = len(voq_mark_data[0]) - age_quant_len = len(voq_mark_data[0][1]) - else: - logging.info("Marking data unavailable for Port {} PG {}." - " Please check if PFC is enabled".format(port, pg_to_test)) + # if pfc_enable is empty or not present, then PFC is not configured on the interface hence skip the check + if port not in port_qos_map_data or "pfc_enable" not in port_qos_map_data[port] or \ + not port_qos_map_data[port]['pfc_enable']: + logging.info("PFC is not enabled on {}".format(port)) continue - voq_drop_data = None - if "voq_drop_prob_g" in data: - voq_drop_data = data["voq_drop_prob_g"] - if not voq_mark_data and voq_drop_data: - sms_quant_len = len(voq_drop_data) - voq_quant_len = len(voq_drop_data[0]) - age_quant_len = len(voq_drop_data[0][1]) - - if voq_mark_data: - for g_idx in range(sms_quant_len): - for voq_idx in range(voq_quant_len): - for age_idx in range(age_quant_len): - actual_value = round(voq_mark_data[g_idx][voq_idx][age_idx], 2) - if age_idx == 0: - mark_level = 0 - elif (voq_idx >= 1 and age_idx == 1): - mark_level = 1 - elif (voq_idx >= 1 and age_idx == 2): - mark_level = 2 - elif (voq_idx >= 1 and age_idx >= 3): - mark_level = 3 - else: - mark_level = 0 - expected_value = round(data["wm_prob"][mark_level], 2) - assert ( - actual_value == expected_value - ), ''' - Marking Probability not as expected for Port {} PG {} - at SMS/VoQ/Age region {}/{}/{} Expected: {} Actual: {} - '''.format(port, pg_to_test, g_idx, voq_idx, - age_idx, expected_value, actual_value) - - ''' Verify drop is 7 for last quant only''' - if voq_drop_data: - for g_idx in range(sms_quant_len): - for voq_idx in range(voq_quant_len): - for age_idx in range(age_quant_len): - actual_value = voq_drop_data[g_idx][voq_idx][age_idx] - expected_value = 7 if voq_idx == (voq_quant_len - 1) else 0 - assert ( - actual_value == expected_value - ), ''' - Drop Probability not as expected for Port {} PG {} at - SMS/VoQ/Age region {}/{}/{} Expected: {} Actual: {} - '''.format(port, pg_to_test, g_idx, voq_idx, - age_idx, expected_value, actual_value) + for pg_to_test in port_qos_map_data[port]['pfc_enable'].split(','): + logging.info("Checking Port: {} pg {}".format(port, pg_to_test)) + cmd = show_command.format(port, pg_to_test, asic_namespace_string) + result = duthost.command(cmd) + verify_command_result(result, cmd) + + json_str = result["stdout"].strip() + try: + data = json.loads(json_str) + except Exception as e: + logging.info("JSon load error: {}".format(e)) + continue + voq_mark_data = None + if "voq_mark_prob_g" in data: + voq_mark_data = data["voq_mark_prob_g"] + if voq_mark_data: + sms_quant_len = len(voq_mark_data) + voq_quant_len = len(voq_mark_data[0]) + age_quant_len = len(voq_mark_data[0][1]) + else: + logging.info("Marking data unavailable for Port {} PG {}." + " Please check if PFC is enabled".format(port, pg_to_test)) + continue + + voq_drop_data = None + if "voq_drop_prob_g" in data: + voq_drop_data = data["voq_drop_prob_g"] + if not voq_mark_data and voq_drop_data: + sms_quant_len = len(voq_drop_data) + voq_quant_len = len(voq_drop_data[0]) + age_quant_len = len(voq_drop_data[0][1]) + + if voq_mark_data: + for g_idx in range(sms_quant_len): + for voq_idx in range(voq_quant_len): + for age_idx in range(age_quant_len): + actual_value = round(voq_mark_data[g_idx][voq_idx][age_idx], 2) + if age_idx == 0: + mark_level = 0 + elif (voq_idx >= 1 and age_idx == 1): + mark_level = 1 + elif (voq_idx >= 1 and age_idx == 2): + mark_level = 2 + elif (voq_idx >= 1 and age_idx >= 3): + mark_level = 3 + else: + mark_level = 0 + expected_value = round(data["wm_prob"][mark_level], 2) + assert ( + actual_value == expected_value + ), ''' + Marking Probability not as expected for Port {} PG {} + at SMS/VoQ/Age region {}/{}/{} Expected: {} Actual: {} + '''.format(port, pg_to_test, g_idx, voq_idx, + age_idx, expected_value, actual_value) + + ''' Verify drop is 7 for last quant only''' + if voq_drop_data: + for g_idx in range(sms_quant_len): + for voq_idx in range(voq_quant_len): + for age_idx in range(age_quant_len): + actual_value = voq_drop_data[g_idx][voq_idx][age_idx] + expected_value = 7 if voq_idx == (voq_quant_len - 1) else 0 + assert ( + actual_value == expected_value + ), ''' + Drop Probability not as expected for Port {} PG {} at + SMS/VoQ/Age region {}/{}/{} Expected: {} Actual: {} + '''.format(port, pg_to_test, g_idx, voq_idx, + age_idx, expected_value, actual_value) diff --git a/tests/qos/test_pfc_counters.py b/tests/qos/test_pfc_counters.py index e3b3e10c90..287e7c13a5 100644 --- a/tests/qos/test_pfc_counters.py +++ b/tests/qos/test_pfc_counters.py @@ -16,7 +16,7 @@ """ pytestmark = [ - pytest.mark.topology('t0') + pytest.mark.topology('any') ] logger = logging.getLogger(__name__) @@ -24,7 +24,8 @@ PFC_GEN_FILE_RELATIVE_PATH = r'../../ansible/roles/test/files/helpers/pfc_gen.py' """ Expected PFC generator path at the leaf fanout switch """ PFC_GEN_FILE_DEST = r'~/pfc_gen.py' -PFC_GEN_FILE_ABSULOTE_PATH = r'/root/pfc_gen.py' +PFC_GEN_FILE_ABSULOTE_PATH = r'/root/pfc_gen_cpu.py' + """ Number of generated packets for each test case """ PKT_COUNT = 10 """ Number of switch priorities """ @@ -58,12 +59,16 @@ def setup_testbed(fanouthosts, duthost, leaf_fanouts): # noqa F811 """ Copy the PFC generator to all the leaf fanout switches """ for peer_device in leaf_fanouts: + if peer_device not in fanouthosts: + continue + peerdev_ans = fanouthosts[peer_device] file_src = os.path.join(os.path.dirname( __file__), PFC_GEN_FILE_RELATIVE_PATH) peerdev_ans.host.copy(src=file_src, dest=PFC_GEN_FILE_DEST, force=True) + def run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fanouts, # noqa F811 is_pfc=True, pause_time=65535, check_continous_pfc=False): """ @@ -90,6 +95,10 @@ def run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fa for intf in active_phy_intfs: peer_device = conn_facts[intf]['peerdevice'] peer_port = conn_facts[intf]['peerport'] + + if peer_device not in fanouthosts: + continue + peerdev_ans = fanouthosts[peer_device] fanout_os = peerdev_ans.get_fanout_os() fanout_hwsku = fanout_graph_facts[peerdev_ans.hostname]["device_info"]["HwSku"] @@ -145,6 +154,10 @@ def run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fa peer_device = conn_facts[intf]['peerdevice'] peer_port = conn_facts[intf]['peerport'] + + if peer_device not in fanouthosts: + continue + peerdev_ans = fanouthosts[peer_device] fanout_os = peerdev_ans.get_fanout_os() fanout_hwsku = fanout_graph_facts[peerdev_ans.hostname]["device_info"]["HwSku"] @@ -180,40 +193,40 @@ def run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fa assert pfc_rx[intf]['Rx'][i] == '0' -def test_pfc_pause(fanouthosts, duthosts, rand_one_dut_hostname, +def test_pfc_pause(fanouthosts, duthosts, rand_one_tgen_dut_hostname, conn_graph_facts, fanout_graph_facts, leaf_fanouts): # noqa F811 """ @Summary: Run PFC pause frame (pause time quanta > 0) tests """ - duthost = duthosts[rand_one_dut_hostname] + duthost = duthosts[rand_one_tgen_dut_hostname] run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fanouts) -def test_pfc_unpause(fanouthosts, duthosts, rand_one_dut_hostname, +def test_pfc_unpause(fanouthosts, duthosts, rand_one_tgen_dut_hostname, conn_graph_facts, fanout_graph_facts, leaf_fanouts): # noqa F811 """ @Summary: Run PFC unpause frame (pause time quanta = 0) tests """ - duthost = duthosts[rand_one_dut_hostname] + duthost = duthosts[rand_one_tgen_dut_hostname] run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fanouts, pause_time=0) -def test_fc_pause(fanouthosts, duthosts, rand_one_dut_hostname, +def test_fc_pause(fanouthosts, duthosts, rand_one_tgen_dut_hostname, conn_graph_facts, fanout_graph_facts, leaf_fanouts): # noqa F811 """ @Summary: Run FC pause frame (pause time quanta > 0) tests """ - duthost = duthosts[rand_one_dut_hostname] + duthost = duthosts[rand_one_tgen_dut_hostname] run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fanouts, is_pfc=False) -def test_fc_unpause(fanouthosts, duthosts, rand_one_dut_hostname, +def test_fc_unpause(fanouthosts, duthosts, rand_one_tgen_dut_hostname, conn_graph_facts, fanout_graph_facts, leaf_fanouts): # noqa F811 """ @Summary: Run FC pause frame (pause time quanta = 0) tests """ - duthost = duthosts[rand_one_dut_hostname] + duthost = duthosts[rand_one_tgen_dut_hostname] run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fanouts, is_pfc=False, pause_time=0) -def test_continous_pfc(fanouthosts, duthosts, rand_one_dut_hostname, +def test_continous_pfc(fanouthosts, duthosts, rand_one_tgen_dut_hostname, conn_graph_facts, fanout_graph_facts, leaf_fanouts): # noqa F811 - duthost = duthosts[rand_one_dut_hostname] + duthost = duthosts[rand_one_tgen_dut_hostname] run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fanouts, check_continous_pfc=True) diff --git a/tests/qos/test_pfc_pause.py b/tests/qos/test_pfc_pause.py index 1064d0c8ba..7b3c95d4a2 100644 --- a/tests/qos/test_pfc_pause.py +++ b/tests/qos/test_pfc_pause.py @@ -188,9 +188,17 @@ def run_test(pfc_test_setup, fanouthosts, duthost, ptfhost, conn_graph_facts, + "vlan_id=%s;" % vlan_id + "testbed_type=\'%s\'" % testbed_type) - cmd = 'ptf --test-dir %s pfc_pause_test %s --test-params="%s"' % ( - os.path.dirname(PTF_FILE_REMOTE_PATH), intf_info, test_params) + # ptf_runner; from tests.ptf_runner import ptf_runner + # need to check the output of ptf cmd, could not use the ptf_runner directly + path_exists = ptfhost.stat(path="/root/env-python3/bin/ptf") + if path_exists["stat"]["exists"]: + cmd = '/root/env-python3/bin/ptf --test-dir %s pfc_pause_test %s --test-params="%s"' % ( + os.path.dirname(PTF_FILE_REMOTE_PATH), intf_info, test_params) + else: + cmd = 'ptf --test-dir %s pfc_pause_test %s --test-params="%s"' % ( + os.path.dirname(PTF_FILE_REMOTE_PATH), intf_info, test_params) print(cmd) + stdout = ansible_stdout_to_str(ptfhost.shell(cmd)['stdout']) words = stdout.split() diff --git a/tests/qos/test_qos_dscp_mapping.py b/tests/qos/test_qos_dscp_mapping.py index 1723774345..44638aa6b5 100644 --- a/tests/qos/test_qos_dscp_mapping.py +++ b/tests/qos/test_qos_dscp_mapping.py @@ -11,10 +11,11 @@ from scapy.all import Ether, IP from tabulate import tabulate +from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor # noqa F401 from tests.common.helpers.ptf_tests_helper import downstream_links, upstream_links, select_random_link,\ get_stream_ptf_ports, get_dut_pair_port_from_ptf_port, apply_dscp_cfg_setup, apply_dscp_cfg_teardown # noqa F401 from tests.common.utilities import get_ipv4_loopback_ip, get_dscp_to_queue_value, find_egress_queue,\ - get_egress_queue_pkt_count_all_prio + get_egress_queue_pkt_count_all_prio, wait_until from tests.common.helpers.assertions import pytest_assert from tests.common.fixtures.duthost_utils import dut_qos_maps_module # noqa F401 @@ -27,8 +28,7 @@ DEFAULT_DSCP = 4 DEFAULT_TTL = 64 DEFAULT_ECN = 1 -DEFAULT_PKT_COUNT = 10000 -TOLERANCE = 0.05 * DEFAULT_PKT_COUNT # Account for noise and polling delays +DEFAULT_PKT_COUNT = 2000 DUMMY_OUTER_SRC_IP = '8.8.8.8' DUMMY_INNER_SRC_IP = '9.9.9.9' DUMMY_INNER_DST_IP = '10.10.10.10' @@ -127,8 +127,6 @@ def send_and_verify_traffic(ptfadapter, logger.info("Received packet(s) on port {}".format(ptf_dst_port_ids[port_index])) global packet_egressed_success packet_egressed_success = True - # Wait for packets to be processed by the DUT - time.sleep(8) return ptf_dst_port_ids[port_index] except AssertionError as detail: @@ -271,22 +269,19 @@ def _run_test(self, ptf_src_port_id=ptf_src_port_id, ptf_dst_port_ids=ptf_dst_port_ids) - except Exception as e: - logger.error(str(e)) + except ConnectionError as e: + # Sending large number of packets can cause socket buffer to be full and leads connection timeout. + logger.error("{}: Try reducing DEFAULT_PKT_COUNT value".format(str(e))) failed_once = True global packet_egressed_success if packet_egressed_success: dut_egress_port = get_dut_pair_port_from_ptf_port(duthost, tbinfo, dst_ptf_port_id) pytest_assert(dut_egress_port, "No egress port on DUT found for ptf port {}".format(dst_ptf_port_id)) + # Wait for the queue counters to be populated. + verification_success = wait_until(60, 2, 0, lambda: find_queue_count_and_value(duthost, + queue_val, dut_egress_port)[0] >= DEFAULT_PKT_COUNT) egress_queue_count, egress_queue_val = find_queue_count_and_value(duthost, queue_val, dut_egress_port) - # Re-poll DUT if queue value could not be accurately found - if egress_queue_val == -1: - time.sleep(2) - egress_queue_count, egress_queue_val = find_queue_count_and_value(duthost, queue_val, - dut_egress_port) - verification_success = abs(egress_queue_count - DEFAULT_PKT_COUNT) < TOLERANCE - if verification_success: logger.info("SUCCESS: Received expected number of packets on queue {}".format(queue_val)) output_table.append([rotating_dscp, queue_val, egress_queue_count, "SUCCESS", queue_val]) @@ -311,6 +306,7 @@ def _run_test(self, failed_once = True else: output_table.append([rotating_dscp, queue_val, 0, "FAILURE - NO PACKETS EGRESSED", "N/A"]) + failed_once = True # Reset packet egress status packet_egressed_success = False @@ -319,6 +315,8 @@ def _run_test(self, .format(tabulate(output_table, headers=["Inner Packet DSCP Value", "Expected Egress Queue", "Egress Queue Count", "Result", "Actual Egress Queue"]))) + # Clear the output_table (for next test functions). + output_table = [] pytest_assert(not failed_once, "FAIL: Test failed. Please check table for details.") @@ -331,18 +329,26 @@ def _teardown_test(self, duthost): """ apply_dscp_cfg_teardown(duthost) - def test_dscp_to_queue_mapping_pipe_mode(self, ptfadapter, duthost, tbinfo, downstream_links, upstream_links, dut_qos_maps_module): # noqa F811 + def test_dscp_to_queue_mapping_pipe_mode(self, ptfadapter, rand_selected_dut, + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + setup_standby_ports_on_rand_unselected_tor, + tbinfo, downstream_links, upstream_links, dut_qos_maps_module): # noqa F811 """ Test QoS SAI DSCP to queue mapping for IP-IP packets in DSCP "pipe" mode """ + duthost = rand_selected_dut test_params = self._setup_test_params(duthost, downstream_links, upstream_links, "pipe") self._run_test(ptfadapter, duthost, tbinfo, test_params, dut_qos_maps_module, "pipe") self._teardown_test(duthost) - def test_dscp_to_queue_mapping_uniform_mode(self, ptfadapter, duthost, tbinfo, downstream_links, upstream_links, dut_qos_maps_module): # noqa F811 + def test_dscp_to_queue_mapping_uniform_mode(self, ptfadapter, rand_selected_dut, + toggle_all_simulator_ports_to_rand_selected_tor, # noqa F811 + setup_standby_ports_on_rand_unselected_tor, + tbinfo, downstream_links, upstream_links, dut_qos_maps_module): # noqa F811 """ Test QoS SAI DSCP to queue mapping for IP-IP packets in DSCP "uniform" mode """ + duthost = rand_selected_dut test_params = self._setup_test_params(duthost, downstream_links, upstream_links, "uniform") self._run_test(ptfadapter, duthost, tbinfo, test_params, dut_qos_maps_module, "uniform") self._teardown_test(duthost) diff --git a/tests/qos/test_qos_sai.py b/tests/qos/test_qos_sai.py index 363d90c373..1b6dfcb86c 100644 --- a/tests/qos/test_qos_sai.py +++ b/tests/qos/test_qos_sai.py @@ -37,7 +37,7 @@ from tests.common.dualtor.dual_tor_utils import dualtor_ports, is_tunnel_qos_remap_enabled # noqa F401 from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.pfc_storm import PFCStorm -from tests.pfcwd.files.pfcwd_helper import set_pfc_timers, start_wd_on_ports +from tests.common.helpers.pfcwd_helper import set_pfc_timers, start_wd_on_ports from tests.common.platform.device_utils import list_dut_fanout_connections from tests.common.utilities import wait_until from .qos_sai_base import QosSaiBase @@ -890,7 +890,7 @@ def testQosSaiSharedReservationSize( RunAnsibleModuleFail if ptf test fails """ if ('modular_chassis' in get_src_dst_asic_and_duts['src_dut'].facts and - get_src_dst_asic_and_duts['src_dut'].facts["modular_chassis"] == "True"): + get_src_dst_asic_and_duts['src_dut'].facts["modular_chassis"]): if dutConfig['dstDutAsic'] != "pac": pytest.skip("This test is skipped since not enough ports on cisco-8000 " "T2 Q200.") @@ -1231,13 +1231,17 @@ def testQosSaiLossyQueueVoq( if not get_src_dst_asic_and_duts['single_asic_test']: pytest.skip("Lossy Queue Voq test is only supported on cisco-8000 single-asic") if "lossy_queue_voq_1" in LossyVoq: - if 'modular_chassis' in get_src_dst_asic_and_duts['src_dut'].facts and\ - get_src_dst_asic_and_duts['src_dut'].facts["modular_chassis"] == "True": - pytest.skip("LossyQueueVoq: This test is skipped since cisco-8000 T2 " - "doesn't support split-voq.") + if ('modular_chassis' in get_src_dst_asic_and_duts['src_dut'].facts and + get_src_dst_asic_and_duts['src_dut'].facts["modular_chassis"]): + if get_src_dst_asic_and_duts['src_dut'].facts['platform'] != 'x86_64-88_lc0_36fh-r0': + pytest.skip("LossyQueueVoq: This test is skipped since cisco-8000 T2 " + "doesn't support split-voq.") elif "lossy_queue_voq_2" in LossyVoq: + if get_src_dst_asic_and_duts['src_dut'].facts['platform'] == 'x86_64-88_lc0_36fh-r0': + pytest.skip("LossyQueueVoq: lossy_queue_voq_2 test is not applicable " + "for x86_64-88_lc0_36fh-r0, with split-voq.") if not ('modular_chassis' in get_src_dst_asic_and_duts['src_dut'].facts and - get_src_dst_asic_and_duts['src_dut'].facts["modular_chassis"] == "True"): + get_src_dst_asic_and_duts['src_dut'].facts["modular_chassis"]): pytest.skip("LossyQueueVoq: lossy_queue_voq_2 test is not applicable " "for split-voq.") portSpeedCableLength = dutQosConfig["portSpeedCableLength"] @@ -1307,7 +1311,8 @@ def testQosSaiLossyQueueVoq( raise def testQosSaiDscpQueueMapping( - self, ptfhost, get_src_dst_asic_and_duts, dutTestParams, dutConfig, dut_qos_maps # noqa F811 + self, ptfhost, get_src_dst_asic_and_duts, dutTestParams, dutConfig, dut_qos_maps, # noqa F811 + tc_to_dscp_count ): """ Test QoS SAI DSCP to queue mapping @@ -1319,6 +1324,7 @@ def testQosSaiDscpQueueMapping( dutConfig (Fixture, dict): Map of DUT config containing dut interfaces, test port IDs, test port IPs, and test ports dut_qos_maps(Fixture): A fixture, return qos maps on DUT host + tc_to_dscp_count(Fixture): A fixture, return tc to dscp_count map on DUT host Returns: None @@ -1342,7 +1348,8 @@ def testQosSaiDscpQueueMapping( "src_port_ip": dutConfig["testPorts"]["src_port_ip"], "hwsku": dutTestParams['hwsku'], "dual_tor": dutConfig['dualTor'], - "dual_tor_scenario": dutConfig['dualTorScenario'] + "dual_tor_scenario": dutConfig['dualTorScenario'], + "tc_to_dscp_count_map": tc_to_dscp_count }) if "platform_asic" in dutTestParams["basicParams"]: @@ -1719,8 +1726,7 @@ def testQosSaiPgHeadroomWatermark( ) def testQosSaiPGDrop( - self, ptfhost, dutTestParams, dutConfig, dutQosConfig, - _check_ingress_speed_gte_400g + self, ptfhost, dutTestParams, dutConfig, dutQosConfig, skip_400g_longlink ): """ Test QoS SAI PG drop counter @@ -2258,3 +2264,89 @@ def testQosSaiLossyQueueVoqMultiSrc( ptfhost, testCase="sai_qos_tests.LossyQueueVoqMultiSrcTest", testParams=testParams ) + + def testQosSaiFullMeshTrafficSanity( + self, ptfhost, dutTestParams, dutConfig, dutQosConfig, + get_src_dst_asic_and_duts, dut_qos_maps, # noqa F811 + set_static_route_ptf64 + ): + """ + Test QoS SAI traffic sanity + Args: + ptfhost (AnsibleHost): Packet Test Framework (PTF) + dutTestParams (Fixture, dict): DUT host test params + dutConfig (Fixture, dict): Map of DUT config containing dut interfaces, test port IDs, test port IPs, + and test ports + dutQosConfig (Fixture, dict): Map containing DUT host QoS configuration + Returns: + None + Raises: + RunAnsibleModuleFail if ptf test fails + """ + # Execution with a specific set of dst port + def run_test_for_dst_port(start, end): + test_params = dict() + test_params.update(dutTestParams["basicParams"]) + test_params.update({ + "testbed_type": dutTestParams["topo"], + "all_src_port_id_to_ip": all_src_port_id_to_ip, + "all_src_port_id_to_name": all_src_port_id_to_name, + "all_dst_port_id_to_ip": {port_id: all_dst_port_id_to_ip[port_id] for port_id in range(start, end)}, + "all_dst_port_id_to_name": {port_id: all_dst_port_id_to_name[port_id] for port_id in range(start, end)}, + "dscp_to_q_map": dscp_to_q_map, + # Add a log_suffix to have separate log and pcap file name + "log_suffix": "_".join([str(port_id) for port_id in range(start, end)]), + "hwsku": dutTestParams['hwsku'] + }) + + self.runPtfTest(ptfhost, testCase="sai_qos_tests.FullMeshTrafficSanity", testParams=test_params) + + src_dut_index = get_src_dst_asic_and_duts['src_dut_index'] + dst_dut_index = get_src_dst_asic_and_duts['dst_dut_index'] + src_asic_index = get_src_dst_asic_and_duts['src_asic_index'] + dst_asic_index = get_src_dst_asic_and_duts['dst_asic_index'] + + src_testPortIps = dutConfig["testPortIps"][src_dut_index][src_asic_index] + dst_testPortIps = dutConfig["testPortIps"][dst_dut_index][dst_asic_index] + + # Fetch all port IDs and IPs + all_src_port_id_to_ip = {port_id: src_testPortIps[port_id]['peer_addr'] for port_id in src_testPortIps.keys()} + + all_src_port_id_to_name = { + port_id: dutConfig["dutInterfaces"][port_id] + for port_id in all_src_port_id_to_ip.keys() + } + + all_dst_port_id_to_ip = { + port_id: set_static_route_ptf64[port_id]['generated_ip'] + for port_id in dst_testPortIps.keys() + } + + all_dst_port_id_to_name = { + port_id: dutConfig["dutInterfaces"][port_id] + for port_id in all_dst_port_id_to_ip.keys() + } + + try: + tc_to_q_map = dut_qos_maps['tc_to_queue_map']['AZURE'] + tc_to_dscp_map = {v: k for k, v in dut_qos_maps['dscp_to_tc_map']['AZURE'].items()} + except KeyError: + pytest.skip( + "Need both TC_TO_PRIORITY_GROUP_MAP and DSCP_TO_TC_MAP" + "and key AZURE to run this test.") + + dscp_to_q_map = {tc_to_dscp_map[tc]: tc_to_q_map[tc] for tc in tc_to_dscp_map if tc != 7} + + # Define the number of splits + # for the dst port list + num_splits = 4 + + # Get all keys and sort them + all_keys = sorted(all_dst_port_id_to_ip.keys()) + + # Calculate the split points + split_points = [all_keys[i * len(all_keys) // num_splits] for i in range(1, num_splits)] + + # Execute with one set of dst port at a time, avoids ptf run getting timed out + for start, end in zip([0] + split_points, split_points + [len(all_keys)]): + run_test_for_dst_port(start, end) diff --git a/tests/route/conftest.py b/tests/route/conftest.py index 27682c8674..4a8cd62eb2 100644 --- a/tests/route/conftest.py +++ b/tests/route/conftest.py @@ -16,7 +16,7 @@ def pytest_addoption(parser): @pytest.fixture(scope='module') -def get_function_conpleteness_level(pytestconfig): +def get_function_completeness_level(pytestconfig): return pytestconfig.getoption("--completeness_level") diff --git a/tests/route/test_default_route.py b/tests/route/test_default_route.py index 0bf5fd51f9..b54d3a559e 100644 --- a/tests/route/test_default_route.py +++ b/tests/route/test_default_route.py @@ -8,7 +8,7 @@ from tests.common.utilities import wait_until from tests.common.utilities import find_duthost_on_role from tests.common.utilities import get_upstream_neigh_type -from tests.syslog.syslog_utils import is_mgmt_vrf_enabled +from tests.common.helpers.syslog_helpers import is_mgmt_vrf_enabled pytestmark = [ @@ -19,6 +19,23 @@ logger = logging.getLogger(__name__) +@pytest.fixture(autouse=True) +def ignore_expected_loganalyzer_exception(loganalyzer, duthosts): + + ignore_errors = [ + r".* ERR syncd#syncd: .*SAI_API_TUNNEL:_brcm_sai_mptnl_tnl_route_event_add:\d+ ecmp table entry lookup " + "failed with error.*", + r".* ERR syncd#syncd: .*SAI_API_TUNNEL:_brcm_sai_mptnl_process_route_add_mode_default_and_host:\d+ " + "_brcm_sai_mptnl_tnl_route_event_add failed with error.*" + ] + + if loganalyzer: + for duthost in duthosts: + loganalyzer[duthost.hostname].ignore_regex.extend(ignore_errors) + + return None + + def get_upstream_neigh(tb, device_neigh_metadata): """ Get the information for upstream neighbors present in the testbed diff --git a/tests/route/test_forced_mgmt_route.py b/tests/route/test_forced_mgmt_route.py index 0b05159a7e..d224207bc3 100644 --- a/tests/route/test_forced_mgmt_route.py +++ b/tests/route/test_forced_mgmt_route.py @@ -3,13 +3,14 @@ import logging import pytest +from tests.common.config_reload import config_reload from tests.common.helpers.assertions import pytest_assert -from tests.common.utilities import wait_until -from tests.override_config_table.utilities import backup_config, restore_config, \ - reload_minigraph_with_golden_config -from tests.syslog.syslog_utils import is_mgmt_vrf_enabled +from tests.common.helpers.syslog_helpers import is_mgmt_vrf_enabled +from tests.common.utilities import wait_until, wait_for_file_changed, backup_config, \ + restore_config, reload_minigraph_with_golden_config, FORCED_MGMT_ROUTE_PRIORITY pytestmark = [ + pytest.mark.disable_loganalyzer, pytest.mark.topology('t0'), pytest.mark.device_type('vs') ] @@ -17,16 +18,6 @@ logger = logging.getLogger(__name__) -# forced mgmt route priority hardcoded to 32764 in following j2 template: -# https://github.com/sonic-net/sonic-buildimage/blob/master/files/image_config/interfaces/interfaces.j2#L82 -FORCED_MGMT_ROUTE_PRIORITY = 32764 - - -# Wait 300 seconds because sometime 'interfaces-config' service take 45 seconds to response -# interfaces-config service issue track by: https://github.com/sonic-net/sonic-buildimage/issues/19045 -FILE_CHANGE_TIMEOUT = 300 - - @pytest.fixture def backup_restore_config(duthosts, enum_rand_one_per_hwsku_hostname): """make sure tacacs server running after UT finish""" @@ -41,36 +32,7 @@ def backup_restore_config(duthosts, enum_rand_one_per_hwsku_hostname): # Restore config after test finish restore_config(duthost, CONFIG_DB, CONFIG_DB_BACKUP) - - -def get_interface_reload_timestamp(duthost): - timestamp = duthost.command("sudo systemctl show --no-pager interfaces-config" - " -p ExecMainExitTimestamp --value")["stdout"] - logger.info("interfaces config timestamp {}".format(timestamp)) - - return timestamp - - -def get_file_hash(duthost, file): - hash = duthost.command("sha1sum {}".format(file))["stdout"] - logger.debug("file hash: {}".format(hash)) - - return hash - - -def wait_for_file_changed(duthost, file, action, *args, **kwargs): - original_hash = get_file_hash(duthost, file) - last_timestamp = get_interface_reload_timestamp(duthost) - - action(*args, **kwargs) - - def hash_and_timestamp_changed(duthost, file): - latest_hash = get_file_hash(duthost, file) - latest_timestamp = get_interface_reload_timestamp(duthost) - return latest_hash != original_hash and latest_timestamp != last_timestamp - - exist = wait_until(FILE_CHANGE_TIMEOUT, 1, 0, hash_and_timestamp_changed, duthost, file) - pytest_assert(exist, "File {} does not change after {} seconds.".format(file, FILE_CHANGE_TIMEOUT)) + config_reload(duthost) def address_type(address): @@ -125,9 +87,14 @@ def test_forced_mgmt_route_add_and_remove_by_mgmt_port_status( config_db_mgmt_interface = config_db_json["MGMT_INTERFACE"] config_db_port = config_db_json["MGMT_PORT"] - # Skip multi-asic because override_config format are different. + # Skip if port does not exist + output = duthost.command("ip link show eth1", module_ignore_errors=True) + if output["failed"]: + pytest.skip("Skip test_forced_mgmt_route_add_and_remove_by_mgmt_port_status, port does not exist") + + # Skip if port is already in use if 'eth1' in config_db_port: - pytest.skip("Skip test_forced_mgmt_route_add_and_remove_by_mgmt_port_status for multi-mgmt device") + pytest.skip("Skip test_forced_mgmt_route_add_and_remove_by_mgmt_port_status, port in use") # Add eth1 to mgmt interface and port ipv4_forced_mgmt_address = "172.17.1.1/24" @@ -158,7 +125,13 @@ def test_forced_mgmt_route_add_and_remove_by_mgmt_port_status( "/etc/network/interfaces", reload_minigraph_with_golden_config, duthost, - override_config) + override_config, + False) + + # for device can't config eth1, ignore this test case + eth1_status = duthost.command("sudo ifconfig eth1")['stdout'] + if "Device not found" in eth1_status: + pytest.skip("Skip test_forced_mgmt_route_add_and_remove_by_mgmt_port_status because hardware can't config eth1") # Get interface and check config generate correct interfaces = duthost.command("cat /etc/network/interfaces")['stdout'] diff --git a/tests/route/test_route_bgp_ecmp.py b/tests/route/test_route_bgp_ecmp.py new file mode 100644 index 0000000000..4b18b61aef --- /dev/null +++ b/tests/route/test_route_bgp_ecmp.py @@ -0,0 +1,107 @@ +import requests +import json +import logging +import time +import pytest + +pytestmark = [ + pytest.mark.topology("t0") +] + +logger = logging.getLogger(__name__) + +EXABGP_BASE_PORT = 5000 +TEST_ROUTE = '20.0.0.1/32' +TEST_AS_PATH = '65000 65001 65002' +NHIPV4 = '10.10.246.254' +WITHDRAW = 'withdraw' +ANNOUNCE = 'announce' + + +@pytest.fixture(scope="module") +def setup_and_teardown(): + # Setup code + logger.info("Setting up the test environment") + + # This is where the test function will be executed + yield + + # Teardown code + logger.info("Tearing down the test environment") + + +def change_route(operation, ptfip, route, nexthop, port, aspath): + url = "http://%s:%d" % (ptfip, port) + data = { + "command": "%s route %s next-hop %s as-path [ %s ]" % (operation, route, nexthop, aspath)} + r = requests.post(url, data=data, timeout=30) + if r.status_code != 200: + raise Exception( + "Change routes failed: url={}, data={}, r.status_code={}, r.reason={}, r.headers={}, r.text={}".format( + url, + json.dumps(data), + r.status_code, + r.reason, + r.headers, + r.text + ) + ) + + +def announce_route(ptfip, route, nexthop, port, aspath): + logger.info("Announce route {} to ptf".format(route)) + change_route(ANNOUNCE, ptfip, route, nexthop, port, aspath) + + +def withdraw_route(ptfip, route, nexthop, port, aspath): + logger.info("Withdraw route {} to ptf".format(route)) + change_route(WITHDRAW, ptfip, route, nexthop, port, aspath) + + +def check_route(duthost, route, operation): + cmd = 'vtysh -c "show ip route {} json"'.format(route) + logger.info("Run cmd: %s" % cmd) + out = json.loads(duthost.shell(cmd, verbose=False)['stdout']) + if (operation == ANNOUNCE): + internalNextHopActiveNum = out[route][0]['internalNextHopActiveNum'] + logger.info("internalNextHopActiveNum = %d" % internalNextHopActiveNum) + if (internalNextHopActiveNum < 2): + logger.info("Cli output: %s" % out) + pytest.fail("Active next hop number is less than 2") + elif (operation == WITHDRAW): + if (out != {}): + logger.info("Cli output is NOT empty unexpectedly: %s" % out) + pytest.fail("Route is not withdrawn") + + +def test_route_bgp_ecmp(duthosts, tbinfo, enum_rand_one_per_hwsku_frontend_hostname, + loganalyzer, setup_and_teardown): # noqa F811 + ptf_ip = tbinfo['ptf_ip'] + common_config = tbinfo['topo']['properties']['configuration_properties'].get( + 'common', {}) + nexthop = common_config.get('nhipv4', NHIPV4) + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + if loganalyzer: + ignoreRegex = [ + ".*ERR.*\"missed_FRR_routes\".*" + ] + loganalyzer[duthost.hostname].ignore_regex.extend(ignoreRegex) + + logger.info("ptf_ip = %s" % ptf_ip) + logger.info("nexthop = %s" % nexthop) + + try: + logger.info("Announce route") + announce_route(ptf_ip, TEST_ROUTE, nexthop, EXABGP_BASE_PORT, TEST_AS_PATH) + announce_route(ptf_ip, TEST_ROUTE, nexthop, EXABGP_BASE_PORT + 1, TEST_AS_PATH) + logger.info("Sleep 5 seconds and check if route is announced") + time.sleep(5) + check_route(duthost, TEST_ROUTE, ANNOUNCE) + + finally: + logger.info("Withdraw route") + withdraw_route(ptf_ip, TEST_ROUTE, nexthop, EXABGP_BASE_PORT, TEST_AS_PATH) + withdraw_route(ptf_ip, TEST_ROUTE, nexthop, EXABGP_BASE_PORT + 1, TEST_AS_PATH) + logger.info("Sleep 5 seconds and check if route is withdrawn") + time.sleep(5) + check_route(duthost, TEST_ROUTE, WITHDRAW) diff --git a/tests/route/test_route_consistency.py b/tests/route/test_route_consistency.py index a54d92dba8..8fb5cf6754 100644 --- a/tests/route/test_route_consistency.py +++ b/tests/route/test_route_consistency.py @@ -28,9 +28,11 @@ def extract_dest_ips(self, route_entries): def get_route_prefix_snapshot_from_asicdb(self, duthosts): prefix_snapshot = {} max_prefix_cnt = 0 - for dut in duthosts.frontend_nodes: + for idx, dut in enumerate(duthosts.frontend_nodes): for asic in dut.asics: dut_instance_name = dut.hostname + '-' + str(asic.asic_index) + if dut.facts['switch_type'] == "voq" and idx == 0: + dut_instance_name = dut_instance_name + "UpstreamLc" prefix_snapshot[dut_instance_name] = \ set(self.extract_dest_ips(asic.run_sonic_db_cli_cmd('ASIC_DB KEYS *ROUTE_ENTRY*')['stdout_lines'])) logger.debug("snapshot of route table from {}: {}".format(dut_instance_name, @@ -43,10 +45,10 @@ def setup(self, duthosts): # take the snapshot of route table from all the DUTs self.__class__.pre_test_route_snapshot, max_prefix_cnt = self.get_route_prefix_snapshot_from_asicdb(duthosts) """sleep interval is calculated based on the max number of prefixes in the route table. - Addtional 100 seconds is added to the sleep interval to account for the time taken to + Addtional 120 seconds is added to the sleep interval to account for the time taken to withdraw and advertise the routes by peers. """ - self.__class__.sleep_interval = math.ceil(max_prefix_cnt/3000) + 100 + self.__class__.sleep_interval = math.ceil(max_prefix_cnt/3000) + 120 logger.debug("max_no_of_prefix: {} sleep_interval: {}".format(max_prefix_cnt, self.sleep_interval)) def test_route_withdraw_advertise(self, duthosts, tbinfo, localhost): @@ -61,18 +63,26 @@ def test_route_withdraw_advertise(self, duthosts, tbinfo, localhost): time.sleep(self.sleep_interval) """ compare the number of routes withdrawn from all the DUTs. In working condition, the number of routes - withdrawn should be same across all the DUTs + withdrawn should be same across all the DUTs. On VOQ upstream LC's will have same route and Downstream LC will have same route. Note: this will be noop for single asic pizzabox duts """ post_withdraw_route_snapshot, _ = self.get_route_prefix_snapshot_from_asicdb(duthosts) num_routes_withdrawn = 0 + num_routes_withdrawn_upstream_lc = 0 for dut_instance_name in self.pre_test_route_snapshot.keys(): - if num_routes_withdrawn == 0: + if num_routes_withdrawn == 0 and not dut_instance_name.endswith("UpstreamLc"): num_routes_withdrawn = len(self.pre_test_route_snapshot[dut_instance_name] - post_withdraw_route_snapshot[dut_instance_name]) logger.debug("num_routes_withdrawn: {}".format(num_routes_withdrawn)) + elif num_routes_withdrawn_upstream_lc == 0 and dut_instance_name.endswith("UpstreamLc"): + num_routes_withdrawn_upstream_lc = len(self.pre_test_route_snapshot[dut_instance_name] - + post_withdraw_route_snapshot[dut_instance_name]) else: - assert num_routes_withdrawn == len(self.pre_test_route_snapshot[dut_instance_name] - + if dut_instance_name.endswith("UpstreamLc"): + assert num_routes_withdrawn_upstream_lc == len(self.pre_test_route_snapshot[dut_instance_name] - + post_withdraw_route_snapshot[dut_instance_name]) + else: + assert num_routes_withdrawn == len(self.pre_test_route_snapshot[dut_instance_name] - post_withdraw_route_snapshot[dut_instance_name]) logger.info("advertise ipv4 and ipv6 routes for {}".format(topo_name)) diff --git a/tests/route/test_route_flap.py b/tests/route/test_route_flap.py index df0693502b..d418e85cb7 100644 --- a/tests/route/test_route_flap.py +++ b/tests/route/test_route_flap.py @@ -375,7 +375,7 @@ def get_dev_port_and_route(duthost, asichost, dst_prefix_set): def test_route_flap(duthosts, tbinfo, ptfhost, ptfadapter, - get_function_conpleteness_level, announce_default_routes, + get_function_completeness_level, announce_default_routes, enum_rand_one_per_hwsku_frontend_hostname, enum_rand_one_frontend_asic_index, setup_standby_ports_on_non_enum_rand_one_per_hwsku_frontend_host_m, # noqa F811 toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m, loganalyzer): # noqa F811 @@ -448,9 +448,9 @@ def test_route_flap(duthosts, tbinfo, ptfhost, ptfadapter, logger.info("exabgp_port = %d" % exabgp_port) ping_ip = route_to_ping.strip('/{}'.format(route_prefix_len)) - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: - normalized_level = 'basic' + normalized_level = 'debug' loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level] diff --git a/tests/route/test_route_flow_counter.py b/tests/route/test_route_flow_counter.py index a5a7310e4e..0fbfd18982 100644 --- a/tests/route/test_route_flow_counter.py +++ b/tests/route/test_route_flow_counter.py @@ -2,15 +2,16 @@ import pytest from tests.common.plugins.allure_wrapper import allure_step_wrapper as allure from tests.common.helpers.assertions import pytest_assert, pytest_require -from tests.flow_counter import flow_counter_utils -from tests.flow_counter.flow_counter_utils import is_route_flow_counter_supported # noqa F401 +from tests.common.flow_counter import flow_counter_utils +from tests.common.flow_counter.flow_counter_utils import is_route_flow_counter_supported # noqa F401 from tests.common.utilities import wait_until logger = logging.getLogger(__name__) allure.logger = logger pytestmark = [ - pytest.mark.topology("any") + pytest.mark.topology("any"), + pytest.mark.device_type('physical') ] test_update_route_pattern_para = [ @@ -214,4 +215,7 @@ def _get_nexthop(self, duthost, ipv6): else: cmd = 'show ip bgp summary' parse_result = duthost.show_and_parse(cmd) - return parse_result[0]['neighbhor'] + if 'neighbor' in parse_result[0]: + return parse_result[0]['neighbor'] + else: + return parse_result[0]['neighbhor'] diff --git a/tests/route/test_static_route.py b/tests/route/test_static_route.py index 4cdf0499fd..a6d8b61e8d 100644 --- a/tests/route/test_static_route.py +++ b/tests/route/test_static_route.py @@ -11,9 +11,11 @@ from tests.common.fixtures.ptfhost_utils import change_mac_addresses, copy_arp_responder_py # noqa F811 from tests.common.dualtor.dual_tor_utils import mux_cable_server_ip from tests.common.dualtor.mux_simulator_control import mux_server_url # noqa F811 +from tests.common.dualtor.dual_tor_utils import show_muxcable_status from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F811 from tests.common.utilities import wait_until, get_intf_by_sub_intf from tests.common.utilities import get_neighbor_ptf_port_list +from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.assertions import pytest_require from tests.common.helpers.constants import UPSTREAM_NEIGHBOR_MAP from tests.common import config_reload @@ -21,7 +23,7 @@ import ptf.mask as mask import ptf.packet as packet from tests.common import constants -from tests.flow_counter.flow_counter_utils import RouteFlowCounterTestContext, is_route_flow_counter_supported # noqa F811 +from tests.common.flow_counter.flow_counter_utils import RouteFlowCounterTestContext, is_route_flow_counter_supported # noqa F811 pytestmark = [ @@ -157,7 +159,7 @@ def _check_routes(): return False return True - assert(wait_until(60, 15, 0, _check_routes)) + assert (wait_until(60, 15, 0, _check_routes)) # output example of ip [-6] route show @@ -195,6 +197,12 @@ def _check_nh_in_output(nexthop): ) +def check_mux_status(duthost, expected_status): + show_mux_status_ret = show_muxcable_status(duthost) + status_values = set([intf_status['status'] for intf_status in show_mux_status_ret.values()]) + return status_values == {expected_status} + + def run_static_route_test(duthost, unselected_duthost, ptfadapter, ptfhost, tbinfo, prefix, nexthop_addrs, prefix_len, nexthop_devs, nexthop_interfaces, is_route_flow_counter_supported, ipv6=False, config_reload_test=False): # noqa F811 @@ -248,6 +256,15 @@ def run_static_route_test(duthost, unselected_duthost, ptfadapter, ptfhost, tbin config_reload(duthost, wait=500) else: config_reload(duthost, wait=450) + # On dualtor, config_reload can result in a switchover (active tor can become standby and viceversa). + # So we need to make sure rand_selected_dut is in active state before verifying traffic. + if is_dual_tor: + duthost.shell("config mux mode active all") + unselected_duthost.shell("config mux mode standby all") + pytest_assert(wait_until(60, 5, 0, check_mux_status, duthost, 'active'), + "Could not config ports to active on {}".format(duthost.hostname)) + pytest_assert(wait_until(60, 5, 0, check_mux_status, unselected_duthost, 'standby'), + "Could not config ports to standby on {}".format(unselected_duthost.hostname)) # FIXME: We saw re-establishing BGP sessions can takes around 7 minutes # on some devices (like 4600) after config reload, so we need below patch wait_all_bgp_up(duthost) @@ -339,6 +356,7 @@ def get_nexthops(duthost, tbinfo, ipv6=False, count=1): def test_static_route(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhost, tbinfo, + setup_standby_ports_on_rand_unselected_tor, # noqa F811 toggle_all_simulator_ports_to_rand_selected_tor_m, is_route_flow_counter_supported): # noqa F811 duthost = rand_selected_dut unselected_duthost = rand_unselected_dut @@ -349,6 +367,7 @@ def test_static_route(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhos @pytest.mark.disable_loganalyzer def test_static_route_ecmp(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhost, tbinfo, + setup_standby_ports_on_rand_unselected_tor, # noqa F811 toggle_all_simulator_ports_to_rand_selected_tor_m, is_route_flow_counter_supported): # noqa F811 duthost = rand_selected_dut unselected_duthost = rand_unselected_dut @@ -359,6 +378,7 @@ def test_static_route_ecmp(rand_selected_dut, rand_unselected_dut, ptfadapter, p def test_static_route_ipv6(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhost, tbinfo, + setup_standby_ports_on_rand_unselected_tor, # noqa F811 toggle_all_simulator_ports_to_rand_selected_tor_m, is_route_flow_counter_supported): # noqa F811 duthost = rand_selected_dut unselected_duthost = rand_unselected_dut @@ -370,6 +390,7 @@ def test_static_route_ipv6(rand_selected_dut, rand_unselected_dut, ptfadapter, p @pytest.mark.disable_loganalyzer def test_static_route_ecmp_ipv6(rand_selected_dut, rand_unselected_dut, ptfadapter, ptfhost, tbinfo, + setup_standby_ports_on_rand_unselected_tor, # noqa F811 toggle_all_simulator_ports_to_rand_selected_tor_m, is_route_flow_counter_supported): # noqa F811 duthost = rand_selected_dut unselected_duthost = rand_unselected_dut diff --git a/tests/route/utils.py b/tests/route/utils.py index c5766babca..c9c80fc9d5 100644 --- a/tests/route/utils.py +++ b/tests/route/utils.py @@ -24,7 +24,7 @@ def generate_intf_neigh(asichost, num_neigh, ip_version, mg_facts=None, is_backe else: interfaces = asichost.show_interface(command="status")["ansible_facts"]["int_status"] for intf, values in list(interfaces.items()): - if values["admin_state"] == "up" and values["oper_state"] == "up": + if values["admin_state"] == "up" and values["oper_state"] == "up" and values["type"] != "DPU-NPU Data Port": up_interfaces.append(intf) if not up_interfaces: raise Exception("DUT does not have up interfaces") diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 2a309e7105..35e801676e 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -71,11 +71,16 @@ function validate_parameters() RET=2 fi - if [[ -z ${TOPOLOGY} && -z ${TEST_CASES} ]]; then - echo "Neither TOPOLOGY (-t) nor test case list (-c) is set.." + if [[ -z ${TOPOLOGY} && -z ${TEST_CASES} && -z ${TEST_CASES_FILE} ]]; then + echo "Neither TOPOLOGY (-t) nor test case list (-c) nor test case list file (-F) is set.." RET=3 fi + if [[ ${TEST_CASES} && ${TEST_CASES_FILE} ]]; then + echo "Specified both a test case list (-c) and a test case list file (-F).." + RET=4 + fi + if [[ ${RET} != 0 ]]; then show_help_and_exit ${RET} fi @@ -132,7 +137,7 @@ function setup_test_options() # expanded to matched test scripts by bash. Among the expanded scripts, we may want to skip a few. Then we can # explicitly specify the script to be skipped. ignores=$(python3 -c "print('|'.join('''$SKIP_FOLDERS'''.split()))") - if [[ -z ${TEST_CASES} ]]; then + if [[ -z ${TEST_CASES} && -z ${TEST_CASES_FILE} ]]; then # When TEST_CASES is not specified, find all the possible scripts, ignore the scripts under $SKIP_FOLDERS all_scripts=$(find ./ -name 'test_*.py' | sed s:^./:: | grep -vE "^(${ignores})") ignore_files=("test_pretest.py" "test_posttest.py") @@ -144,6 +149,9 @@ function setup_test_options() fi done else + if [[ ${TEST_CASES_FILE} ]]; then + TEST_CASES="${TEST_CASES} $(cat ${TEST_CASES_FILE} | tr '\n' ' ')" + fi # When TEST_CASES is specified, ignore the scripts under $SKIP_FOLDERS all_scripts="" for test_script in ${TEST_CASES}; do @@ -250,6 +258,7 @@ function run_debug_tests() echo "SKIP_SCRIPTS: ${SKIP_SCRIPTS}" echo "SKIP_FOLDERS: ${SKIP_FOLDERS}" echo "TEST_CASES: ${TEST_CASES}" + echo "TEST_CASES_FILE: ${TEST_CASES_FILE}" echo "TEST_FILTER: ${TEST_FILTER}" echo "TEST_INPUT_ORDER: ${TEST_INPUT_ORDER}" echo "TEST_MAX_FAIL: ${TEST_MAX_FAIL}" @@ -354,7 +363,7 @@ function run_individual_tests() setup_environment -while getopts "h?a:b:c:C:d:e:Ef:i:I:k:l:m:n:oOp:q:rs:S:t:ux" opt; do +while getopts "h?a:b:c:C:d:e:Ef:F:i:I:k:l:m:n:oOp:q:rs:S:t:ux" opt; do case ${opt} in h|\? ) show_help_and_exit 0 @@ -384,6 +393,9 @@ while getopts "h?a:b:c:C:d:e:Ef:i:I:k:l:m:n:oOp:q:rs:S:t:ux" opt; do f ) TESTBED_FILE=${OPTARG} ;; + F ) + TEST_CASES_FILE="${OPTARG}" + ;; i ) INVENTORY=${OPTARG} ;; diff --git a/tests/saitests/py3/sai_base_test.py b/tests/saitests/py3/sai_base_test.py index c3483a5768..c31cf59744 100644 --- a/tests/saitests/py3/sai_base_test.py +++ b/tests/saitests/py3/sai_base_test.py @@ -31,7 +31,9 @@ interface_to_front_mapping = {} from switch import (sai_thrift_port_tx_enable, # noqa E402 - sai_thrift_port_tx_disable) + sai_thrift_port_tx_disable, + sai_thrift_credit_wd_enable, + sai_thrift_credit_wd_disable) DATA_PLANE_QUEUE_LIST = ["0", "1", "2", "3", "4", "5", "6", "7"] DEFAULT_QUEUE_SCHEDULER_CONFIG = {"0": "scheduler.0", @@ -189,50 +191,34 @@ def sai_thrift_port_tx_enable( else: sai_thrift_port_tx_enable(client, asic_type, port_list, target=target) if self.platform_asic and self.platform_asic == "broadcom-dnx" and last_port: - # need to enable watchdog on the source asic using cint script - cmd = "bcmcmd -n {} \"BCMSAI credit-watchdog enable\"".format(self.src_asic_index) - stdOut, stdErr, retValue = self.exec_cmd_on_dut(self.src_server_ip, - self.test_params['dut_username'], - self.test_params['dut_password'], - cmd) - if retValue != 0 or 'Success rv = 0' not in stdOut[1]: - # Retry credit-wd command max 3 times on failure - while count < 3: + # need to enable watchdog on the source asic + # max 3 retries + while count < 3: + retValue = sai_thrift_credit_wd_enable(self.src_client) + if retValue == 0: + break + else: print("Retrying credit_wd_enable") - time.sleep(5) - stdOut, stdErr, retValue = self.exec_cmd_on_dut(self.src_server_ip, - self.test_params['dut_username'], - self.test_params['dut_password'], - cmd) - if stdOut and 'Success rv = 0' in stdOut[1]: - break - count += 1 - assert 'Success rv = 0' in stdOut[1] if stdOut else retValue == 0,\ - "enable wd failed '{}' on asic '{}' on '{}'".format(cmd, self.src_asic_index, self.src_server_ip) + time.sleep(1) + count += 1 + assert retValue == 0, "enable wd failed on asic '{}' on '{}' with error '{}'".format( + self.src_asic_index, self.src_server_ip, retValue) def sai_thrift_port_tx_disable(self, client, asic_type, port_list, target='dst', disable_port_by_block_queue=True): count = 0 if self.platform_asic and self.platform_asic == "broadcom-dnx": - # need to enable watchdog on the source asic using cint script - cmd = "bcmcmd -n {} \"BCMSAI credit-watchdog disable\"".format(self.src_asic_index) - stdOut, stdErr, retValue = self.exec_cmd_on_dut(self.src_server_ip, - self.test_params['dut_username'], - self.test_params['dut_password'], - cmd) - if retValue != 0 or 'Success rv = 0' not in stdOut[1]: - # Retry credit-wd command max 3 times on failure - while count < 3: - print("Retrying credit_wd_enable") - time.sleep(5) - stdOut, stdErr, retValue = self.exec_cmd_on_dut(self.src_server_ip, - self.test_params['dut_username'], - self.test_params['dut_password'], - cmd) - if stdOut and 'Success rv = 0' in stdOut[1]: - break - count += 1 - assert 'Success rv = 0' in stdOut[1] if stdOut else retValue == 0, \ - "disable wd failed '{}' on asic '{}' on '{}'".format(cmd, self.src_asic_index, self.src_server_ip) + # need to disable watchdog on the source asic + # max 3 retries + while count < 3: + retValue = sai_thrift_credit_wd_disable(self.src_client) + if retValue == 0: + break + else: + print("Retrying credit_wd_disable") + time.sleep(1) + count += 1 + assert retValue == 0, "disable wd failed on asic '{}' on '{}' with error '{}'".format( + self.src_asic_index, self.src_server_ip, retValue) if asic_type == 'mellanox' and disable_port_by_block_queue: self.disable_mellanox_egress_data_plane(port_list) diff --git a/tests/saitests/py3/sai_qos_tests.py b/tests/saitests/py3/sai_qos_tests.py index 28a90b11dc..5b5ae81173 100755 --- a/tests/saitests/py3/sai_qos_tests.py +++ b/tests/saitests/py3/sai_qos_tests.py @@ -12,6 +12,7 @@ import texttable import math import os +import concurrent.futures from ptf.testutils import (ptf_ports, dp_poll, simple_arp_packet, @@ -311,10 +312,10 @@ def get_ip_addr(): def get_tcp_port(): val = 1234 while True: - if val == 65535: - raise RuntimeError("We ran out of tcp ports!") - val = max(val, (val+10) % 65535) yield val + val += 10 + if val > 65534: + val = 1234 TCP_PORT_GEN = get_tcp_port() @@ -366,7 +367,8 @@ def construct_tcp_pkt(pkt_len, dst_mac, src_mac, src_ip, dst_ip, dscp, src_vlan, return pkt -def get_multiple_flows(dp, dst_mac, dst_id, dst_ip, src_vlan, dscp, ecn, ttl, pkt_len, src_details, packets_per_port=1): +def get_multiple_flows(dp, dst_mac, dst_id, dst_ip, src_vlan, dscp, ecn, ttl, + pkt_len, src_details, packets_per_port=1, check_actual_dst_id=True): ''' Returns a dict of format: src_id : [list of (pkt, exp_pkt) pairs that go to the given dst_id] @@ -421,7 +423,10 @@ def get_rx_port_pkt(dp, src_port_id, pkt, exp_pkt): all_pkts[src_tuple[0]] except KeyError: all_pkts[src_tuple[0]] = [] - actual_dst_id = get_rx_port_pkt(dp, src_tuple[0], pkt, masked_exp_pkt) + if check_actual_dst_id is False: + actual_dst_id = dst_id + else: + actual_dst_id = get_rx_port_pkt(dp, src_tuple[0], pkt, masked_exp_pkt) if actual_dst_id == dst_id: all_pkts[src_tuple[0]].append((pkt, masked_exp_pkt, dst_id)) num_of_pkts += 1 @@ -825,6 +830,7 @@ def runTest(self): dual_tor = self.test_params.get('dual_tor', None) leaf_downstream = self.test_params.get('leaf_downstream', None) asic_type = self.test_params['sonic_asic_type'] + tc_to_dscp_count_map = self.test_params.get('tc_to_dscp_count_map', None) exp_ip_id = 101 exp_ttl = 63 pkt_dst_mac = router_mac if router_mac != '' else dst_port_mac @@ -933,33 +939,39 @@ def runTest(self): # queue 3/4 1 1 1 1 1 # noqa E501 # queue 5 1 1 1 1 1 # noqa E501 # queue 7 0 1 1 1 1 # noqa E501 - assert (queue_results[QUEUE_0] == 1 + queue_results_base[QUEUE_0]) - assert (queue_results[QUEUE_3] == 1 + queue_results_base[QUEUE_3]) - assert (queue_results[QUEUE_4] == 1 + queue_results_base[QUEUE_4]) - assert (queue_results[QUEUE_5] == 1 + queue_results_base[QUEUE_5]) - if dual_tor or (dual_tor_scenario is False) or (leaf_downstream is False): - assert (queue_results[QUEUE_2] == 1 + - queue_results_base[QUEUE_2]) - assert (queue_results[QUEUE_6] == 1 + - queue_results_base[QUEUE_6]) + + if tc_to_dscp_count_map: + for tc in range(7): + assert (queue_results[tc] == tc_to_dscp_count_map[tc] + queue_results_base[tc]) + assert (queue_results[7] >= tc_to_dscp_count_map[7] + queue_results_base[7]) else: - assert (queue_results[QUEUE_2] == queue_results_base[QUEUE_2]) - assert (queue_results[QUEUE_6] == queue_results_base[QUEUE_6]) - if dual_tor_scenario: - if (dual_tor is False) or leaf_downstream: - assert (queue_results[QUEUE_1] == - 59 + queue_results_base[QUEUE_1]) + assert (queue_results[QUEUE_0] == 1 + queue_results_base[QUEUE_0]) + assert (queue_results[QUEUE_3] == 1 + queue_results_base[QUEUE_3]) + assert (queue_results[QUEUE_4] == 1 + queue_results_base[QUEUE_4]) + assert (queue_results[QUEUE_5] == 1 + queue_results_base[QUEUE_5]) + if dual_tor or (dual_tor_scenario is False) or (leaf_downstream is False): + assert (queue_results[QUEUE_2] == 1 + + queue_results_base[QUEUE_2]) + assert (queue_results[QUEUE_6] == 1 + + queue_results_base[QUEUE_6]) else: - assert (queue_results[QUEUE_1] == - 57 + queue_results_base[QUEUE_1]) - # LAG ports can have LACP packets on queue 7, hence using >= comparison - assert (queue_results[QUEUE_7] >= 1 + - queue_results_base[QUEUE_7]) - else: - assert (queue_results[QUEUE_1] == 58 + - queue_results_base[QUEUE_1]) - # LAG ports can have LACP packets on queue 7, hence using >= comparison - assert (queue_results[QUEUE_7] >= queue_results_base[QUEUE_7]) + assert (queue_results[QUEUE_2] == queue_results_base[QUEUE_2]) + assert (queue_results[QUEUE_6] == queue_results_base[QUEUE_6]) + if dual_tor_scenario: + if (dual_tor is False) or leaf_downstream: + assert (queue_results[QUEUE_1] == + 59 + queue_results_base[QUEUE_1]) + else: + assert (queue_results[QUEUE_1] == + 57 + queue_results_base[QUEUE_1]) + # LAG ports can have LACP packets on queue 7, hence using >= comparison + assert (queue_results[QUEUE_7] >= 1 + + queue_results_base[QUEUE_7]) + else: + assert (queue_results[QUEUE_1] == 58 + + queue_results_base[QUEUE_1]) + # LAG ports can have LACP packets on queue 7, hence using >= comparison + assert (queue_results[QUEUE_7] >= queue_results_base[QUEUE_7]) finally: print("END OF TEST", file=sys.stderr) @@ -2896,7 +2908,7 @@ def setUp(self): self.src_port_macs = [self.dataplane.get_mac( 0, ptid) for ptid in self.src_port_ids] - if self.testbed_type in ['dualtor', 'dualtor-56', 't0', 't0-64', 't0-116', 't0-120']: + if self.testbed_type in ['dualtor', 'dualtor-56', 't0', 't0-28', 't0-64', 't0-116', 't0-120']: # populate ARP # sender's MAC address is corresponding PTF port's MAC address # sender's IP address is caculated in tests/qos/qos_sai_base.py::QosSaiBase::__assignTestPortIps() @@ -3123,7 +3135,7 @@ def runTest(self): self.sai_thrift_port_tx_enable(self.dst_client, self.asic_type, self.uniq_dst_ports) sys.exit("Too many pkts needed to trigger pfc: %d" % (pkt_cnt)) assert (recv_counters[sidx_dscp_pg_tuples[i][2]] > - recv_counters_bases[sidx_dscp_pg_tuples[i][0]][sidx_dscp_pg_tuples[i][2]]) + recv_counters_bases[sidx_dscp_pg_tuples[i][0]][sidx_dscp_pg_tuples[i][2]]) print("%d packets for sid: %d, pg: %d to trigger pfc" % ( pkt_cnt, self.src_port_ids[sidx_dscp_pg_tuples[i][0]], sidx_dscp_pg_tuples[i][2] - 2), file=sys.stderr) @@ -3661,37 +3673,46 @@ def runTest(self): pkts_num_leak_out = int(self.test_params['pkts_num_leak_out']) topo = self.test_params['topo'] platform_asic = self.test_params['platform_asic'] + prio_list = self.test_params.get('dscp_list', []) + q_pkt_cnt = self.test_params.get('q_pkt_cnt', []) + q_list = self.test_params.get('q_list', []) + + self.sai_thrift_port_tx_enable(self.dst_client, asic_type, [dst_port_id], enable_port_by_unblock_queue=False) - if 'backend' not in topo: - if not qos_remap_enable: - # When qos_remap is disabled, the map is as below - # DSCP TC QUEUE - # 3 3 3 - # 4 4 4 - # 8 0 0 - # 0 1 1 - # 5 2 2 - # 46 5 5 - # 48 6 6 - prio_list = [3, 4, 8, 0, 5, 46, 48] - q_pkt_cnt = [queue_3_num_of_pkts, queue_4_num_of_pkts, queue_0_num_of_pkts, - queue_1_num_of_pkts, queue_2_num_of_pkts, queue_5_num_of_pkts, queue_6_num_of_pkts] + if not (prio_list and q_pkt_cnt and q_list): + if 'backend' not in topo: + if not qos_remap_enable: + # When qos_remap is disabled, the map is as below + # DSCP TC QUEUE + # 3 3 3 + # 4 4 4 + # 8 0 0 + # 0 1 1 + # 5 2 2 + # 46 5 5 + # 48 6 6 + prio_list = [3, 4, 8, 0, 5, 46, 48] + q_pkt_cnt = [queue_3_num_of_pkts, queue_4_num_of_pkts, queue_0_num_of_pkts, + queue_1_num_of_pkts, queue_2_num_of_pkts, queue_5_num_of_pkts, queue_6_num_of_pkts] + q_list = [3, 4, 0, 1, 2, 5, 6] + else: + # When qos_remap is enabled, the map is as below + # DSCP TC QUEUE + # 3 3 3 + # 4 4 4 + # 8 0 0 + # 0 1 1 + # 46 5 5 + # 48 7 7 + prio_list = [3, 4, 8, 0, 46, 48] + q_pkt_cnt = [queue_3_num_of_pkts, queue_4_num_of_pkts, queue_0_num_of_pkts, + queue_1_num_of_pkts, queue_5_num_of_pkts, queue_7_num_of_pkts] + q_list = [3, 4, 0, 1, 5, 7] else: - # When qos_remap is enabled, the map is as below - # DSCP TC QUEUE - # 3 3 3 - # 4 4 4 - # 8 0 0 - # 0 1 1 - # 46 5 5 - # 48 7 7 - prio_list = [3, 4, 8, 0, 46, 48] - q_pkt_cnt = [queue_3_num_of_pkts, queue_4_num_of_pkts, queue_0_num_of_pkts, - queue_1_num_of_pkts, queue_5_num_of_pkts, queue_7_num_of_pkts] - else: - prio_list = [3, 4, 1, 0, 2, 5, 6] - q_pkt_cnt = [queue_3_num_of_pkts, queue_4_num_of_pkts, queue_1_num_of_pkts, - queue_0_num_of_pkts, queue_2_num_of_pkts, queue_5_num_of_pkts, queue_6_num_of_pkts] + prio_list = [3, 4, 1, 0, 2, 5, 6] + q_pkt_cnt = [queue_3_num_of_pkts, queue_4_num_of_pkts, queue_1_num_of_pkts, + queue_0_num_of_pkts, queue_2_num_of_pkts, queue_5_num_of_pkts, queue_6_num_of_pkts] + q_list = [3, 4, 1, 0, 2, 5, 6] q_cnt_sum = sum(q_pkt_cnt) # Send packets to leak out pkt_dst_mac = router_mac if router_mac != '' else dst_port_mac @@ -3730,7 +3751,7 @@ def runTest(self): self.dst_client, asic_type, port_list['dst'][dst_port_id]) # Send packets to each queue based on priority/dscp field - for prio, pkt_cnt in zip(prio_list, q_pkt_cnt): + for prio, pkt_cnt, queue in zip(prio_list, q_pkt_cnt, q_list): pkt = construct_ip_pkt(default_packet_length, pkt_dst_mac, src_port_mac, @@ -3741,12 +3762,27 @@ def runTest(self): ip_id=exp_ip_id, ecn=ecn, ttl=64) + if 'cisco-8000' in asic_type and pkt_cnt > 0: + fill_leakout_plus_one(self, src_port_id, dst_port_id, pkt, queue, asic_type) + pkt_cnt -= 1 send_packet(self, src_port_id, pkt, pkt_cnt) # Set receiving socket buffers to some big value for p in list(self.dataplane.ports.values()): p.socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 41943040) + # recv packets for leakout + if 'cisco-8000' in asic_type: + recv_pkt = scapy.Ether() + + while recv_pkt: + received = self.dataplane.poll( + device_number=0, port_number=dst_port_id, timeout=2) + if isinstance(received, self.dataplane.PollFailure): + recv_pkt = None + break + recv_pkt = scapy.Ether(received.packet) + # Release port self.sai_thrift_port_tx_enable(self.dst_client, asic_type, [dst_port_id], enable_port_by_unblock_queue=False) @@ -3773,8 +3809,8 @@ def runTest(self): # Ignore captured non-IP packet continue - queue_pkt_counters = [0] * (prio_list[-1] + 1) - queue_num_of_pkts = [0] * (prio_list[-1] + 1) + queue_pkt_counters = [0] * (max(prio_list) + 1) + queue_num_of_pkts = [0] * (max(prio_list) + 1) for prio, q_cnt in zip(prio_list, q_pkt_cnt): queue_num_of_pkts[prio] = q_cnt @@ -5046,7 +5082,7 @@ def runTest(self): if pkts_num_fill_min: assert (q_wm_res[queue] == 0) - elif 'cisco-8000' in asic_type or "ACS-SN5600" in hwsku: + elif 'cisco-8000' in asic_type or "SN5600" in hwsku: assert (q_wm_res[queue] <= (margin + 1) * cell_size) else: if platform_asic and platform_asic == "broadcom-dnx": @@ -5366,8 +5402,7 @@ def runTest(self): file=sys.stderr, ) assert (buffer_pool_wm <= (expected_wm + upper_bound_margin) * cell_size) - assert ((expected_wm - lower_bound_margin) - * cell_size <= buffer_pool_wm) + assert ((expected_wm - lower_bound_margin) * cell_size <= buffer_pool_wm) pkts_num = pkts_inc @@ -5561,8 +5596,7 @@ def runTest(self): if 'cisco-8000' in asic_type: # Queue is always the inner_dscp due to the TC_TO_QUEUE_MAP redirection queue = inner_dscp - assert (fill_leakout_plus_one(self, src_port_id, - dst_port_id, pkt, queue, asic_type)) + assert (fill_leakout_plus_one(self, src_port_id, dst_port_id, pkt, queue, asic_type)) num_pkts = pkts_num_trig_pfc - pkts_num_margin - 1 send_packet(self, src_port_id, pkt, num_pkts) print("Sending {} packets to port {}".format(num_pkts, src_port_id), file=sys.stderr) @@ -5872,3 +5906,180 @@ def runTest(self): print("Successfully dropped {} packets".format(drops), file=sys.stderr) finally: self.sai_thrift_port_tx_enable(self.dst_client, self.asic_type, [self.dst_port_id]) + + +class FullMeshTrafficSanity(sai_base_test.ThriftInterfaceDataPlane): + def setUp(self): + sai_base_test.ThriftInterfaceDataPlane.setUp(self) + time.sleep(5) + switch_init(self.clients) + + # Parse input parameters + self.testbed_type = self.test_params['testbed_type'] + self.router_mac = self.test_params['router_mac'] + self.sonic_version = self.test_params['sonic_version'] + + dscp_to_q_map = self.test_params['dscp_to_q_map'] + self.dscps = [int(key) for key in dscp_to_q_map.keys()] + self.queues = [int(value) for value in dscp_to_q_map.values()] + self.all_src_port_id_to_ip = self.test_params['all_src_port_id_to_ip'] + self.all_src_port_id_to_name = self.test_params['all_src_port_id_to_name'] + self.all_dst_port_id_to_ip = self.test_params['all_dst_port_id_to_ip'] + self.all_dst_port_id_to_name = self.test_params['all_dst_port_id_to_name'] + + self.all_port_id_to_ip = dict() + self.all_port_id_to_ip.update(self.all_src_port_id_to_ip) + self.all_port_id_to_ip.update(self.all_dst_port_id_to_ip) + + self.all_port_id_to_name = dict() + self.all_port_id_to_name.update(self.all_src_port_id_to_name) + self.all_port_id_to_name.update(self.all_dst_port_id_to_name) + + self.src_port_ids = list(self.all_src_port_id_to_ip.keys()) + self.dst_port_ids = list(self.all_dst_port_id_to_ip.keys()) + self.all_port_ids = self.src_port_ids + list(set(self.dst_port_ids) - set(self.src_port_ids)) + + self.asic_type = self.test_params['sonic_asic_type'] + self.packet_size = 100 + logging.info("Using packet size", self.packet_size) + self.flows_per_port = 6 + + self.all_port_id_to_mac = {port_id: self.dataplane.get_mac(0, port_id) + for port_id in self.all_port_id_to_ip.keys()} + + def tearDown(self): + sai_base_test.ThriftInterfaceDataPlane.tearDown(self) + + def config_traffic(self, dst_port_id, dscp, ecn_bit): + if type(ecn_bit) == bool: + ecn_bit = 1 if ecn_bit else 0 + self.dscp = dscp + self.dst_port_id = dst_port_id + self.tos = (dscp << 2) | ecn_bit + self.ttl = 64 + logging.debug("Getting multiple flows to {:>2}, dscp={}, dst_ip={}".format( + self.dst_port_id, self.dscp, self.dst_port_ip) + ) + self.pkt = get_multiple_flows( + self, + self.dst_port_mac, + dst_port_id, + self.dst_port_ip, + None, + dscp, + ecn_bit, + 64, + self.packet_size, + [(src_port_id, src_port_ip) for src_port_id, src_port_ip in self.all_src_port_id_to_ip.items()], + self.flows_per_port, + False) + logging.debug("Got multiple flows to {:>2}, dscp={}, dst_ip={}".format( + self.dst_port_id, self.dscp, self.dst_port_ip) + ) + + def runTest(self): + failed_pairs = set() + logging.info("Total traffic src_dst_pairs being tested {}".format( + len(self.src_port_ids)*len(self.dst_port_ids)) + ) + pkt_count = 10 + + # Split the src port list for concurrent pkt injection + num_splits = 2 + split_points = [i * len(self.src_port_ids) // num_splits for i in range(1, num_splits)] + parts = [self.src_port_ids[i:j] for i, j in zip([0] + split_points, split_points + [None])] + + def runTestPerSrcList(src_port_list, checkCounter=False): + for src_port_id in src_port_list: + logging.debug( + "Sending {} packets X {} flows with dscp/queue {}/{} from src {} -> dst {}".format( + pkt_count, + len(self.pkt[src_port_id]), dscp, queue, + self.all_port_id_to_name.get(src_port_id, 'Not Found'), + dst_port_name) + ) + if checkCounter: + port_cnt_base, q_cntrs_base = sai_thrift_read_port_counters( + self.dst_client, self.asic_type, + port_list['dst'][real_dst_port_id] + ) + + for pkt_tuple in self.pkt[src_port_id]: + logging.debug( + "Sending {} packets with dscp/queue {}/{} from src {} -> dst {} Pkt {}".format( + pkt_count, dscp, queue, + self.all_port_id_to_name.get(src_port_id, 'Not Found'), + dst_port_name, pkt_tuple[0]) + ) + send_packet(self, src_port_id, pkt_tuple[0], pkt_count) + + if checkCounter: + time.sleep(1) + port_cntrs, q_cntrs = sai_thrift_read_port_counters( + self.dst_client, self.asic_type, + port_list['dst'][real_dst_port_id] + ) + pkts_enqueued = q_cntrs[queue] - q_cntrs_base[queue] + if pkts_enqueued < self.flows_per_port*pkt_count: + logging.info("Faulty src/dst {}/{} pair on queue {}".format( + self.all_port_id_to_name.get(src_port_id, 'Not Found'), + dst_port_name, queue + )) + logging.info("q_cntrs_base {}".format(q_cntrs_base)) + logging.info("q_cntrs {}".format(q_cntrs)) + logging.info("port_cnt_base {}".format(port_cnt_base)) + logging.info("port_cntrs {}".format(port_cntrs)) + failed_pairs.add( + ( + self.all_port_id_to_name.get(src_port_id, 'Not Found'), + dst_port_name, queue + ) + ) + + def findFaultySrcDstPair(dscp, queue): + ecn_bit = 1 if queue in [3, 4] else 0 + self.config_traffic(real_dst_port_id, dscp, ecn_bit) + runTestPerSrcList(self.src_port_ids, True) + + for dst_port_id in self.dst_port_ids: + real_dst_port_id = dst_port_id + dst_port_name = self.all_port_id_to_name.get(real_dst_port_id, 'Not Found') + logging.info("Starting Test for dst {}".format(dst_port_name)) + dst_port_mac = self.all_port_id_to_mac[real_dst_port_id] + self.dst_port_mac = self.router_mac if self.router_mac != '' else dst_port_mac + self.dst_port_ip = self.all_port_id_to_ip[real_dst_port_id] + + for i, dscp in enumerate(self.dscps): + queue = self.queues[i] # Need queue for occupancy verification + ecn_bit = 1 if queue in [3, 4] else 0 + self.config_traffic(real_dst_port_id, dscp, ecn_bit) + + port_cnt_base, q_cntrs_base = sai_thrift_read_port_counters( + self.dst_client, self.asic_type, + port_list['dst'][real_dst_port_id] + ) + + with concurrent.futures.ThreadPoolExecutor(max_workers=num_splits) as executor: + # Submit the tasks to the executor + futures = [executor.submit(runTestPerSrcList, part) for part in parts] + + # Wait for all tasks to complete + concurrent.futures.wait(futures) + + time.sleep(1) + port_cntrs, q_cntrs = sai_thrift_read_port_counters( + self.dst_client, self.asic_type, + port_list['dst'][real_dst_port_id] + ) + pkts_enqueued = q_cntrs[queue] - q_cntrs_base[queue] + logging.info("Enqueued on queue {} pkts {}".format(queue, pkts_enqueued)) + if pkts_enqueued < self.flows_per_port*pkt_count*len(self.src_port_ids): + logging.info("q_cntrs_base {}".format(q_cntrs_base)) + logging.info("q_cntrs {}".format(q_cntrs)) + logging.info("port_cnt_base {}".format(port_cnt_base)) + logging.info("port_cntrs {}".format(port_cntrs)) + # Craft pkt for given queue and + # inject from each src to find which src/dst pair is dropping pkt + findFaultySrcDstPair(dscp, queue) + + assert len(failed_pairs) == 0, "Traffic failed between {}".format(failed_pairs) diff --git a/tests/saitests/py3/switch.py b/tests/saitests/py3/switch.py index 225aadf12d..9c6a2e9cc9 100644 --- a/tests/saitests/py3/switch.py +++ b/tests/saitests/py3/switch.py @@ -65,7 +65,7 @@ SAI_SWITCH_ATTR_PORT_NUMBER, SAI_VIRTUAL_ROUTER_ATTR_ADMIN_V4_STATE, SAI_VIRTUAL_ROUTER_ATTR_ADMIN_V6_STATE,\ SAI_VLAN_MEMBER_ATTR_VLAN_ID, SAI_PORT_STAT_IF_IN_UCAST_PKTS,\ SAI_PORT_STAT_IF_IN_NON_UCAST_PKTS, SAI_PORT_STAT_IF_OUT_NON_UCAST_PKTS, SAI_PORT_STAT_IF_OUT_QLEN, \ - SAI_INGRESS_PRIORITY_GROUP_STAT_CURR_OCCUPANCY_BYTES + SAI_INGRESS_PRIORITY_GROUP_STAT_CURR_OCCUPANCY_BYTES, SAI_SWITCH_ATTR_CREDIT_WD from switch_sai_thrift.sai_headers import SAI_SWITCH_ATTR_SRC_MAC_ADDRESS, SAI_SYSTEM_PORT_ATTR_QOS_VOQ_LIST @@ -773,6 +773,24 @@ def sai_thrift_port_tx_enable(client, asic_type, port_ids, target='dst'): client.sai_thrift_set_port_attribute(port_list[target][port_id], attr) +def sai_thrift_credit_wd_disable(client): + # Disable credit-watchdog on target asic index + attr_value = sai_thrift_attribute_value_t(booldata=0) + attr = sai_thrift_attribute_t( + id=SAI_SWITCH_ATTR_CREDIT_WD, value=attr_value) + status = client.sai_thrift_set_switch_attribute(attr) + return status + + +def sai_thrift_credit_wd_enable(client): + # Enable credit-watchdog on target asic-index + attr_value = sai_thrift_attribute_value_t(booldata=1) + attr = sai_thrift_attribute_t( + id=SAI_SWITCH_ATTR_CREDIT_WD, value=attr_value) + status = client.sai_thrift_set_switch_attribute(attr) + return status + + def sai_thrift_read_port_counters(client, asic_type, port): port_cnt_ids = [] port_cnt_ids.append(SAI_PORT_STAT_IF_OUT_DISCARDS) diff --git a/tests/scripts/find_la_start_marker.sh b/tests/scripts/find_la_start_marker.sh new file mode 100644 index 0000000000..51883ed9c7 --- /dev/null +++ b/tests/scripts/find_la_start_marker.sh @@ -0,0 +1 @@ +sudo grep 'start-LogAnalyzer-' /var/log/syslog | grep -v ansible | tail -n 1 | awk -F 'INFO ' '{print $2}' diff --git a/tests/scripts/find_log_msg.sh b/tests/scripts/find_log_msg.sh new file mode 100644 index 0000000000..8535aef9af --- /dev/null +++ b/tests/scripts/find_log_msg.sh @@ -0,0 +1 @@ +sudo awk "/$1/{p=1; next} p{print}" /var/log/syslog | grep "$2" diff --git a/tests/scripts/getbuild.py b/tests/scripts/getbuild.py index 01e4df096b..9c7ab4dfff 100755 --- a/tests/scripts/getbuild.py +++ b/tests/scripts/getbuild.py @@ -44,6 +44,8 @@ def reporthook(count, block_size, total_size): def validate_url_or_abort(url): + print("Validating URL: {}".format(url)) + # Attempt to retrieve HTTP response code try: urlfile = urlopen(url) @@ -62,16 +64,24 @@ def validate_url_or_abort(url): sys.exit(1) -def get_download_url(buildid, artifact_name, url_prefix, access_token): +def get_download_url(buildid, artifact_name, url_prefix, access_token, token): """get download url""" artifact_req = Request("https://dev.azure.com/{}/_apis/build/builds/{}/artifacts?artifactName={}&api-version=5.0" .format(url_prefix, buildid, artifact_name)) - # If access token is not empty, set headers - if access_token: - artifact_req.add_header('Authorization', - 'Basic {}'.format(base64.b64encode(access_token.encode('utf-8')).decode('utf-8'))) + # Here "access_token" indeed is Azure DevOps PAT token. + # "token" should be the actual bearer token for Azure DevOps. + # PAT should be deprecated. below logic is to handle both cases for smooth transition from PAT to bearer token. + if token: + artifact_req.add_header( + 'Authorization', 'Bearer {}'.format(token) + ) + elif access_token: + artifact_req.add_header( + 'Authorization', + 'Basic {}'.format(base64.b64encode(access_token.encode('utf-8')).decode('utf-8')) + ) resp = urlopen(artifact_req) @@ -83,7 +93,7 @@ def get_download_url(buildid, artifact_name, url_prefix, access_token): return (download_url, artifact_size) -def download_artifacts(url, content_type, platform, buildid, num_asic, access_token): +def download_artifacts(url, content_type, platform, buildid, num_asic, access_token, token): """find latest successful build id for a branch""" if content_type == 'image': @@ -107,15 +117,29 @@ def download_artifacts(url, content_type, platform, buildid, num_asic, access_to download_times = 0 while download_times < MAX_DOWNLOAD_TIMES: try: - print(('Downloading {} from build {}...'.format(filename, buildid))) + print(('Downloading {} from build {}, url: {}'.format(filename, buildid, url))) download_times += 1 - # If access token is not empty, set headers - if access_token: + opener = build_opener() + # Here "access_token" indeed is Azure DevOps PAT token. + # "token" should be the actual bearer token for Azure DevOps. + # PAT should be deprecated. + # Below logic is to handle both cases for smooth transition from PAT to bearer token. + if token: + opener.addheaders = [ + ( + "Authorization", + "Bearer {}".format(token) + ) + ] + elif access_token: opener = build_opener() opener.addheaders = [ - ('Authorization', - 'Basic {}'.format(base64.b64encode(access_token.encode('utf-8')).decode('utf-8')))] - install_opener(opener) + ( + 'Authorization', + 'Basic {}'.format(base64.b64encode(access_token.encode('utf-8')).decode('utf-8')) + ) + ] + install_opener(opener) urlretrieve(url, filename, reporthook) print('\nDownload finished!') break @@ -172,7 +196,9 @@ def main(): parser.add_argument('--url_prefix', metavar='url_prefix', type=str, default='mssonic/build', help='url prefix') parser.add_argument('--access_token', metavar='access_token', type=str, - default='', nargs='?', const='', required=False, help='access token') + default='', nargs='?', const='', required=False, help='access token (PAT)') + parser.add_argument('--token', metavar='token', type=str, + default='', nargs='?', const='', required=False, help='bearer token') args = parser.parse_args() @@ -193,10 +219,11 @@ def main(): (dl_url, artifact_size) = get_download_url(buildid, artifact_name, url_prefix=args.url_prefix, - access_token=args.access_token) + access_token=args.access_token, + token=args.token) download_artifacts(dl_url, args.content, args.platform, - buildid, args.num_asic, access_token=args.access_token) + buildid, args.num_asic, access_token=args.access_token, token=args.token) if __name__ == '__main__': diff --git a/tests/show_techsupport/test_auto_techsupport.py b/tests/show_techsupport/test_auto_techsupport.py index ff77a39327..db44872653 100644 --- a/tests/show_techsupport/test_auto_techsupport.py +++ b/tests/show_techsupport/test_auto_techsupport.py @@ -12,6 +12,7 @@ from tests.common.utilities import wait_until from tests.common.multibranch.cli import SonicCli from dateutil.parser import ParserError +from tests.common.plugins.loganalyzer import DisableLogrotateCronContext try: import allure @@ -288,8 +289,9 @@ def test_sanity(self, cleanup_list): Force log rotate - because in some cases, when there's no file older than since, there will be no syslog file in techsupport dump """ - with allure.step('Rotate logs'): - self.duthost.shell('/usr/sbin/logrotate -f /etc/logrotate.conf > /dev/null 2>&1') + with DisableLogrotateCronContext(self.duthost): + with allure.step('Rotate logs'): + self.duthost.shell('/usr/sbin/logrotate -f /etc/logrotate.conf > /dev/null 2>&1') with allure.step('Validate since value: {}'.format(since_value)): with allure.step('Set since value to: {}'.format(since_value)): diff --git a/tests/show_techsupport/test_techsupport.py b/tests/show_techsupport/test_techsupport.py index 43a7f2053e..087dcfba03 100644 --- a/tests/show_techsupport/test_techsupport.py +++ b/tests/show_techsupport/test_techsupport.py @@ -528,7 +528,7 @@ def commands_to_check(duthosts, enum_rand_one_per_hwsku_frontend_hostname): ) # Remove /proc/dma for armh elif duthost.facts["asic_type"] == "marvell": - if 'armhf-' in duthost.facts["platform"]: + if 'armhf-' in duthost.facts["platform"] or 'arm64-' in duthost.facts["platform"]: cmds.copy_proc_files.remove("/proc/dma") return cmds_to_check diff --git a/tests/snappi_tests/DockerCommands.md b/tests/snappi_tests/DockerCommands.md new file mode 100644 index 0000000000..0d371ac7d3 --- /dev/null +++ b/tests/snappi_tests/DockerCommands.md @@ -0,0 +1,51 @@ +# Prerequisites +* An Ubuntu Linux box +* The sonic docker image in your home directory. + * Pre-built sonic-mgmt can also be downloaded from [here](https://sonic-jenkins.westus2.cloudapp.azure.com/job/bldenv/job/docker-sonic-mgmt/lastSuccessfulBuild/artifact/sonic-buildimage/target/docker-sonic-mgmt.gz) +* Basic knowledge of docker commands. +* Docker-tools should be there installed in your system. +# sonic-mgmt docker environment preparation: useful commands (for Ubuntu system) +**Installing docker**
+``sudo apt-get update``
+``sudo apt-get remove docker docker-engine docker.io``
+``sudo apt install docker.io``
+``sudo systemctl start docker``
+``sudo systemctl enable docker``

+**Unzip sonic Image**
+``gzip -d docker-sonic-mgmt.gz``

+**Load the docker Image**
+``sudo docker images``
+``sudo docker load -i docker-sonic-mgmt``
+``sudo docker run -it --name sonic docker-sonic-mgmt``

+**Stopping a docer session**
+``sudo docker stop sonic``

+**Reconnect to a stopped docer session**
+``sudo docker start -i sonic``

+**When you are done you may remove the image sonic**
+``sudo docker rm sonic``

+**Remove docker by image Id**
+``sudo docker rmi -f ``

+**Running a sonic docker with local directoy mounted in it.**
+``sudo docker run -it --name sonic --privileged -v /home/ubuntu/adhar/:/var/johnar/adhar --workdir /var/johnar/adhar --user johnar:gjohnar docker-sonic-mgmt``

+ + +# How to run a docker with a port number +**Run a docker container with port number -p**
+* -itd will run docker in a detached state, I'm using port 2222 you can use any port
+``sudo docker run -itd --name sonic -p 2222:22 docker-sonic-mgmt``

+ +**Enter the docker container using exec**
+``sudo docker exec -it sonic bash``

+ +**Check ssh service is running inside the docker**
+``johnar@1ed3a9afe70f:~$ service --status-all``

+ +**If ssh service is not running start ssh**
+``johnar@1ed3a9afe70f:~$ sudo service ssh start``

+ +**update johnar user passwd**
+* update passwd of your choice +``johnar@1ed3a9afe70f:~$ sudo passwd johnar``

+ +**use ssh from any machine in the network to login to docker directly**
+``ssh johnar@10.39.71.246 -p 2222`` diff --git a/tests/snappi_tests/README.md b/tests/snappi_tests/README.md new file mode 100644 index 0000000000..49456a4ea5 --- /dev/null +++ b/tests/snappi_tests/README.md @@ -0,0 +1,70 @@ +# SONiC testbed management infrastructure +* The SONiC testbed consists of multiple interrelated configuration files that are difficult and error-prone edit manually. +* They are not frequently modified and only handful of persons like owners/admin has authority to modify them. But creating the initial setup or retrieving a broken setup is more challenging task for them. +* User scripts runs inside the docker container loads and access these files, and if the container is corrupted or crashed these files will be lost and will be difficult to get back easily. This will be challenging time for the user who doesn’t have the knowledge of interrelationship of the files. +# So how do we onboard engineer to write tests? +* Keep the testbed files in the separate repository outside the SONiC docker image. +* Provision the engineers to keep the code in their local machine and mount them while loading the docker container. So, code will be in the local directory and won’t get lost if the container is wrecked. +* Give the engineer a script to build the testbed from the stored files in the repository. + +# Workflows +Before going to the work flows please look into the [basic docker commands to create the sonic-mgmt environment](DockerCommands.md). +Also before getting invloved into any of the workflow1 or workflow2 please make sure that you have loaded the sonic docker image to be executed using locker load command. +```sudo docker load -i docker-sonic-mgmt``` +### workflow1 +* Fork the sonic-mgmt(https://github.com/Azure/sonic-mgmt.git) repo. + * Make sure you clone the forked version from your repo + - Ex: git clone https://github.com/sonic-net/sonic-mgmt +* load the docker image such that it mounts sonic-mgmt inside the container. + * Make sure the path is matching the criteria + * sudo docker run -it --name sonic --privileged -v /home/ubuntu/sonic-mgmt/:/var/johnar/sonic-mgmt --user johnar:gjohnar docker-sonic-mgmt +* Install Snappi packages + * python -m pip install --upgrade "snappi==0.9.1" + * python -m pip install --upgrade "snappi[convergence]==0.4.1" + * python -m pip install --upgrade "snappi[ixnetwork]==0.9.1" +* Mention the topology details in the following files + - ansible/files/graph_groups.yml + - ansible/files/sonic_snappi-sonic_devices.csv + - ansible/files/sonic_snappi-sonic_links.csv + - ansible/group_vars/snappi-sonic/secrets.yml + - ansible/group_vars/snappi-sonic/snappi-sonic.yml + - ansible/snappi-sonic + - ansible/testbed.csv +* Run the test + * cd ~/sonic-mgmt/tests/ + * Add environment variables + * export ANSIBLE_CONFIG=../ansible + * export ANSIBLE_LIBRARY=../ansible + * Run Single-Dut case + * py.test --inventory ../ansible/snappi-sonic --host-pattern sonic-s6100-dut1 --testbed vms-snappi-sonic --testbed_file ../ansible/testbed.csv --show-capture=stdout --log-cli-level info --showlocals -ra --allow_recover --skip_sanity --disable_loganalyzer test_pretest.py + * Run Multi-Dut case + * py.test --inventory ../ansible/snappi-sonic --host-pattern all --testbed vms-snappi-sonic-multidut --testbed_file ../ansible/testbed.csv --show-capture=stdout --log-cli-level info --showlocals -ra --allow_recover --skip_sanity --disable_loganalyzer test_pretest.py + * In this workflow your test script or code will remain intact even if docker image is destroyed unintentionally since you are actually keeping the code in the (mounted) local directory. + +### workflow2 +* Simply load the docker image no mounts of local folders are required. + * sudo docker run -it --name sonic docker-sonic-mgmt +* Inside the container clone the forked version of sonic-mgmt(https://github.com/Azure/sonic-mgmt.git) + - Ex: git clone https://github.com/sonic-net/sonic-mgmt +* Install Snappi packages + * python -m pip install --upgrade "snappi==0.9.1" + * python -m pip install --upgrade "snappi[convergence]==0.4.1" + * python -m pip install --upgrade "snappi[ixnetwork]==0.9.1" +* Mention the topology details in the following files (create the files if not present already) + - ansible/files/graph_groups.yml + - ansible/files/sonic_snappi-sonic_devices.csv + - ansible/files/sonic_snappi-sonic_links.csv + - ansible/group_vars/snappi-sonic/secrets.yml + - ansible/group_vars/snappi-sonic/snappi-sonic.yml + - ansible/snappi-sonic + - ansible/testbed.csv +* Run the test + * cd ~/sonic-mgmt/tests/ + * Add environment variables + * export ANSIBLE_CONFIG=../ansible + * export ANSIBLE_LIBRARY=../ansible + * Run Single-Dut case + * py.test --inventory ../ansible/snappi-sonic --host-pattern sonic-s6100-dut1 --testbed vms-snappi-sonic --testbed_file ../ansible/testbed.csv --show-capture=stdout --log-cli-level info --showlocals -ra --allow_recover --skip_sanity --disable_loganalyzer test_pretest.py + * Run Multi-Dut case + * py.test --inventory ../ansible/snappi-sonic --host-pattern all --testbed vms-snappi-sonic-multidut --testbed_file ../ansible/testbed.csv --show-capture=stdout --log-cli-level info --showlocals -ra --allow_recover --skip_sanity --disable_loganalyzer test_pretest.py +* In this workflow if you make certain local change inside the folder ~/sonic-mgmt/ that will not be saved if the container got corrupted somehow. diff --git a/tests/snappi_tests/bgp/files/bgp_test_gap_helper.py b/tests/snappi_tests/bgp/files/bgp_test_gap_helper.py index 4d2221a281..d00bcf8c11 100644 --- a/tests/snappi_tests/bgp/files/bgp_test_gap_helper.py +++ b/tests/snappi_tests/bgp/files/bgp_test_gap_helper.py @@ -1,7 +1,6 @@ from tabulate import tabulate from tests.common.utilities import (wait, wait_until) from tests.common.helpers.assertions import pytest_assert -import json import logging logger = logging.getLogger(__name__) @@ -157,31 +156,38 @@ def duthost_bgp_3port_config(duthost, logger.info('Configuring %s to PortChannel%s with IPs %s,%s' % (tgen_ports[i]['peer_port'], i + 1, tgen_ports[i]['peer_ip'], tgen_ports[i]['peer_ipv6'])) duthost.shell(portchannel_config) - logger.info('Configuring BGP in config_db.json') - bgp_neighbors = dict() + + bgp_config = ( + "vtysh " + "-c 'configure terminal' " + "-c 'router bgp %s' " + "-c 'no bgp ebgp-requires-policy' " + "-c 'bgp bestpath as-path multipath-relax' " + "-c 'maximum-paths %s' " + "-c 'exit' " + ) + bgp_config %= (DUT_AS_NUM, port_count-1) + duthost.shell(bgp_config) + for i in range(1, port_count): - bgp_neighbors[tgen_ports[i]['ipv6']] = {"rrclient": "0", "name": "ARISTA08T0", - "local_addr": tgen_ports[i]['peer_ipv6'], - "nhopself": "0", "holdtime": "90", - "asn": TGEN_AS_NUM, "keepalive": "30"} - bgp_neighbors[tgen_ports[i]['ip']] = {"rrclient": "0", "name": "ARISTA08T0", - "local_addr": tgen_ports[i]['peer_ip'], - "nhopself": "0", "holdtime": "90", "asn": TGEN_AS_NUM, "keepalive": "30"} - - cdf = json.loads(duthost.shell("sonic-cfggen -d --print-data")['stdout']) - for neighbor, neighbor_info in list(bgp_neighbors.items()): - cdf["BGP_NEIGHBOR"][neighbor] = neighbor_info - - cdf["DEVICE_METADATA"]['localhost']['bgp_asn'] = DUT_AS_NUM - with open("/tmp/sconfig_db.json", 'w') as fp: - json.dump(cdf, fp, indent=4) - duthost.copy(src="/tmp/sconfig_db.json", dest="/tmp/config_db_temp.json") - cdf = json.loads(duthost.shell("sonic-cfggen -j /tmp/config_db_temp.json --print-data")['stdout']) - logger.info(cdf) - duthost.command("sudo cp {} {} \n".format("/tmp/config_db_temp.json", "/etc/sonic/config_db.json")) - logger.info('Reloading config to apply BGP config') - duthost.shell("sudo config reload -y \n") - wait(TIMEOUT + 20, "For Config to reload \n") + bgp_config_neighbor = ( + "vtysh " + "-c 'configure terminal' " + "-c 'router bgp %s' " + "-c 'neighbor %s remote-as %s' " + "-c 'address-family ipv4 unicast' " + "-c 'neighbor %s activate' " + "-c 'neighbor %s remote-as %s' " + "-c 'address-family ipv6 unicast' " + "-c 'neighbor %s activate' " + "-c 'exit' " + ) + bgp_config_neighbor %= ( + DUT_AS_NUM, tgen_ports[i]['ip'], TGEN_AS_NUM, tgen_ports[i]['ip'], + tgen_ports[i]['ipv6'], TGEN_AS_NUM, tgen_ports[i]['ipv6']) + logger.info('Configuring BGP v4 Neighbor {}, v6 Neighbor {}'.format(tgen_ports[i]['ip'], + tgen_ports[i]['ipv6'])) + duthost.shell(bgp_config_neighbor) def duthost_bgp_scalability_config(duthost, tgen_ports, multipath): @@ -221,35 +227,38 @@ def duthost_bgp_scalability_config(duthost, tgen_ports, multipath): tgen_ports[i]['ipv6_prefix']) logger.info('Configuring %s to PortChannel%s' % (tgen_ports[i]['peer_port'], i + 1)) duthost.shell(portchannel_config) - bgp_neighbors = dict() - logger.info('Configuring BGP in config_db.json') - ''' - bgp_neighbors = {tgen_ports[1]['ipv6']: {"rrclient": "0", "name": "ARISTA08T0", - "local_addr": tgen_ports[1]['peer_ipv6'], "nhopself": "0", - "holdtime": "90", "asn": TGEN_AS_NUM,"keepalive": "30"}, - tgen_ports[1]['ip']: {"rrclient": "0", "name": "ARISTA08T0", - "local_addr": tgen_ports[1]['peer_ip'], "nhopself": "0", - "holdtime": "90", "asn": TGEN_AS_NUM,"keepalive": "30"}} - ''' - bgp_neighbors[tgen_ports[1]['ipv6']] = {"rrclient": "0", "name": "ARISTA08T0", - "local_addr": tgen_ports[1]['peer_ipv6'], - "nhopself": "0", "holdtime": "90", "asn": TGEN_AS_NUM, "keepalive": "30"} - bgp_neighbors[tgen_ports[1]['ip']] = {"rrclient": "0", "name": "ARISTA08T0", - "local_addr": tgen_ports[1]['peer_ip'], - "nhopself": "0", "holdtime": "90", "asn": TGEN_AS_NUM, "keepalive": "30"} - cdf = json.loads(duthost.shell("sonic-cfggen -d --print-data")['stdout']) - for neighbor, neighbor_info in list(bgp_neighbors.items()): - cdf["BGP_NEIGHBOR"][neighbor] = neighbor_info - cdf["DEVICE_METADATA"]['localhost']['bgp_asn'] = DUT_AS_NUM - with open("/tmp/sconfig_db.json", 'w') as fp: - json.dump(cdf, fp, indent=4) - duthost.copy(src="/tmp/sconfig_db.json", dest="/tmp/config_db_temp.json") - cdf = json.loads(duthost.shell("sonic-cfggen -j /tmp/config_db_temp.json --print-data")['stdout']) - logger.info(cdf) - duthost.command("sudo cp {} {} \n".format("/tmp/config_db_temp.json", "/etc/sonic/config_db.json")) - logger.info('Reloading config to apply BGP config') - duthost.shell("sudo config reload -f -y \n") - wait(TIMEOUT + 20, "For Config to reload \n") + + bgp_config = ( + "vtysh " + "-c 'configure terminal' " + "-c 'router bgp %s' " + "-c 'no bgp ebgp-requires-policy' " + "-c 'bgp bestpath as-path multipath-relax' " + "-c 'maximum-paths %s' " + "-c 'exit' " + ) + bgp_config %= (DUT_AS_NUM, port_count-1) + duthost.shell(bgp_config) + + for i in range(1, port_count): + bgp_config_neighbor = ( + "vtysh " + "-c 'configure terminal' " + "-c 'router bgp %s' " + "-c 'neighbor %s remote-as %s' " + "-c 'address-family ipv4 unicast' " + "-c 'neighbor %s activate' " + "-c 'neighbor %s remote-as %s' " + "-c 'address-family ipv6 unicast' " + "-c 'neighbor %s activate' " + "-c 'exit' " + ) + bgp_config_neighbor %= ( + DUT_AS_NUM, tgen_ports[i]['ip'], TGEN_AS_NUM, tgen_ports[i]['ip'], + tgen_ports[i]['ipv6'], TGEN_AS_NUM, tgen_ports[i]['ipv6']) + logger.info('Configuring BGP v4 Neighbor {}, v6 Neighbor {}'.format(tgen_ports[i]['ip'], + tgen_ports[i]['ipv6'])) + duthost.shell(bgp_config_neighbor) def __tgen_bgp_config(cvg_api, diff --git a/tests/snappi_tests/bgp/test_bgp_local_link_failover.py b/tests/snappi_tests/bgp/test_bgp_local_link_failover.py index 02cd88ede7..28eda635f5 100644 --- a/tests/snappi_tests/bgp/test_bgp_local_link_failover.py +++ b/tests/snappi_tests/bgp/test_bgp_local_link_failover.py @@ -5,6 +5,8 @@ conn_graph_facts, fanout_graph_facts) import pytest +pytestmark = [pytest.mark.topology('tgen')] + @pytest.mark.parametrize('multipath', [2]) @pytest.mark.parametrize('convergence_test_iterations', [1]) diff --git a/tests/snappi_tests/bgp/test_bgp_remote_link_failover.py b/tests/snappi_tests/bgp/test_bgp_remote_link_failover.py index 215021647c..d12726dc48 100755 --- a/tests/snappi_tests/bgp/test_bgp_remote_link_failover.py +++ b/tests/snappi_tests/bgp/test_bgp_remote_link_failover.py @@ -5,6 +5,8 @@ conn_graph_facts, fanout_graph_facts) import pytest +pytestmark = [pytest.mark.topology('tgen')] + @pytest.mark.parametrize('multipath', [2]) @pytest.mark.parametrize('convergence_test_iterations', [1]) diff --git a/tests/snappi_tests/bgp/test_bgp_rib_in_capacity.py b/tests/snappi_tests/bgp/test_bgp_rib_in_capacity.py index f58686f577..62581578d8 100644 --- a/tests/snappi_tests/bgp/test_bgp_rib_in_capacity.py +++ b/tests/snappi_tests/bgp/test_bgp_rib_in_capacity.py @@ -5,6 +5,8 @@ conn_graph_facts, fanout_graph_facts) import pytest +pytestmark = [pytest.mark.topology('tgen')] + @pytest.mark.parametrize('multipath', [2]) @pytest.mark.parametrize('start_value', [1000]) diff --git a/tests/snappi_tests/bgp/test_bgp_rib_in_convergence.py b/tests/snappi_tests/bgp/test_bgp_rib_in_convergence.py index afed29098b..f114dfe1b8 100644 --- a/tests/snappi_tests/bgp/test_bgp_rib_in_convergence.py +++ b/tests/snappi_tests/bgp/test_bgp_rib_in_convergence.py @@ -5,6 +5,8 @@ conn_graph_facts, fanout_graph_facts) import pytest +pytestmark = [pytest.mark.topology('tgen')] + @pytest.mark.parametrize('multipath', [2]) @pytest.mark.parametrize('convergence_test_iterations', [1]) diff --git a/tests/snappi_tests/ecn/files/helper.py b/tests/snappi_tests/ecn/files/helper.py index 067afdf070..91f6911d57 100644 --- a/tests/snappi_tests/ecn/files/helper.py +++ b/tests/snappi_tests/ecn/files/helper.py @@ -1,4 +1,5 @@ import logging +import time from tests.common.helpers.assertions import pytest_assert from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ @@ -9,14 +10,14 @@ from tests.common.snappi_tests.common_helpers import config_wred, \ enable_ecn, config_ingress_lossless_buffer_alpha, stop_pfcwd, disable_packet_aging, \ config_capture_pkt, traffic_flow_mode, calc_pfc_pause_flow_rate -from tests.common.snappi_tests.read_pcap import get_ip_pkts +from tests.common.snappi_tests.read_pcap import get_ipv4_pkts from tests.common.snappi_tests.traffic_generation import setup_base_traffic_config, generate_test_flows, \ generate_pause_flows, run_traffic logger = logging.getLogger(__name__) -EXP_DURATION_SEC = 1 -DATA_START_DELAY_SEC = 0.1 +EXP_DURATION_SEC = 2.1 +DATA_START_DELAY_SEC = 1 PAUSE_FLOW_NAME = 'Pause Storm' DATA_FLOW_NAME = 'Data Flow' @@ -77,6 +78,9 @@ def run_ecn_test(api, pytest_assert(config_result is True, 'Failed to configure PFC threshold to 8') + logger.info("Waiting on ECN and dynamic buffer configuration to take effect. Sleeping for 10 seconds.") + time.sleep(10) + # Get the ID of the port to test port_id = get_dut_port_id(dut_hostname=duthost.hostname, dut_port=dut_port, @@ -158,6 +162,6 @@ def run_ecn_test(api, exp_dur_sec=EXP_DURATION_SEC, snappi_extra_params=snappi_extra_params) - result.append(get_ip_pkts(snappi_extra_params.packet_capture_file + ".pcapng")) + result.append(get_ipv4_pkts(snappi_extra_params.packet_capture_file + ".pcapng")) return result diff --git a/tests/snappi_tests/ecn/test_red_accuracy_with_snappi.py b/tests/snappi_tests/ecn/test_red_accuracy_with_snappi.py index 62cc74f477..1fd8f73d9d 100644 --- a/tests/snappi_tests/ecn/test_red_accuracy_with_snappi.py +++ b/tests/snappi_tests/ecn/test_red_accuracy_with_snappi.py @@ -63,10 +63,10 @@ def test_red_accuracy(request, snappi_extra_params = SnappiTestParams() snappi_extra_params.packet_capture_type = packet_capture.IP_CAPTURE snappi_extra_params.is_snappi_ingress_port_cap = True - snappi_extra_params.ecn_params = {'kmin': 500000, 'kmax': 2000000, 'pmax': 5} + snappi_extra_params.ecn_params = {'kmin': 500000, 'kmax': 900000, 'pmax': 5} data_flow_pkt_size = 1024 - data_flow_pkt_count = 2100 - num_iterations = 100 + data_flow_pkt_count = 910 + num_iterations = 5 logger.info("Running ECN red accuracy test with ECN params: {}".format(snappi_extra_params.ecn_params)) logger.info("Running ECN red accuracy test for {} iterations".format(num_iterations)) diff --git a/tests/snappi_tests/lacp/test_add_remove_link_from_dut.py b/tests/snappi_tests/lacp/test_add_remove_link_from_dut.py index de0675469f..6c8d9e40e8 100755 --- a/tests/snappi_tests/lacp/test_add_remove_link_from_dut.py +++ b/tests/snappi_tests/lacp/test_add_remove_link_from_dut.py @@ -6,6 +6,8 @@ conn_graph_facts, fanout_graph_facts) import pytest +pytestmark = [pytest.mark.topology('tgen')] + @pytest.mark.parametrize('port_count', [4]) @pytest.mark.parametrize('number_of_routes', [1000]) diff --git a/tests/snappi_tests/lacp/test_add_remove_link_physically.py b/tests/snappi_tests/lacp/test_add_remove_link_physically.py index 0683f79149..313ec1c8d6 100755 --- a/tests/snappi_tests/lacp/test_add_remove_link_physically.py +++ b/tests/snappi_tests/lacp/test_add_remove_link_physically.py @@ -6,6 +6,8 @@ conn_graph_facts, fanout_graph_facts) import pytest +pytestmark = [pytest.mark.topology('tgen')] + @pytest.mark.parametrize('port_count', [4]) @pytest.mark.parametrize('number_of_routes', [1000]) diff --git a/tests/snappi_tests/lacp/test_lacp_timers_effect.py b/tests/snappi_tests/lacp/test_lacp_timers_effect.py index 8acc81543a..7ced05d159 100644 --- a/tests/snappi_tests/lacp/test_lacp_timers_effect.py +++ b/tests/snappi_tests/lacp/test_lacp_timers_effect.py @@ -6,6 +6,8 @@ conn_graph_facts, fanout_graph_facts) import pytest +pytestmark = [pytest.mark.topology('tgen')] + @pytest.mark.parametrize('port_count', [4]) @pytest.mark.parametrize('number_of_routes', [1000]) diff --git a/tests/snappi_tests/multidut/bgp/__init__.py b/tests/snappi_tests/multidut/bgp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/snappi_tests/multidut/bgp/files/__init__.py b/tests/snappi_tests/multidut/bgp/files/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py b/tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py new file mode 100755 index 0000000000..b77345db20 --- /dev/null +++ b/tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py @@ -0,0 +1,1769 @@ +import logging +import random +import paramiko +import json +import time +import math +import pexpect +from ixnetwork_restpy import SessionAssistant +from ixnetwork_restpy.testplatform.testplatform import TestPlatform +from ixnetwork_restpy.assistants.statistics.statviewassistant import StatViewAssistant +from tabulate import tabulate +from statistics import mean +from tests.common.utilities import (wait, wait_until) # noqa: F401 +from tests.common.helpers.assertions import pytest_assert # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import create_ip_list # noqa: F401 +from tests.snappi_tests.variables import T1_SNAPPI_AS_NUM, T2_SNAPPI_AS_NUM, T1_DUT_AS_NUM, T2_DUT_AS_NUM, t1_ports, \ + t2_uplink_portchannel_members, t1_t2_dut_ipv4_list, v4_prefix_length, v6_prefix_length, \ + t1_t2_dut_ipv6_list, t1_t2_snappi_ipv4_list, portchannel_count, \ + t1_t2_snappi_ipv6_list, t2_dut_portchannel_ipv4_list, t2_dut_portchannel_ipv6_list, \ + snappi_portchannel_ipv4_list, snappi_portchannel_ipv6_list, AS_PATHS, \ + BGP_TYPE, t1_side_interconnected_port, t2_side_interconnected_port, router_ids, \ + snappi_community_for_t1, snappi_community_for_t2, SNAPPI_TRIGGER, DUT_TRIGGER, \ + fanout_presence, t2_uplink_fanout_info # noqa: F401 + +logger = logging.getLogger(__name__) +total_routes = 0 +fanout_uplink_snappi_info = [] + + +def run_dut_configuration(snappi_extra_params): + """ + Configures the dut for the test + + snappi_extra_params (SnappiTestParams obj): additional parameters for Snappi traffic + """ + duthost1 = snappi_extra_params.multi_dut_params.duthost1 + duthost2 = snappi_extra_params.multi_dut_params.duthost2 + duthost3 = snappi_extra_params.multi_dut_params.duthost3 + duthosts = [duthost1, duthost2, duthost3] + test_name = snappi_extra_params.test_name + snappi_ports = snappi_extra_params.multi_dut_params.multi_dut_ports + + duthost_bgp_config(duthosts, + snappi_ports, + test_name) + + +def run_bgp_outbound_uplink_blackout_test(api, + snappi_extra_params, + creds): + """ + Run outbound test for uplink blackout + Args: + api (pytest fixture): snappi API + creds (dict): DUT credentials + snappi_extra_params (SnappiTestParams obj): additional parameters for Snappi traffic + """ + + if snappi_extra_params is None: + snappi_extra_params = SnappiTestParams() # noqa F821 + + duthost1 = snappi_extra_params.multi_dut_params.duthost1 + duthost2 = snappi_extra_params.multi_dut_params.duthost2 + duthost3 = snappi_extra_params.multi_dut_params.duthost3 + duthosts = [duthost1, duthost2, duthost3] + route_ranges = snappi_extra_params.ROUTE_RANGES + snappi_ports = snappi_extra_params.multi_dut_params.multi_dut_ports + blackout_percentage = snappi_extra_params.multi_dut_params.BLACKOUT_PERCENTAGE + iteration = snappi_extra_params.iteration + test_name = snappi_extra_params.test_name + + """ Create snappi config """ + for route_range in route_ranges: + traffic_type = [] + for key, value in route_range.items(): + traffic_type.append(key) + snappi_bgp_config = __snappi_bgp_config(api, + duthosts, + snappi_ports, + traffic_type, + route_range) + + get_convergence_for_blackout(duthosts, + api, + snappi_bgp_config, + traffic_type, + iteration, + blackout_percentage, + route_range, + test_name, + creds) + + +def run_bgp_outbound_tsa_tsb_test(api, + snappi_extra_params, + creds, + is_supervisor): + """ + Run outbound test with TSA TSB on the dut + + Args: + api (pytest fixture): snappi API + snappi_extra_params (SnappiTestParams obj): additional parameters for Snappi traffic + """ + if snappi_extra_params is None: + snappi_extra_params = SnappiTestParams() # noqa F821 + + duthost1 = snappi_extra_params.multi_dut_params.duthost1 + duthost2 = snappi_extra_params.multi_dut_params.duthost2 + duthost3 = snappi_extra_params.multi_dut_params.duthost3 + duthost4 = snappi_extra_params.multi_dut_params.duthost4 + duthosts = [duthost1, duthost2, duthost3, duthost4] + route_ranges = snappi_extra_params.ROUTE_RANGES + snappi_ports = snappi_extra_params.multi_dut_params.multi_dut_ports + device_name = snappi_extra_params.device_name + iteration = snappi_extra_params.iteration + test_name = snappi_extra_params.test_name + + """ Create snappi config """ + for route_range in route_ranges: + traffic_type = [] + for key, value in route_range.items(): + traffic_type.append(key) + snappi_bgp_config = __snappi_bgp_config(api, + duthosts, + snappi_ports, + traffic_type, + route_range) + + get_convergence_for_tsa_tsb(duthosts, + api, + snappi_bgp_config, + traffic_type, + iteration, + device_name, + route_range, + test_name, + creds, + is_supervisor) + + +def run_bgp_outbound_process_restart_test(api, + creds, + snappi_extra_params): + """ + Run Local link failover test + + Args: + api (pytest fixture): snappi API + creds (dict): DUT credentials + snappi_extra_params (SnappiTestParams obj): additional parameters for Snappi traffic + """ + + if snappi_extra_params is None: + snappi_extra_params = SnappiTestParams() # noqa F821 + + duthost1 = snappi_extra_params.multi_dut_params.duthost1 + duthost2 = snappi_extra_params.multi_dut_params.duthost2 + duthost3 = snappi_extra_params.multi_dut_params.duthost3 + duthosts = [duthost1, duthost2, duthost3] + route_ranges = snappi_extra_params.ROUTE_RANGES + snappi_ports = snappi_extra_params.multi_dut_params.multi_dut_ports + process_names = snappi_extra_params.multi_dut_params.process_names + host_name = snappi_extra_params.multi_dut_params.host_name + iteration = snappi_extra_params.iteration + test_name = snappi_extra_params.test_name + + """ Create bgp config on dut """ + duthost_bgp_config(duthosts, + snappi_ports, + test_name) + + """ Create snappi config """ + for route_range in route_ranges: + traffic_type = [] + for key, value in route_range.items(): + traffic_type.append(key) + snappi_bgp_config = __snappi_bgp_config(api, + duthosts, + snappi_ports, + traffic_type, + route_range) + + get_convergence_for_process_flap(duthosts, + api, + snappi_bgp_config, + traffic_type, + iteration, + process_names, + host_name, + route_range, + test_name, + creds) + + +def run_bgp_outbound_link_flap_test(api, + creds, + snappi_extra_params): + """ + Run Local link failover test + + Args: + api (pytest fixture): snappi API + snappi_extra_params (SnappiTestParams obj): additional parameters for Snappi traffic + """ + + if snappi_extra_params is None: + snappi_extra_params = SnappiTestParams() # noqa F821 + + duthost1 = snappi_extra_params.multi_dut_params.duthost1 + duthost2 = snappi_extra_params.multi_dut_params.duthost2 + duthost3 = snappi_extra_params.multi_dut_params.duthost3 + duthosts = [duthost1, duthost2, duthost3] + route_ranges = snappi_extra_params.ROUTE_RANGES + snappi_ports = snappi_extra_params.multi_dut_params.multi_dut_ports + iteration = snappi_extra_params.iteration + flap_details = snappi_extra_params.multi_dut_params.flap_details + test_name = snappi_extra_params.test_name + + """ Create bgp config on dut """ + duthost_bgp_config(duthosts, + snappi_ports, + test_name) + + """ Create snappi config """ + for route_range in route_ranges: + traffic_type = [] + for key, value in route_range.items(): + traffic_type.append(key) + snappi_bgp_config = __snappi_bgp_config(api, + duthosts, + snappi_ports, + traffic_type, + route_range) + + get_convergence_for_link_flap(duthosts, + api, + snappi_bgp_config, + flap_details, + traffic_type, + iteration, + route_range, + test_name, + creds) + + +def duthost_bgp_config(duthosts, + snappi_ports, + test_name): + """ + Configures BGP on the DUT with N-1 ecmp + + Args: + duthosts (pytest fixture): duthosts fixture + snappi_ports (pytest fixture): Ports mapping info of T0 testbed + test_name: Name of the test + """ + logger.info('\n') + logger.info('--------------- T1 Snappi Section --------------------') + logger.info('\n') + t1_config_db = json.loads(duthosts[0].shell("sonic-cfggen -d --print-data")['stdout']) + interfaces = dict() + loopback_interfaces = dict() + loopback_interfaces.update({"Loopback0": {}}) + loopback_interfaces.update({"Loopback0|1.1.1.1/32": {}}) + loopback_interfaces.update({"Loopback0|1::1/128": {}}) + for index, custom_port in enumerate(t1_ports[duthosts[0].hostname]): + interface_name = {custom_port: {}} + v4_interface = {f"{custom_port}|{t1_t2_dut_ipv4_list[index]}/{v4_prefix_length}": {}} + v6_interface = {f"{custom_port}|{t1_t2_dut_ipv6_list[index]}/{v6_prefix_length}": {}} + interfaces.update(interface_name) + interfaces.update(v4_interface) + interfaces.update(v6_interface) + logger.info('Configuring IPs {}/{} , {}/{} on {} in {}'. + format(t1_t2_dut_ipv4_list[index], v4_prefix_length, + t1_t2_dut_ipv6_list[index], v6_prefix_length, custom_port, duthosts[0].hostname)) + + bgp_neighbors = dict() + device_neighbors = dict() + device_neighbor_metadatas = dict() + for index, custom_port in enumerate(t1_ports[duthosts[0].hostname]): + for snappi_port in snappi_ports: + if custom_port == snappi_port['peer_port'] and snappi_port['peer_device'] == duthosts[0].hostname: + bgp_neighbor = \ + { + t1_t2_snappi_ipv4_list[index]: + { + "admin_status": "up", + "asn": T1_SNAPPI_AS_NUM, + "holdtime": "10", + "keepalive": "3", + "local_addr": t1_t2_dut_ipv4_list[index], + "name": "snappi-sonic"+str(index), + "nhopself": "0", + "rrclient": "0" + }, + t1_t2_snappi_ipv6_list[index]: + { + "admin_status": "up", + "asn": T1_SNAPPI_AS_NUM, + "holdtime": "10", + "keepalive": "3", + "local_addr": t1_t2_dut_ipv6_list[index], + "name": "snappi-sonic"+str(index), + "nhopself": "0", + "rrclient": "0" + }, + } + bgp_neighbors.update(bgp_neighbor) + device_neighbor = { + custom_port: + { + "name": "snappi-sonic"+str(index), + "port": "Ethernet1" + } + } + device_neighbors.update(device_neighbor) + device_neighbor_metadata = { + "snappi-sonic"+str(index): + { + "hwsku": "Snappi", + "mgmt_addr": "172.16.149.206", + "type": "ToRRouter" + } + } + device_neighbor_metadatas.update(device_neighbor_metadata) + logger.info('T1 Dut AS Number: {}'.format(T1_DUT_AS_NUM)) + logger.info('T1 side Snappi AS Number: {}'.format(T1_SNAPPI_AS_NUM)) + logger.info('\n') + logger.info('---------------T1 Inter-Connectivity Section --------------------') + logger.info('\n') + index = len(t1_ports[duthosts[0].hostname]) + interface_name = {t1_side_interconnected_port: {}} + v4_interface = {f"{t1_side_interconnected_port}|{t1_t2_dut_ipv4_list[index]}/{v4_prefix_length}": {}} + v6_interface = {f"{t1_side_interconnected_port}|{t1_t2_dut_ipv6_list[index]}/{v6_prefix_length}": {}} + interfaces.update(interface_name) + interfaces.update(v4_interface) + interfaces.update(v6_interface) + logger.info('Configuring IP {}/{} , {}/{} on {} in {} for the T1 interconnectivity'. + format(t1_t2_dut_ipv4_list[index], v4_prefix_length, + t1_t2_dut_ipv6_list[index], v6_prefix_length, t1_side_interconnected_port, + duthosts[0].hostname)) + + logger.info('Configuring BGP in T1 by writing into config_db') + bgp_neighbor = { + t1_t2_snappi_ipv4_list[index]: + { + "admin_status": "up", + "asn": T2_DUT_AS_NUM, + "holdtime": "10", + "keepalive": "3", + "local_addr": t1_t2_dut_ipv4_list[index], + "name": "T2", + "nhopself": "0", + "rrclient": "0" + }, + t1_t2_snappi_ipv6_list[index]: + { + "admin_status": "up", + "asn": T2_DUT_AS_NUM, + "holdtime": "10", + "keepalive": "3", + "local_addr": t1_t2_dut_ipv6_list[index], + "name": "T2", + "nhopself": "0", + "rrclient": "0" + }, + } + bgp_neighbors.update(bgp_neighbor) + device_neighbor = { + t1_side_interconnected_port: + { + "name": "T2", + "port": "Ethernet1" + } + } + device_neighbors.update(device_neighbor) + device_neighbor_metadata = { + "T2": + { + "hwsku": "Sonic-Dut", + "mgmt_addr": "172.16.149.206", + "type": "SpineRouter" + } + } + device_neighbor_metadatas.update(device_neighbor_metadata) + if "INTERFACE" not in t1_config_db.keys(): + t1_config_db["INTERFACE"] = interfaces + else: + t1_config_db["INTERFACE"].update(interfaces) + + if "LOOPBACK_INTERFACE" not in t1_config_db.keys(): + t1_config_db["LOOPBACK_INTERFACE"] = loopback_interfaces + else: + t1_config_db["LOOPBACK_INTERFACE"].update(loopback_interfaces) + + if "BGP_NEIGHBOR" not in t1_config_db.keys(): + t1_config_db["BGP_NEIGHBOR"] = bgp_neighbors + else: + t1_config_db["BGP_NEIGHBOR"].update(bgp_neighbors) + + if "DEVICE_NEIGHBOR" not in t1_config_db.keys(): + t1_config_db["DEVICE_NEIGHBOR"] = device_neighbors + else: + t1_config_db["DEVICE_NEIGHBOR"].update(device_neighbors) + + if 'DEVICE_NEIGHBOR_METADATA' not in t1_config_db.keys(): + t1_config_db["DEVICE_NEIGHBOR_METADATA"] = device_neighbor_metadatas + else: + t1_config_db["DEVICE_NEIGHBOR_METADATA"].update(device_neighbor_metadatas) + + with open("/tmp/temp_config.json", 'w') as fp: + json.dump(t1_config_db, fp, indent=4) + duthosts[0].copy(src="/tmp/temp_config.json", dest="/etc/sonic/config_db.json") + + logger.info('Reloading config_db.json to apply IP and BGP configuration on {}'.format(duthosts[0].hostname)) + pytest_assert('Error' not in duthosts[0].shell("sudo config reload -f -y \n")['stderr'], + 'Error while reloading config in {} !!!!!'.format(duthosts[0].hostname)) + logger.info('Config Reload Successful in {} !!!'.format(duthosts[0].hostname)) + + logger.info('\n') + logger.info('---------------T2 Downlink Inter-Connectivity Section --------------------') + logger.info('\n') + logger.info('T1 Dut AS Number: {}'.format(T1_DUT_AS_NUM)) + logger.info('T2 Dut AS Number: {}'.format(T2_DUT_AS_NUM)) + + interfaces = dict() + loopback_interfaces = dict() + loopback_interfaces.update({"Loopback0": {}}) + loopback_interfaces.update({"Loopback0|2.2.2.2/32": {}}) + loopback_interfaces.update({"Loopback0|2::2/128": {}}) + index = len(t1_ports[duthosts[0].hostname]) + interface_name = {t2_side_interconnected_port['port_name']: {}} + v4_interface = { + f"{t2_side_interconnected_port['port_name']}|{t1_t2_snappi_ipv4_list[index]}/{v4_prefix_length}": {} + } + v6_interface = { + f"{t2_side_interconnected_port['port_name']}|{t1_t2_snappi_ipv6_list[index]}/{v6_prefix_length}": {} + } + interfaces.update(interface_name) + interfaces.update(v4_interface) + interfaces.update(v6_interface) + device_neighbor = { + t2_side_interconnected_port['port_name']: + { + "name": "T1", + "port": "Ethernet1" + } + } + + device_neighbor_metadata = { + "T1": + { + "hwsku": "Sonic-Dut", + "mgmt_addr": t1_t2_dut_ipv4_list[index], + "type": "LeafRouter" + } + } + bgp_neighbor = { + t1_t2_dut_ipv4_list[index]: + { + "admin_status": "up", + "asn": T1_DUT_AS_NUM, + "holdtime": "10", + "keepalive": "3", + "local_addr": t1_t2_snappi_ipv4_list[index], + "name": "T1", + "nhopself": "0", + "rrclient": "0" + }, + t1_t2_dut_ipv6_list[index]: + { + "admin_status": "up", + "asn": T1_DUT_AS_NUM, + "holdtime": "10", + "keepalive": "3", + "local_addr": t1_t2_snappi_ipv6_list[index], + "name": "T1", + "nhopself": "0", + "rrclient": "0" + }, + } + + if t2_side_interconnected_port['asic_value'] is not None: + config_db = 'config_db'+list(t2_side_interconnected_port['asic_value'])[-1]+'.json' + t2_config_db = json.loads(duthosts[2].shell("sonic-cfggen -d -n {} --print-data". + format(t2_side_interconnected_port['asic_value']))['stdout']) + else: + config_db = 'config_db.json' + t2_config_db = json.loads(duthosts[2].shell("sonic-cfggen -d --print-data")['stdout']) + + if "INTERFACE" not in t2_config_db.keys(): + t2_config_db["INTERFACE"] = interfaces + else: + t2_config_db["INTERFACE"].update(interfaces) + logger.info('Configuring IP {}/{} , {}/{} on {} in {} for the T1 interconnectivity'. + format(t1_t2_snappi_ipv4_list[index], v4_prefix_length, + t1_t2_snappi_ipv6_list[index], v6_prefix_length, + t2_side_interconnected_port['port_name'], duthosts[2].hostname)) + if "LOOPBACK_INTERFACE" not in t2_config_db.keys(): + t2_config_db["LOOPBACK_INTERFACE"] = loopback_interfaces + else: + t2_config_db["LOOPBACK_INTERFACE"].update(loopback_interfaces) + + if "DEVICE_NEIGHBOR" not in t2_config_db.keys(): + t2_config_db["DEVICE_NEIGHBOR"] = device_neighbor + else: + t2_config_db["DEVICE_NEIGHBOR"].update(device_neighbor) + + if 'DEVICE_NEIGHBOR_METADATA' not in t2_config_db.keys(): + t2_config_db["DEVICE_NEIGHBOR_METADATA"] = device_neighbor_metadata + else: + t2_config_db["DEVICE_NEIGHBOR_METADATA"].update(device_neighbor_metadata) + + if "BGP_NEIGHBOR" not in t2_config_db.keys(): + t2_config_db["BGP_NEIGHBOR"] = bgp_neighbor + else: + t2_config_db["BGP_NEIGHBOR"].update(bgp_neighbor) + + with open("/tmp/temp_config.json", 'w') as fp: + json.dump(t2_config_db, fp, indent=4) + duthosts[2].copy(src="/tmp/temp_config.json", dest="/etc/sonic/%s" % config_db) + + logger.info('Reloading config_db.json to apply IP and BGP configuration on {}'.format(duthosts[2].hostname)) + + pytest_assert('Error' not in duthosts[2].shell("sudo config reload -f -y \n")['stderr'], + 'Error while reloading config in {} !!!!!'.format(duthosts[2].hostname)) + logger.info('Config Reload Successful in {} !!!'.format(duthosts[2].hostname)) + logger.info('\n') + logger.info('--------------- T2 Uplink - Tgen Section --------------------') + logger.info('\n') + logger.info('T2 Dut AS Number: {}'.format(T2_DUT_AS_NUM)) + logger.info('T2 side Snappi AS Number: {}'.format(T2_SNAPPI_AS_NUM)) + loopback_interfaces = dict() + loopback_interfaces.update({"Loopback0": {}}) + loopback_interfaces.update({"Loopback0|3.3.3.3/32": {}}) + loopback_interfaces.update({"Loopback0|3::3/128": {}}) + index = 0 + index_2 = 0 + for asic_value, portchannel_info in t2_uplink_portchannel_members[duthosts[1].hostname].items(): + bgp_neighbors = dict() + device_neighbors = dict() + device_neighbor_metadatas = dict() + PORTCHANNELS = dict() + PORTCHANNEL_INTERFACES = dict() + PORTCHANNEL_MEMBERS = dict() + if asic_value is not None: + config_db = 'config_db'+list(asic_value)[-1]+'.json' + t2_config_db = json.loads(duthosts[1].shell("sonic-cfggen -d -n {} --print-data". + format(asic_value))['stdout']) + else: + config_db = 'config_db.json' + t2_config_db = json.loads(duthosts[1].shell("sonic-cfggen -d --print-data")['stdout']) + for portchannel, port_set in portchannel_info.items(): + for port in port_set: + device_neighbor = { + port: { + "name": "snappi_"+portchannel, + "port": "snappi_"+port, + } + } + device_neighbors.update(device_neighbor) + MEMBER = {f"{portchannel}|{port}": {}} + PORTCHANNEL_MEMBERS.update(MEMBER) + if 'Portchannel Flap' in test_name: + min_link = len(port_set) + else: + min_link = 1 + PORTCHANNEL = { + portchannel: + { + "admin_status": "up", + "lacp_key": "auto", + "min_links": str(min_link), + "mtu": "9100" + } + } + PORTCHANNELS.update(PORTCHANNEL) + logger.info('\n') + logger.info('Creating {} in {}'.format(portchannel, duthosts[1].hostname)) + logger.info('Setting min_links to {} for {}'.format(min_link, portchannel)) + interface_name = {portchannel: {}} + v4_interface = {f"{portchannel}|{t2_dut_portchannel_ipv4_list[index_2]}/{v4_prefix_length}": {}} + v6_interface = {f"{portchannel}|{t2_dut_portchannel_ipv6_list[index_2]}/{v6_prefix_length}": {}} + PORTCHANNEL_INTERFACES.update(interface_name) + PORTCHANNEL_INTERFACES.update(v4_interface) + PORTCHANNEL_INTERFACES.update(v6_interface) + logger.info('Configuring IPs {}/{} , {}/{} on {} in {}'. + format(t2_dut_portchannel_ipv4_list[index_2], v4_prefix_length, + t2_dut_portchannel_ipv6_list[index_2], v6_prefix_length, + portchannel, duthosts[1].hostname)) + index_2 = index_2 + 1 + for portchannel in portchannel_info: + device_neighbor_metadata = { + "snappi_"+portchannel: + { + "hwsku": "Ixia", + "mgmt_addr": snappi_portchannel_ipv4_list[index], + "type": "AZNGHub" + }, + } + bgp_neighbor = { + snappi_portchannel_ipv4_list[index]: + { + "admin_status": "up", + "asn": T2_SNAPPI_AS_NUM, + "holdtime": "10", + "keepalive": "3", + "local_addr": t2_dut_portchannel_ipv4_list[index], + "name": "snappi_"+portchannel, + "nhopself": "0", + "rrclient": "0" + }, + snappi_portchannel_ipv6_list[index]: + { + "admin_status": "up", + "asn": T2_SNAPPI_AS_NUM, + "holdtime": "10", + "keepalive": "3", + "local_addr": t2_dut_portchannel_ipv6_list[index], + "name": "snappi_"+portchannel, + "nhopself": "0", + "rrclient": "0" + }, + } + bgp_neighbors.update(bgp_neighbor) + device_neighbor_metadatas.update(device_neighbor_metadata) + index = index + 1 + if "LOOPBACK_INTERFACE" not in t2_config_db.keys(): + t2_config_db["LOOPBACK_INTERFACE"] = loopback_interfaces + else: + t2_config_db["LOOPBACK_INTERFACE"].update(loopback_interfaces) + + if "PORTCHANNEL_INTERFACE" not in t2_config_db.keys(): + t2_config_db["PORTCHANNEL_INTERFACE"] = PORTCHANNEL_INTERFACES + else: + t2_config_db["PORTCHANNEL_INTERFACE"].update(PORTCHANNEL_INTERFACES) + + if "PORTCHANNEL" not in t2_config_db.keys(): + t2_config_db["PORTCHANNEL"] = PORTCHANNELS + else: + t2_config_db["PORTCHANNEL"].update(PORTCHANNELS) + + if "PORTCHANNEL_MEMBER" not in t2_config_db.keys(): + t2_config_db["PORTCHANNEL_MEMBER"] = PORTCHANNEL_MEMBERS + else: + t2_config_db["PORTCHANNEL_MEMBER"].update(PORTCHANNEL_MEMBERS) + + if "DEVICE_NEIGHBOR" not in t2_config_db.keys(): + t2_config_db["DEVICE_NEIGHBOR"] = device_neighbors + else: + t2_config_db["DEVICE_NEIGHBOR"].update(device_neighbors) + + if 'DEVICE_NEIGHBOR_METADATA' not in t2_config_db.keys(): + t2_config_db["DEVICE_NEIGHBOR_METADATA"] = device_neighbor_metadatas + else: + t2_config_db["DEVICE_NEIGHBOR_METADATA"].update(device_neighbor_metadatas) + + if "BGP_NEIGHBOR" not in t2_config_db.keys(): + t2_config_db["BGP_NEIGHBOR"] = bgp_neighbors + else: + t2_config_db["BGP_NEIGHBOR"].update(bgp_neighbors) + with open("/tmp/temp_config.json", 'w') as fp: + json.dump(t2_config_db, fp, indent=4) + duthosts[1].copy(src="/tmp/temp_config.json", dest="/etc/sonic/%s" % config_db) + + logger.info('Reloading config to apply IP and BGP configuration on {}'.format(duthosts[1].hostname)) + pytest_assert('Error' not in duthosts[1].shell("sudo config reload -f -y \n")['stderr'], + 'Error while reloading config in {} !!!!!'.format(duthosts[1].hostname)) + logger.info('Config Reload Successful in {} !!!'.format(duthosts[1].hostname)) + wait(DUT_TRIGGER, "For configs to be loaded on the duts") + + +def generate_mac_address(): + mac = [random.randint(0x00, 0xff) for _ in range(6)] + return ':'.join(map(lambda x: "%02x" % x, mac)) + + +def __snappi_bgp_config(api, + duthosts, + snappi_ports, + traffic_type, + route_range): + """ + Creating BGP config on TGEN + + Args: + api (pytest fixture): snappi API + duthosts(pytest fixture): duthosts fixture + snappi_ports : Snappi port list + traffic_type: IPv4 or IPv6 traffic + route_range: v4 and v6 route combination + """ + global fanout_uplink_snappi_info + ipv4_src, ipv6_src = [], [] + ipv4_dest, ipv6_dest = [], [] + global total_routes + total_routes = 0 + config = api.config() + # get all the t1 and uplink ports from variables + t1_variable_ports = t1_ports[duthosts[0].hostname] + t2_variable_ports = [] + port_tuple = [] + for asic_value, portchannel_info in t2_uplink_portchannel_members[duthosts[1].hostname].items(): + for portchannel, ports in portchannel_info.items(): + port_tuple.append(ports) + for port in ports: + t2_variable_ports.append(port) + + snappi_t1_ports = [] + snappi_t2_ports = [] + for snappi_port in snappi_ports: + for port in t1_variable_ports: + if snappi_port['peer_device'] == duthosts[0].hostname and snappi_port['peer_port'] == port: + snappi_t1_ports.append(snappi_port) + for port in t2_variable_ports: + if snappi_port['peer_device'] == duthosts[1].hostname and snappi_port['peer_port'] == port: + snappi_t2_ports.append(snappi_port) + # Adding Ports + for index, snappi_test_port in enumerate(snappi_t1_ports): + if index == 0: + snappi_test_port['name'] = 'Snappi_Tx_Port' + else: + snappi_test_port['name'] = 'Snappi_Backup_T2_%d' % index + config.ports.port(name=snappi_test_port['name'], location=snappi_test_port['location']) + + for _, snappi_test_port in enumerate(snappi_t2_ports): + po = 1 + for asic_value, portchannel_info in t2_uplink_portchannel_members[duthosts[1].hostname].items(): + for portchannel, portchannel_members in portchannel_info.items(): + for index, mem_port in enumerate(portchannel_members, 1): + if snappi_test_port['peer_port'] == mem_port and \ + snappi_test_port['peer_device'] == duthosts[1].hostname: + snappi_test_port['name'] = 'Snappi_Uplink_PO_{}_Link_{}'.format(po, index) + fanout_uplink_snappi_info.append(snappi_test_port) + config.ports.port(name=snappi_test_port['name'], location=snappi_test_port['location']) + else: + continue + po = po + 1 + + config.options.port_options.location_preemption = True + layer1 = config.layer1.layer1()[-1] + layer1.name = 'port settings' + layer1.port_names = [port.name for port in config.ports] + layer1.ieee_media_defaults = False + layer1.auto_negotiation.rs_fec = True + layer1.auto_negotiation.link_training = False + layer1.speed = snappi_ports[0]['speed'] + layer1.auto_negotiate = False + + temp = 0 + for lag_count, port_set in enumerate(port_tuple): + lag = config.lags.lag(name="LAG %d" % lag_count)[-1] + lag.protocol.lacp.actor_system_id = generate_mac_address() + m = '0' + hex(lag_count % 15+1).split('0x')[1] + + for index, port in enumerate(port_set): + n = '0'+hex(index % 15+1).split('0x')[1] + for snappi_t2_port in snappi_t2_ports: + if port == snappi_t2_port['peer_port']: + lp = lag.ports.port(port_name=snappi_t2_port['name'])[-1] + lp.ethernet.name = "Eth%d" % temp + lp.ethernet.mac = "00:%s:00:00:00:%s" % (n, m) + logger.info('\n') + temp += 1 + + device = config.devices.device(name="T3 Device {}".format(lag_count))[-1] + eth = device.ethernets.add() + eth.port_name = lag.name + eth.name = 'T3_Ethernet_%d' % lag_count + eth.mac = "00:00:00:00:00:%s" % m + + ipv4 = eth.ipv4_addresses.add() + ipv4.name = 'T3_IPv4_%d' % lag_count + ipv4.address = snappi_portchannel_ipv4_list[lag_count] + ipv4.gateway = t2_dut_portchannel_ipv4_list[lag_count] + ipv4.prefix = v4_prefix_length + ipv6 = eth.ipv6_addresses.add() + ipv6.name = 'T3_IPv6_%d' % lag_count + ipv6.address = snappi_portchannel_ipv6_list[lag_count] + ipv6.gateway = t2_dut_portchannel_ipv6_list[lag_count] + ipv6.prefix = v6_prefix_length + + bgpv4 = device.bgp + bgpv4.router_id = t2_dut_portchannel_ipv4_list[lag_count] + bgpv4_int = bgpv4.ipv4_interfaces.add() + bgpv4_int.ipv4_name = ipv4.name + bgpv4_peer = bgpv4_int.peers.add() + bgpv4_peer.name = 'T3_BGP_%d' % lag_count + bgpv4_peer.as_type = BGP_TYPE + bgpv4_peer.peer_address = t2_dut_portchannel_ipv4_list[lag_count] + bgpv4_peer.as_number = int(T2_SNAPPI_AS_NUM) + + route_range1 = bgpv4_peer.v4_routes.add(name="T3_IPv4_Routes_%d" % (lag_count)) + for route_index, routes in enumerate(route_range['IPv4']): + route_range1.addresses.add( + address=routes[0], prefix=routes[1], count=routes[2]) + for community in snappi_community_for_t2: + manual_as_community = route_range1.communities.add() + manual_as_community.type = manual_as_community.MANUAL_AS_NUMBER + manual_as_community.as_number = int(community.split(":")[0]) + manual_as_community.as_custom = int(community.split(":")[1]) + ipv4_dest.append(route_range1.name) + + bgpv6 = device.bgp + bgpv6.router_id = t2_dut_portchannel_ipv4_list[lag_count] + bgpv6_int = bgpv6.ipv6_interfaces.add() + bgpv6_int.ipv6_name = ipv6.name + bgpv6_peer = bgpv6_int.peers.add() + bgpv6_peer.name = 'T3_BGP+_%d' % lag_count + bgpv6_peer.as_type = BGP_TYPE + bgpv6_peer.peer_address = t2_dut_portchannel_ipv6_list[lag_count] + bgpv6_peer.as_number = int(T2_SNAPPI_AS_NUM) + + route_range2 = bgpv6_peer.v6_routes.add(name="T3_IPv6_Routes_%d" % (lag_count)) + for route_index, routes in enumerate(route_range['IPv6']): + route_range2.addresses.add( + address=routes[0], prefix=routes[1], count=routes[2]) + for community in snappi_community_for_t2: + manual_as_community = route_range2.communities.add() + manual_as_community.type = manual_as_community.MANUAL_AS_NUMBER + manual_as_community.as_number = int(community.split(":")[0]) + manual_as_community.as_custom = int(community.split(":")[1]) + ipv6_dest.append(route_range2.name) + + for index, port in enumerate(snappi_t1_ports): + if len(str(hex(index+1).split('0x')[1])) == 1: + m = '0'+hex(index+1).split('0x')[1] + else: + m = hex(index+1).split('0x')[1] + + if index == 0: + device = config.devices.device(name="T0 Device {}".format(index))[-1] + eth = device.ethernets.add() + eth.port_name = port['name'] + eth.name = 'T0_Ethernet_%d' % index + eth.mac = "00:10:00:00:00:%s" % m + ipv4 = eth.ipv4_addresses.add() + ipv4.name = 'T0_IPv4_%d' % index + ipv4.address = t1_t2_snappi_ipv4_list[index] + ipv4.gateway = t1_t2_dut_ipv4_list[index] + ipv4.prefix = v4_prefix_length + ipv6 = eth.ipv6_addresses.add() + ipv6.name = 'T0_IPv6_%d' % index + ipv6.address = t1_t2_snappi_ipv6_list[index] + ipv6.gateway = t1_t2_dut_ipv6_list[index] + ipv6.prefix = v6_prefix_length + ipv4_src.append(ipv4.name) + ipv6_src.append(ipv6.name) + else: + device = config.devices.device(name="Backup T2 Device {}".format(index))[-1] + eth = device.ethernets.add() + eth.port_name = port['name'] + eth.name = 'Backup_T2_Ethernet_%d' % index + eth.mac = "00:10:00:00:00:%s" % m + ipv4 = eth.ipv4_addresses.add() + ipv4.name = 'Backup_T2_IPv4_%d' % index + ipv4.address = t1_t2_snappi_ipv4_list[index] + ipv4.gateway = t1_t2_dut_ipv4_list[index] + ipv4.prefix = v4_prefix_length + ipv6 = eth.ipv6_addresses.add() + ipv6.name = 'Backup_T2_IPv6_%d' % index + ipv6.address = t1_t2_snappi_ipv6_list[index] + ipv6.gateway = t1_t2_dut_ipv6_list[index] + ipv6.prefix = v6_prefix_length + + bgpv4 = device.bgp + bgpv4.router_id = t1_t2_snappi_ipv4_list[index] + bgpv4_int = bgpv4.ipv4_interfaces.add() + bgpv4_int.ipv4_name = ipv4.name + bgpv4_peer = bgpv4_int.peers.add() + bgpv4_peer.name = 'Backup_T2_BGP_%d' % index + bgpv4_peer.as_type = BGP_TYPE + bgpv4_peer.peer_address = t1_t2_dut_ipv4_list[index] + bgpv4_peer.as_number = int(T1_SNAPPI_AS_NUM) + + if 'IPv4' in route_range.keys(): + route_range1 = bgpv4_peer.v4_routes.add(name="Backup_T2_IPv4_Routes_%d" % (index)) + for route_index, routes in enumerate(route_range['IPv4']): + route_range1.addresses.add( + address=routes[0], prefix=routes[1], count=routes[2]) + ipv4_dest.append(route_range1.name) + as_path = route_range1.as_path + as_path_segment = as_path.segments.add() + as_path_segment.type = as_path_segment.AS_SEQ + as_path_segment.as_numbers = AS_PATHS + for community in snappi_community_for_t1: + manual_as_community = route_range1.communities.add() + manual_as_community.type = manual_as_community.MANUAL_AS_NUMBER + manual_as_community.as_number = int(community.split(":")[0]) + manual_as_community.as_custom = int(community.split(":")[1]) + + bgpv6 = device.bgp + bgpv6.router_id = t1_t2_snappi_ipv4_list[index] + bgpv6_int = bgpv6.ipv6_interfaces.add() + bgpv6_int.ipv6_name = ipv6.name + bgpv6_peer = bgpv6_int.peers.add() + bgpv6_peer.name = 'Backup_T2_BGP+_%d' % index + bgpv6_peer.as_type = BGP_TYPE + bgpv6_peer.peer_address = t1_t2_dut_ipv6_list[index] + bgpv6_peer.as_number = int(T1_SNAPPI_AS_NUM) + + if 'IPv6' in route_range.keys(): + route_range2 = bgpv6_peer.v6_routes.add(name="Backup_T2_IPv6_Routes_%d" % (index)) + for route_index, routes in enumerate(route_range['IPv6']): + route_range2.addresses.add( + address=routes[0], prefix=routes[1], count=routes[2]) + ipv6_dest.append(route_range2.name) + as_path = route_range2.as_path + as_path_segment = as_path.segments.add() + as_path_segment.type = as_path_segment.AS_SEQ + as_path_segment.as_numbers = AS_PATHS + for community in snappi_community_for_t1: + manual_as_community = route_range2.communities.add() + manual_as_community.type = manual_as_community.MANUAL_AS_NUMBER + manual_as_community.as_number = int(community.split(":")[0]) + manual_as_community.as_custom = int(community.split(":")[1]) + + def createTrafficItem(traffic_name, source, destination): + logger.info('{} Source : {}'.format(traffic_name, source)) + logger.info('{} Destination : {}'.format(traffic_name, destination)) + flow1 = config.flows.flow(name=str(traffic_name))[-1] + flow1.tx_rx.device.tx_names = source + flow1.tx_rx.device.rx_names = destination + flow1.size.fixed = 1024 + flow1.rate.percentage = 10 + flow1.metrics.enable = True + flow1.metrics.loss = True + + if 'IPv4' in traffic_type and 'IPv6' in traffic_type: + for route in route_range['IPv4']: + total_routes = total_routes+route[2] + for route in route_range['IPv6']: + total_routes = total_routes+route[2] + createTrafficItem("IPv4_Traffic", [ipv4_src[0]], ipv4_dest) + createTrafficItem("IPv6_Traffic", [ipv6_src[0]], ipv6_dest) + elif 'IPv6' in traffic_type and 'IPv4' not in traffic_type: + for route in route_range['IPv6']: + total_routes = total_routes+route[2] + createTrafficItem("IPv6 Traffic", [ipv6_src[0]], ipv6_dest) + elif 'IPv4' in traffic_type and 'IPv6' not in traffic_type: + for route in route_range['IPv4']: + total_routes = total_routes+route[2] + createTrafficItem("IPv4 Traffic", [ipv4_src[0]], ipv4_dest) + return config + + +def get_flow_stats(api): + """ + Args: + api (pytest fixture): Snappi API + """ + request = api.metrics_request() + request.flow.flow_names = [] + return api.get_metrics(request).flow_metrics + + +def get_port_stats(api): + """ + Args: + api (pytest fixture): Snappi API + """ + request = api.metrics_request() + return api.get_metrics(request).port_metrics + + +def flap_single_fanout_port(fanout_ip, creds, port_name, state): + """ + Args: + fanout_ip (pytest fixture): IP of the fanout device + creds (dict): DUT credentials + port_name: Name of the fanout port to be flapped + state: State of the interface to be up/down + """ + username = creds.get('sonicadmin_user') + password = creds.get('sonicadmin_password') + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(fanout_ip, port=22, username=username, password=password) + if state == 'up': + command = f'sudo config interface startup {port_name}' + elif state == 'down': + command = f'sudo config interface shutdown {port_name}' + stdin, stdout, stderr = ssh.exec_command(command) + + +def get_convergence_for_link_flap(duthosts, + api, + bgp_config, + flap_details, + traffic_type, + iteration, + route_range, + test_name, + creds): + """ + Args: + duthost (pytest fixture): duthost fixture + api (pytest fixture): Snappi API + bgp_config: __snappi_bgp_config + flap_details: contains device name and port / services that needs to be flapped + traffic_type : IPv4 / IPv6 traffic type + iteration : Number of iterations + test_name: Name of the test + creds (pytest fixture): DUT credentials + """ + api.set_config(bgp_config) + avg_pld = [] + avg_pld2 = [] + test_platform = TestPlatform(api._address) + test_platform.Authenticate(api._username, api._password) + session = SessionAssistant(IpAddress=api._address, UserName=api._username, + SessionId=test_platform.Sessions.find()[-1].Id, Password=api._password) + ixnetwork = session.Ixnetwork + for index, topology in enumerate(ixnetwork.Topology.find()): + try: + topology.DeviceGroup.find()[0].RouterData.find().RouterId.Single(router_ids[index]) + logger.info('Setting Router id {} for {}'.format(router_ids[index], topology.DeviceGroup.find()[0].Name)) + except Exception: + logger.info('Skipping Router id for {}, Since bgp is not configured'. + format(topology.DeviceGroup.find()[0].Name)) + continue + logger.info('\n') + logger.info('Testing with Route Range: {}'.format(route_range)) + logger.info('\n') + for i in range(0, iteration): + logger.info( + '|--------------------------- Iteration : {} -----------------------|'.format(i+1)) + logger.info("Starting all protocols ...") + ps = api.protocol_state() + ps.state = ps.START + api.set_protocol_state(ps) + wait(SNAPPI_TRIGGER, "For Protocols To start") + logger.info('Verifying protocol sessions state') + protocolsSummary = StatViewAssistant(ixnetwork, 'Protocols Summary') + protocolsSummary.CheckCondition('Sessions Down', StatViewAssistant.EQUAL, 0) + logger.info('Starting Traffic') + ts = api.transmit_state() + ts.state = ts.START + api.set_transmit_state(ts) + wait(SNAPPI_TRIGGER, "For Traffic To start") + + flow_stats = get_flow_stats(api) + port_stats = get_port_stats(api) + + logger.info('\n') + logger.info('Rx Snappi Port Name : Rx Frame Rate') + for port_stat in port_stats: + if 'Snappi_Tx_Port' not in port_stat.name: + logger.info('{} : {}'.format(port_stat.name, port_stat.frames_rx_rate)) + pytest_assert(port_stat.frames_rx_rate > 0, '{} is not receiving any packet'.format(port_stat.name)) + logger.info('\n') + for i in range(0, len(traffic_type)): + logger.info('{} Loss %: {}'.format(flow_stats[i].name, int(flow_stats[i].loss))) + pytest_assert(int(flow_stats[i].loss) == 0, f'Loss Observed in {flow_stats[i].name} before link Flap') + + sum_t2_rx_frame_rate = 0 + for port_stat in port_stats: + if 'Snappi_Uplink' in port_stat.name: + sum_t2_rx_frame_rate = sum_t2_rx_frame_rate + int(port_stat.frames_rx_rate) + # Flap the required test port + if duthosts[0].hostname == flap_details['device_name']: + logger.info(' Shutting down {} port of {} dut !!'. + format(flap_details['port_name'], flap_details['device_name'])) + duthosts[0].command('sudo config interface shutdown {} \n'. + format(flap_details['port_name'])) + wait(DUT_TRIGGER, "For link to shutdown") + elif 'Ixia' == flap_details['device_name']: + if fanout_presence is False: + ixn_port = ixnetwork.Vport.find(Name=flap_details['port_name'])[0] + ixn_port.LinkUpDn("down") + logger.info('Shutting down snappi port : {}'.format(flap_details['port_name'])) + wait(SNAPPI_TRIGGER, "For link to shutdown") + else: + for port in fanout_uplink_snappi_info: + if flap_details['port_name'] == port['name']: + uplink_port = port['peer_port'] + for fanout_info in t2_uplink_fanout_info: + for port_mapping in fanout_info['port_mapping']: + if uplink_port == port_mapping['uplink_port']: + fanout_port = port_mapping['fanout_port'] + fanout_ip = fanout_info['fanout_ip'] + break + pytest_assert(fanout_port is not None, 'Unable to get fanout port info') + flap_single_fanout_port(fanout_ip, creds, fanout_port, state='down') + logger.info(' Shutting down {} from {}'.format(fanout_port, fanout_ip)) + wait(DUT_TRIGGER, "For link to shutdown") + flow_stats = get_flow_stats(api) + for i in range(0, len(traffic_type)): + pytest_assert(float((int(flow_stats[i].frames_tx_rate) - int(flow_stats[i].frames_rx_rate)) / + int(flow_stats[i].frames_tx_rate)) < 0.005, + 'Traffic has not converged after link flap') + logger.info('Traffic has converged after link flap') + + delta_frames = 0 + for i in range(0, len(traffic_type)): + delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx + pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate) + logger.info('Delta Frames : {}'.format(delta_frames)) + logger.info('PACKET LOSS DURATION After Link Down (ms): {}'.format(pkt_loss_duration)) + avg_pld.append(pkt_loss_duration) + + logger.info('Performing Clear Stats') + ixnetwork.ClearStats() + if duthosts[0].hostname == flap_details['device_name']: + logger.info(' Starting up {} port of {} dut !!'. + format(flap_details['port_name'], flap_details['device_name'])) + duthosts[0].command('sudo config interface startup {} \n'. + format(flap_details['port_name'])) + wait(DUT_TRIGGER, "For link to startup") + elif 'Ixia' == flap_details['device_name']: + if fanout_presence is False: + ixn_port = ixnetwork.Vport.find(Name=flap_details['port_name'])[0] + ixn_port.LinkUpDn("up") + logger.info('Starting up snappi port : {}'.format(flap_details['port_name'])) + wait(SNAPPI_TRIGGER, "For link to startup") + else: + flap_single_fanout_port(fanout_ip, creds, fanout_port, state='up') + logger.info('Starting up {} from {}'.format(fanout_port, fanout_ip)) + wait(DUT_TRIGGER, "For link to startup") + logger.info('\n') + port_stats = get_port_stats(api) + logger.info('Rx Snappi Port Name : Rx Frame Rate') + for port_stat in port_stats: + if 'Snappi_Tx_Port' not in port_stat.name: + logger.info('{} : {}'.format(port_stat.name, port_stat.frames_rx_rate)) + pytest_assert(port_stat.frames_rx_rate > 0, '{} is not receiving any packet'.format(port_stat.name)) + + flow_stats = get_flow_stats(api) + delta_frames = 0 + for i in range(0, len(traffic_type)): + delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx + pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate) + logger.info('Delta Frames : {}'.format(delta_frames)) + logger.info('PACKET LOSS DURATION After Link Up (ms): {}'.format(pkt_loss_duration)) + avg_pld2.append(pkt_loss_duration) + logger.info('Stopping Traffic') + ts = api.transmit_state() + ts.state = ts.STOP + api.set_transmit_state(ts) + + logger.info("Stopping all protocols ...") + ps = api.protocol_state() + ps.state = ps.STOP + api.set_protocol_state(ps) + logger.info('\n') + + columns = ['Test Name', 'Iterations', 'Traffic Type', 'Uplink ECMP Paths', 'Route Count', + 'Avg Calculated Packet Loss Duration (ms)'] + logger.info("\n%s" % tabulate([[test_name+' (Link Down)', iteration, traffic_type, portchannel_count, + total_routes, mean(avg_pld)], [test_name+' (Link Up)', iteration, + traffic_type, portchannel_count, total_routes, mean(avg_pld2)]], headers=columns, + tablefmt="psql")) + + +def kill_process_inside_container(duthost, container_name, process_id, creds): + """ + Args: + duthost (pytest fixture): duthost fixture + container_name (str): Container name running in dut + process_id: process id that needs to be killed inside container + creds (dict): DUT credentials + """ + username = creds.get('sonicadmin_user') + password = creds.get('sonicadmin_password') + ip = duthost.mgmt_ip + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, port=22, username=username, password=password) + command = f'docker exec {container_name} kill {process_id}' + stdin, stdout, stderr = ssh.exec_command(command) + + +def get_container_names(duthost): + """ + Args: + duthost (pytest fixture): duthost fixture + """ + container_names = duthost.shell('docker ps --format \{\{.Names\}\}')['stdout_lines'] # noqa: W605 + return container_names + + +def check_container_status_up(duthost, container_name, timeout): + """ + Args: + duthost (pytest fixture): duthost fixture + container_name (str): Container name running in dut + timeout(secs): Maximum time limit for polling + """ + start_time = time.time() + while True: + running_containers_list = get_container_names(duthost) + if container_name in running_containers_list: + logger.info('PASS: {} is RUNNING after process kill'.format(container_name)) + break + logger.info('Polling for {} to come UP.....'.format(container_name)) + elapsed_time = time.time() - start_time + pytest_assert(elapsed_time < timeout, "Container did not come up in {} \ + seconds after process kill".format(timeout)) + time.sleep(5) + + +def check_container_status_down(duthost, container_name, timeout): + """ + Args: + duthost (pytest fixture): duthost fixture + container_name (str): Container name running in dut + timeout(secs): Maximum time limit for polling + """ + start_time = time.time() + while True: + running_containers_list = get_container_names(duthost) + if container_name not in running_containers_list: + logger.info('PASS: {} is DOWN after process kill'.format(container_name)) + break + logger.info('Polling for {} to go Down.....'.format(container_name)) + elapsed_time = time.time() - start_time + pytest_assert(elapsed_time < timeout, "Container is still running for {} \ + seconds after process kill".format(timeout)) + time.sleep(5) + + +def get_container_names_from_asic_count(duthost, container_name): + """ + Args: + duthost (pytest fixture): duthost fixture + container_name (str): Container name running in dut + """ + container_names = [] + platform_summary = duthost.shell('show platform summary')['stdout_lines'] + for line in platform_summary: + if 'ASIC Count' in line: + count = int(line.split(':')[-1].lstrip()) + for i in range(0, count): + container_names.append(container_name+str(i)) + return container_names + + +def get_convergence_for_process_flap(duthosts, + api, + bgp_config, + traffic_type, + iteration, + process_names, + host_name, + route_range, + test_name, + creds): + """ + Args: + duthost (pytest fixture): duthost fixture + api (pytest fixture): Snappi API + bgp_config: __snappi_bgp_config + traffic_type : IPv4 / IPv6 traffic type + iteration : Number of iterations + process_names : Name of the container in which specific process needs to be killed + host_name : Dut hostname + test_name: Name of the test + creds (dict): DUT credentials + """ + api.set_config(bgp_config) + test_platform = TestPlatform(api._address) + test_platform.Authenticate(api._username, api._password) + session = SessionAssistant(IpAddress=api._address, UserName=api._username, + SessionId=test_platform.Sessions.find()[-1].Id, Password=api._password) + ixnetwork = session.Ixnetwork + for index, topology in enumerate(ixnetwork.Topology.find()): + try: + topology.DeviceGroup.find()[0].RouterData.find().RouterId.Single(router_ids[index]) + logger.info('Setting Router id {} for {}'.format(router_ids[index], topology.DeviceGroup.find()[0].Name)) + except Exception: + logger.info('Skipping Router id for {}, Since bgp is not configured'. + format(topology.DeviceGroup.find()[0].Name)) + continue + + table = [] + logger.info('\n') + logger.info('Testing with Route Range: {}'.format(route_range)) + logger.info('\n') + for container_name, process_name in process_names.items(): + for duthost in duthosts: + container_names = get_container_names_from_asic_count(duthost, container_name) + if duthost.hostname == host_name: + for container in container_names: + row = [] + avg_pld = [] + for i in range(0, iteration): + logger.info( + '|---------------------------{} Iteration : {} --------------\ + ---------|'.format(container, i+1)) + logger.info("Starting all protocols ...") + ps = api.protocol_state() + ps.state = ps.START + api.set_protocol_state(ps) + wait(SNAPPI_TRIGGER, "For Protocols To start") + logger.info('Verifying protocol sessions state') + protocolsSummary = StatViewAssistant(ixnetwork, 'Protocols Summary') + protocolsSummary.CheckCondition('Sessions Down', StatViewAssistant.EQUAL, 0) + logger.info('Starting Traffic') + ts = api.transmit_state() + ts.state = ts.START + api.set_transmit_state(ts) + wait(SNAPPI_TRIGGER, "For Traffic To start") + + flow_stats = get_flow_stats(api) + for i in range(0, len(traffic_type)): + logger.info('{} Loss %: {}'. + format(flow_stats[i].name, int(flow_stats[i].loss))) + logger.info('\n') + port_stats = get_port_stats(api) + logger.info('Rx Snappi Port Name : Rx Frame Rate') + for port_stat in port_stats: + if 'Snappi_Tx_Port' not in port_stat.name: + logger.info('{} : {}'.format(port_stat.name, port_stat.frames_rx_rate)) + pytest_assert(port_stat.frames_rx_rate > 0, '{} is not receiving \ + any packet'.format(port_stat.name)) + pytest_assert(int(flow_stats[0].loss) == 0, 'Loss Observed in traffic \ + flow before killing service in {}') + logger.info('\n') + sum_t2_rx_frame_rate = 0 + for port_stat in port_stats: + if 'Snappi_Uplink' in port_stat.name: + sum_t2_rx_frame_rate = sum_t2_rx_frame_rate + int(port_stat.frames_rx_rate) + logger.info('Killing {}:{} service in {}'.format(container, process_name, host_name)) + PID = duthost.shell('docker exec {} ps aux | grep {} \n'. + format(container, process_name))['stdout'].split(' ')[10] + all_containers = get_container_names(duthost) + logger.info('Runnnig containers before process kill: {}'.format(all_containers)) + kill_process_inside_container(duthost, container, PID, creds) + check_container_status_down(duthost, container, timeout=60) + check_container_status_up(duthost, container, timeout=DUT_TRIGGER) + wait(DUT_TRIGGER, "For Flows to be evenly distributed") + port_stats = get_port_stats(api) + for port_stat in port_stats: + if 'Snappi_Tx_Port' not in port_stat.name: + logger.info('{}: {}'.format(port_stat.name, port_stat.frames_rx_rate)) + pytest_assert(port_stat.frames_rx_rate > 0, '{} is not receiving any packet \ + after container is up'.format(port_stat.name)) + flow_stats = get_flow_stats(api) + delta_frames = 0 + for i in range(0, len(traffic_type)): + delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx + pkt_loss_duration = 1000*(delta_frames/sum_t2_rx_frame_rate) + logger.info('Delta Frames : {}'.format(delta_frames)) + logger.info('PACKET LOSS DURATION (ms): {}'.format(pkt_loss_duration)) + avg_pld.append(pkt_loss_duration) + + logger.info('Stopping Traffic') + ts = api.transmit_state() + ts.state = ts.STOP + api.set_transmit_state(ts) + wait(SNAPPI_TRIGGER, "For Traffic To stop") + + logger.info("Stopping all protocols ...") + ps = api.protocol_state() + ps.state = ps.STOP + api.set_protocol_state(ps) + wait(SNAPPI_TRIGGER, "For Protocols To stop") + logger.info('\n') + row.append(test_name) + row.append(f'{container}') + row.append(f'{process_name}') + row.append(iteration) + row.append(traffic_type) + row.append(portchannel_count) + row.append(total_routes) + row.append(mean(avg_pld)) + table.append(row) + columns = ['Test Name', 'Container Name', 'Process Name', 'Iterations', 'Traffic Type', + 'Uplink ECMP Paths', 'Route Count', 'Avg Calculated Packet Loss Duration (ms)'] + logger.info("\n%s" % tabulate(table, headers=columns, tablefmt="psql")) + + +def exec_tsa_tsb_cmd_on_linecard(duthost, creds, tsa_tsb_cmd): + """ + @summary: Issue TSA/TSB command on supervisor card using user credentials + Verify command is executed on supervisor card + @returns: None + """ + try: + dut_ip = duthost.mgmt_ip + sonic_username = creds['sonicadmin_user'] + sonic_password = creds['sonicadmin_password'] + logger.info('sonic-username: {}, sonic_password: {}'.format(sonic_username, sonic_password)) + ssh_cmd = "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no {}@{}".format(sonic_username, dut_ip) + connect = pexpect.spawn(ssh_cmd) + time.sleep(10) + connect.expect('.*[Pp]assword:') + connect.sendline(sonic_password) + time.sleep(10) + connect.sendline(tsa_tsb_cmd) + time.sleep(10) + connect.expect('.*[Pp]assword for username \'{}\':'.format(sonic_username)) + connect.sendline(sonic_password) + time.sleep(20) + except pexpect.exceptions.EOF: + pytest_assert(False, "EOF reached") + except pexpect.exceptions.TIMEOUT: + pytest_assert(False, "Timeout reached") + except Exception as e: + pytest_assert(False, "Cannot connect to DUT {} host via SSH: {}".format(duthost.hostname, e)) + + +def get_convergence_for_tsa_tsb(duthosts, + api, + snappi_bgp_config, + traffic_type, + iteration, + device_name, + route_range, + test_name, + creds, + is_supervisor): + + """ + Args: + duthost (pytest fixture): duthost fixture + api (pytest fixture): Snappi API + snappi_bgp_config: __snappi_bgp_config + flap_details: contains device name and port / services that needs to be flapped + traffic_type : IPv4 / IPv6 traffic type + iteration : Number of iterations + device_name: Device in which TSA, TSB needs to be performed + route_range: V4 and v6 routes + test_name: Name of the test + """ + api.set_config(snappi_bgp_config) + avg_pld = [] + avg_pld2 = [] + test_platform = TestPlatform(api._address) + test_platform.Authenticate(api._username, api._password) + session = SessionAssistant(IpAddress=api._address, UserName=api._username, + SessionId=test_platform.Sessions.find()[-1].Id, Password=api._password) + ixnetwork = session.Ixnetwork + for index, topology in enumerate(ixnetwork.Topology.find()): + try: + topology.DeviceGroup.find()[0].RouterData.find().RouterId.Single(router_ids[index]) + logger.info('Setting Router id {} for {}'.format(router_ids[index], topology.DeviceGroup.find()[0].Name)) + except Exception: + logger.info('Skipping Router id for {}, Since bgp is not configured'. + format(topology.DeviceGroup.find()[0].Name)) + continue + logger.info('\n') + logger.info('Testing with Route Range: {}'.format(route_range)) + logger.info('\n') + logger.info('Issuing TSB before starting test to ensure DUT to be in proper state') + for duthost in duthosts: + if duthost.hostname == device_name: + if is_supervisor is True: + exec_tsa_tsb_cmd_on_linecard(duthost, creds, "sudo TSB") + else: + duthost.command('sudo TSB') + wait(DUT_TRIGGER, "For TSB") + try: + for i in range(0, iteration): + logger.info( + '|--------------------------- Iteration : {} -----------------------|'.format(i+1)) + logger.info("Starting all protocols ...") + ps = api.protocol_state() + ps.state = ps.START + api.set_protocol_state(ps) + wait(SNAPPI_TRIGGER, "For Protocols To start") + logger.info('Verifying protocol sessions state') + protocolsSummary = StatViewAssistant(ixnetwork, 'Protocols Summary') + protocolsSummary.CheckCondition('Sessions Down', StatViewAssistant.EQUAL, 0) + logger.info('Starting Traffic') + ts = api.transmit_state() + ts.state = ts.START + api.set_transmit_state(ts) + wait(SNAPPI_TRIGGER, "For Traffic To start") + flow_stats = get_flow_stats(api) + port_stats = get_port_stats(api) + + logger.info('\n') + logger.info('Rx Snappi Port Name : Rx Frame Rate') + for port_stat in port_stats: + if 'Snappi_Tx_Port' not in port_stat.name: + logger.info('{} : {}'.format(port_stat.name, port_stat.frames_rx_rate)) + pytest_assert(port_stat.frames_rx_rate > 0, '{} is not receiving any packet'.format(port_stat.name)) + logger.info('\n') + for i in range(0, len(traffic_type)): + logger.info('{} Loss %: {}'.format(flow_stats[i].name, int(flow_stats[i].loss))) + pytest_assert(int(flow_stats[i].loss) == 0, f'Loss Observed in {flow_stats[i].name} before link Flap') + + # Getting rx rate on uplink ports + sum_t2_rx_frame_rate = 0 + for port_stat in port_stats: + if 'Snappi_Uplink' in port_stat.name: + sum_t2_rx_frame_rate = sum_t2_rx_frame_rate + int(port_stat.frames_rx_rate) + + logger.info('Issuing TSA on {}'.format(device_name)) + for duthost in duthosts: + if duthost.hostname == device_name: + if is_supervisor is True: + exec_tsa_tsb_cmd_on_linecard(duthost, creds, "sudo TSA") + else: + duthost.command('sudo TSA') + wait(DUT_TRIGGER, "For TSA") + flow_stats = get_flow_stats(api) + for i in range(0, len(traffic_type)): + logger.info(flow_stats[i].frames_tx_rate) + logger.info(flow_stats[i].frames_rx_rate) + pytest_assert(float((int(flow_stats[i].frames_tx_rate) - int(flow_stats[i].frames_rx_rate)) / + int(flow_stats[i].frames_tx_rate)) < 0.005, + 'Traffic has not converged after TSA') + logger.info('Traffic has converged after issuing TSA command in {}'.format(device_name)) + flow_stats = get_flow_stats(api) + delta_frames = 0 + for i in range(0, len(traffic_type)): + delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx + pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate) + logger.info('Delta Frames : {}'.format(delta_frames)) + logger.info('PACKET LOSS DURATION After TSA (ms): {}'.format(pkt_loss_duration)) + avg_pld.append(pkt_loss_duration) + + logger.info('Performing Clear Stats') + ixnetwork.ClearStats() + logger.info('Issuing TSB on {}'.format(device_name)) + for duthost in duthosts: + if duthost.hostname == device_name: + if is_supervisor is True: + exec_tsa_tsb_cmd_on_linecard(duthost, creds, "sudo TSB") + else: + duthost.command('sudo TSB') + + wait(DUT_TRIGGER, "For TSB") + logger.info('\n') + port_stats = get_port_stats(api) + logger.info('Rx Snappi Port Name : Rx Frame Rate') + for port_stat in port_stats: + if 'Snappi_Tx_Port' not in port_stat.name: + logger.info('{} : {}'.format(port_stat.name, port_stat.frames_rx_rate)) + pytest_assert(port_stat.frames_rx_rate > 0, '{} is not receiving any packet'.format(port_stat.name)) + + flow_stats = get_flow_stats(api) + delta_frames = 0 + for i in range(0, len(traffic_type)): + delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx + pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate) + logger.info('Delta Frames : {}'.format(delta_frames)) + logger.info('PACKET LOSS DURATION After TSB (ms): {}'.format(pkt_loss_duration)) + avg_pld2.append(pkt_loss_duration) + logger.info('Stopping Traffic') + ts = api.transmit_state() + ts.state = ts.STOP + api.set_transmit_state(ts) + + logger.info("Stopping all protocols ...") + ps = api.protocol_state() + ps.state = ps.STOP + api.set_protocol_state(ps) + logger.info('\n') + + columns = ['Test Name', 'Iterations', 'Traffic Type', 'Uplink ECMP Paths', 'Route Count', + 'Avg Calculated Packet Loss Duration (ms)'] + logger.info("\n%s" % tabulate([[test_name+' (TSA)', iteration, traffic_type, portchannel_count, + total_routes, mean(avg_pld)], [test_name+' (TSB)', iteration, + traffic_type, portchannel_count, total_routes, mean(avg_pld2)]], + headers=columns, tablefmt="psql")) + except Exception as e: + logger.info(e) + logger.info('Since an exception occurred, Issuing TSB, to ensure DUT to be in proper state') + for duthost in duthosts: + if duthost.hostname == device_name: + if is_supervisor is True: + exec_tsa_tsb_cmd_on_linecard(duthost, creds, "sudo TSB") + else: + duthost.command('sudo TSB') + wait(DUT_TRIGGER, "For TSB") + + +def flap_fanout_ports(fanout_ip_port_mapping, creds, state): + """ + Args: + + """ + username = creds.get('sonicadmin_user') + password = creds.get('sonicadmin_password') + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + for fanout_ip, req_ports in fanout_ip_port_mapping.items(): + ssh.connect(fanout_ip, port=22, username=username, password=password) + if state == 'down': + for port_name in req_ports: + time.sleep(0.05) + stdin, stdout, stderr = ssh.exec_command(f'sudo config interface shutdown {port_name}') + logger.info('Shutting down {}'.format(port_name)) + elif state == 'up': + for port_name in req_ports: + time.sleep(0.05) + stdin, stdout, stderr = ssh.exec_command(f'sudo config interface startup {port_name}') + logger.info('Starting up {}'.format(port_name)) + + +def add_value_to_key(dictionary, key, value): + if key in dictionary: + dictionary[key] = dictionary[key] + [value] + else: + dictionary[key] = [value] + + +def get_convergence_for_blackout(duthosts, + api, + snappi_bgp_config, + traffic_type, + iteration, + blackout_percentage, + route_range, + test_name, + creds): + """ + Args: + duthost (pytest fixture): duthost fixture + api (pytest fixture): Snappi API + bgp_config: __snappi_bgp_config + flap_details: contains device name and port / services that needs to be flapped + traffic_type : IPv4 / IPv6 traffic type + iteration : Number of iterations + test_name: Name of the test + """ + api.set_config(snappi_bgp_config) + avg_pld = [] + avg_pld2 = [] + test_platform = TestPlatform(api._address) + test_platform.Authenticate(api._username, api._password) + session = SessionAssistant(IpAddress=api._address, UserName=api._username, + SessionId=test_platform.Sessions.find()[-1].Id, Password=api._password) + ixnetwork = session.Ixnetwork + for index, topology in enumerate(ixnetwork.Topology.find()): + try: + topology.DeviceGroup.find()[0].RouterData.find().RouterId.Single(router_ids[index]) + logger.info('Setting Router id {} for {}'.format(router_ids[index], topology.DeviceGroup.find()[0].Name)) + except Exception: + logger.info('Skipping Router id for {}, Since bgp is not configured'. + format(topology.DeviceGroup.find()[0].Name)) + continue + logger.info('\n') + logger.info('Testing with Route Range: {}'.format(route_range)) + logger.info('\n') + for i in range(0, iteration): + logger.info( + '|--------------------------- Iteration : {} -----------------------|'.format(i+1)) + logger.info("Starting all protocols ...") + ps = api.protocol_state() + ps.state = ps.START + api.set_protocol_state(ps) + wait(SNAPPI_TRIGGER, "For Protocols To start") + logger.info('Verifying protocol sessions state') + protocolsSummary = StatViewAssistant(ixnetwork, 'Protocols Summary') + protocolsSummary.CheckCondition('Sessions Down', StatViewAssistant.EQUAL, 0) + logger.info('Starting Traffic') + ts = api.transmit_state() + ts.state = ts.START + api.set_transmit_state(ts) + wait(SNAPPI_TRIGGER, "For Traffic To start") + + flow_stats = get_flow_stats(api) + port_stats = get_port_stats(api) + + logger.info('\n') + logger.info('Rx Snappi Port Name : Rx Frame Rate') + for port_stat in port_stats: + if 'Snappi_Tx_Port' not in port_stat.name: + logger.info('{} : {}'.format(port_stat.name, port_stat.frames_rx_rate)) + pytest_assert(port_stat.frames_rx_rate > 0, '{} is not receiving any packet'.format(port_stat.name)) + logger.info('\n') + for i in range(0, len(traffic_type)): + logger.info('{} Loss %: {}'.format(flow_stats[i].name, int(flow_stats[i].loss))) + pytest_assert(int(flow_stats[i].loss) == 0, f'Loss Observed in {flow_stats[i].name} before link Flap') + + sum_t2_rx_frame_rate = 0 + for port_stat in port_stats: + if 'Snappi_Uplink' in port_stat.name: + sum_t2_rx_frame_rate = sum_t2_rx_frame_rate + int(port_stat.frames_rx_rate) + + # Link Down + portchannel_dict = {} + for asic_value, portchannel_info in t2_uplink_portchannel_members[duthosts[1].hostname].items(): + portchannel_dict.update(portchannel_info) + number_of_po = math.ceil(blackout_percentage * len(portchannel_dict)/100) + snappi_port_names = [] + for snappi_port in fanout_uplink_snappi_info: + uplink_ports = [] + for i, (key, value) in enumerate(portchannel_dict.items(), 1): + if i <= number_of_po: + uplink_ports += value + if i == int(snappi_port['name'].split('_')[3]): + snappi_port_names.append(snappi_port['name']) + if fanout_presence is False: + for snappi_port_name in snappi_port_names: + time.sleep(0.05) + ixn_port = ixnetwork.Vport.find(Name=snappi_port_name)[0] + ixn_port.LinkUpDn("down") + logger.info('Shutting down snappi port : {}'.format(snappi_port_name)) + wait(SNAPPI_TRIGGER, "For links to shutdown") + else: + required_fanout_mapping = {} + for uplink_port in uplink_ports: + for fanout_info in t2_uplink_fanout_info: + for port_mapping in fanout_info['port_mapping']: + if uplink_port == port_mapping['uplink_port']: + fanout_ip = fanout_info['fanout_ip'] + add_value_to_key(required_fanout_mapping, fanout_ip, port_mapping['fanout_port']) + flap_fanout_ports(required_fanout_mapping, creds, state='down') + wait(DUT_TRIGGER, "For links to shutdown") + + flow_stats = get_flow_stats(api) + for i in range(0, len(traffic_type)): + pytest_assert(float((int(flow_stats[i].frames_tx_rate) - int(flow_stats[i].frames_rx_rate)) / + int(flow_stats[i].frames_tx_rate)) < 0.005, + 'Traffic has not converged after link flap') + logger.info('Traffic has converged after link flap') + + delta_frames = 0 + for i in range(0, len(traffic_type)): + delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx + pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate) + logger.info('Delta Frames : {}'.format(delta_frames)) + logger.info('PACKET LOSS DURATION After Link Down (ms): {}'.format(pkt_loss_duration)) + avg_pld.append(pkt_loss_duration) + + logger.info('Performing Clear Stats') + ixnetwork.ClearStats() + + # Link Up + if fanout_presence is False: + for snappi_port_name in snappi_port_names: + time.sleep(0.05) + ixn_port = ixnetwork.Vport.find(Name=snappi_port_name)[0] + ixn_port.LinkUpDn("up") + logger.info('Starting up snappi port : {}'.format(snappi_port_name)) + wait(SNAPPI_TRIGGER, "For links to shutdown") + else: + flap_fanout_ports(required_fanout_mapping, creds, state='up') + wait(DUT_TRIGGER, "For links to startup") + + logger.info('\n') + port_stats = get_port_stats(api) + logger.info('Rx Snappi Port Name : Rx Frame Rate') + for port_stat in port_stats: + if 'Snappi_Tx_Port' not in port_stat.name: + logger.info('{} : {}'.format(port_stat.name, port_stat.frames_rx_rate)) + pytest_assert(port_stat.frames_rx_rate > 0, '{} is not receiving any packet'.format(port_stat.name)) + + flow_stats = get_flow_stats(api) + delta_frames = 0 + for i in range(0, len(traffic_type)): + delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx + pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate) + logger.info('Delta Frames : {}'.format(delta_frames)) + logger.info('PACKET LOSS DURATION After Link Up (ms): {}'.format(pkt_loss_duration)) + avg_pld2.append(pkt_loss_duration) + logger.info('Stopping Traffic') + ts = api.transmit_state() + ts.state = ts.STOP + api.set_transmit_state(ts) + + logger.info("Stopping all protocols ...") + ps = api.protocol_state() + ps.state = ps.STOP + api.set_protocol_state(ps) + logger.info('\n') + + columns = ['Test Name', 'Iterations', 'Traffic Type', 'Uplink ECMP Paths', 'Route Count', + 'Avg Calculated Packet Loss Duration (ms)'] + logger.info("\n%s" % tabulate([[test_name+' (Link Down)', iteration, traffic_type, portchannel_count, + total_routes, mean(avg_pld)], [test_name+' (Link Up)', iteration, + traffic_type, portchannel_count, total_routes, mean(avg_pld2)]], headers=columns, + tablefmt="psql")) diff --git a/tests/snappi_tests/multidut/bgp/test_bgp_outbound_downlink_port_flap.py b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_downlink_port_flap.py new file mode 100755 index 0000000000..2c5b48533c --- /dev/null +++ b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_downlink_port_flap.py @@ -0,0 +1,96 @@ +import pytest +import logging +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ + snappi_api, multidut_snappi_ports_for_bgp # noqa: F401 +from tests.snappi_tests.variables import t1_t2_device_hostnames # noqa: F401 +from tests.snappi_tests.multidut.bgp.files.bgp_outbound_helper import ( + run_bgp_outbound_link_flap_test) # noqa: F401 +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams # noqa: F401 + +logger = logging.getLogger(__name__) + +pytestmark = [pytest.mark.topology('multidut-tgen')] + +FLAP_DETAILS = { + 'device_name': t1_t2_device_hostnames[0], + 'port_name': 'Ethernet120' + } + +ITERATION = 1 +ROUTE_RANGES = [{ + 'IPv4': [ + ['100.1.1.1', 24, 500], + ['200.1.1.1', 24, 500] + ], + 'IPv6': [ + ['5000::1', 64, 500], + ['4000::1', 64, 500] + ], + }, + { + 'IPv4': [ + ['100.1.1.1', 24, 2500], + ['200.1.1.1', 24, 2500] + ], + 'IPv6': [ + ['5000::1', 64, 2500], + ['4000::1', 64, 2500] + ], + }] + + +def test_bgp_outbound_downlink_port_flap(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): + """ + Gets the packet loss duration on flapping the interconnected port between T1 and downlink in T1 side + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + creds (pytest fixture): DUT credentials + Returns: + N/A + """ + + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.multi_dut_params.flap_details = FLAP_DETAILS + snappi_extra_params.test_name = "T1 Interconnectivity flap" + if (len(t1_t2_device_hostnames) < 3) or (len(duthosts) < 3): + pytest_assert(False, "Need minimum of 3 devices : One T1 and Two T2 line cards") + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_bgp_outbound_link_flap_test(api=snappi_api, + creds=creds, + snappi_extra_params=snappi_extra_params) diff --git a/tests/snappi_tests/multidut/bgp/test_bgp_outbound_downlink_process_crash.py b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_downlink_process_crash.py new file mode 100755 index 0000000000..a0ac0f9f15 --- /dev/null +++ b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_downlink_process_crash.py @@ -0,0 +1,94 @@ +import pytest +import logging +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ + snappi_api, multidut_snappi_ports_for_bgp # noqa: F401 +from tests.snappi_tests.variables import t1_t2_device_hostnames # noqa: F401 +from tests.snappi_tests.multidut.bgp.files.bgp_outbound_helper import ( + run_bgp_outbound_process_restart_test) # noqa: F401 +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams # noqa: F401 + +logger = logging.getLogger(__name__) + +pytestmark = [pytest.mark.topology('multidut-tgen')] + +ITERATION = 1 +ROUTE_RANGES = [{ + 'IPv4': [ + ['100.1.1.1', 24, 500], + ['200.1.1.1', 24, 500] + ], + 'IPv6': [ + ['5000::1', 64, 500], + ['4000::1', 64, 500] + ], + }, + { + 'IPv4': [ + ['100.1.1.1', 24, 2500], + ['200.1.1.1', 24, 2500] + ], + 'IPv6': [ + ['5000::1', 64, 2500], + ['4000::1', 64, 2500] + ], + }] + + +def test_bgp_outbound_downlink_process_crash(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, # noqa: F811 + creds): # noqa: F811 + """ + Gets the packet loss duration on killing certain processes in downlink + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + creds (dict): DUT credentials + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "T2 Downlink Process Crash" + snappi_extra_params.multi_dut_params.process_names = { + 'swss': "/usr/bin/orchagent", + 'syncd': "/usr/bin/syncd", + } + snappi_extra_params.multi_dut_params.host_name = t1_t2_device_hostnames[2] + if (len(t1_t2_device_hostnames) < 3) or (len(duthosts) < 3): + pytest_assert(False, "Need minimum of 3 devices : One T1 and Two T2 line cards") + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_bgp_outbound_process_restart_test(api=snappi_api, + creds=creds, + snappi_extra_params=snappi_extra_params) diff --git a/tests/snappi_tests/multidut/bgp/test_bgp_outbound_tsa.py b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_tsa.py new file mode 100644 index 0000000000..2db762e9dc --- /dev/null +++ b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_tsa.py @@ -0,0 +1,248 @@ +import pytest +import logging +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ + snappi_api, multidut_snappi_ports_for_bgp # noqa: F401 +from tests.snappi_tests.variables import t1_t2_device_hostnames # noqa: F401 +from tests.snappi_tests.multidut.bgp.files.bgp_outbound_helper import ( + run_bgp_outbound_tsa_tsb_test, run_dut_configuration) # noqa: F401 +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams # noqa: F401 + +logger = logging.getLogger(__name__) + +pytestmark = [pytest.mark.topology('multidut-tgen')] + +ITERATION = 1 +ROUTE_RANGES = [{ + 'IPv4': [ + ['100.1.1.1', 24, 500], + ['200.1.1.1', 24, 500] + ], + 'IPv6': [ + ['5000::1', 64, 500], + ['4000::1', 64, 500] + ], + }, + { + 'IPv4': [ + ['100.1.1.1', 24, 2500], + ['200.1.1.1', 24, 2500] + ], + 'IPv6': [ + ['5000::1', 64, 2500], + ['4000::1', 64, 2500] + ], + }] + + +def test_dut_configuration(multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts): # noqa: F811 + """ + Configures BGP in T1, T2 Uplink and T2 Downlink + + Args: + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.test_name = "Dut Configuration" + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + else: + continue + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_dut_configuration(snappi_extra_params) + + +def test_bgp_outbound_uplink_tsa(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): # noqa: F811 + """ + Gets the packet loss duration on issuing TSA/TSB in uplink + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + logger.info("uplink\n") + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "Uplink" + snappi_extra_params.device_name = t1_t2_device_hostnames[1] + + if (len(t1_t2_device_hostnames) < 3) or (len(duthosts) < 3): + pytest_assert(False, "Need minimum of 3 devices : One T1 and Two T2 line cards") + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + elif t1_t2_device_hostnames[3] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost4 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + + run_bgp_outbound_tsa_tsb_test(api=snappi_api, + snappi_extra_params=snappi_extra_params, + creds=creds, + is_supervisor=False) + + +def test_bgp_outbound_downlink_tsa(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): # noqa: F811 + """ + Gets the packet loss duration on issuing TSA/TSB in downlink + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + logger.info("downlink") + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "Downlink" + snappi_extra_params.device_name = t1_t2_device_hostnames[2] + + if (len(t1_t2_device_hostnames) < 3) or (len(duthosts) < 3): + pytest_assert(False, "Need minimum of 3 devices : One T1 and Two T2 line cards") + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + elif t1_t2_device_hostnames[3] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost4 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_bgp_outbound_tsa_tsb_test(api=snappi_api, + snappi_extra_params=snappi_extra_params, + creds=creds, + is_supervisor=False) + + +def test_bgp_outbound_supervisor_tsa(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): # noqa: F811 + """ + Gets the packet loss duration on issuing TSA/TSB in supervisor + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "Supervisor" + snappi_extra_params.device_name = t1_t2_device_hostnames[3] + + if (len(t1_t2_device_hostnames) < 3) or (len(duthosts) < 3): + pytest_assert(False, "Need minimum of 3 devices : One T1 and Two T2 line cards") + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + elif t1_t2_device_hostnames[3] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost4 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_bgp_outbound_tsa_tsb_test(api=snappi_api, + snappi_extra_params=snappi_extra_params, + creds=creds, + is_supervisor=True) diff --git a/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_multi_po_flap.py b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_multi_po_flap.py new file mode 100644 index 0000000000..a983e3642d --- /dev/null +++ b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_multi_po_flap.py @@ -0,0 +1,183 @@ +import pytest +import logging +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ + snappi_api, multidut_snappi_ports_for_bgp # noqa: F401 +from tests.snappi_tests.variables import t1_t2_device_hostnames # noqa: F401 +from tests.snappi_tests.multidut.bgp.files.bgp_outbound_helper import ( + run_bgp_outbound_uplink_blackout_test, run_dut_configuration) # noqa: F401 +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams # noqa: F401 + +logger = logging.getLogger(__name__) + +pytestmark = [pytest.mark.topology('multidut-tgen')] + +ITERATION = 1 +ROUTE_RANGES = [{ + 'IPv4': [ + ['100.1.1.1', 24, 500], + ['200.1.1.1', 24, 500] + ], + 'IPv6': [ + ['5000::1', 64, 500], + ['4000::1', 64, 500] + ], + }, + { + 'IPv4': [ + ['100.1.1.1', 24, 2500], + ['200.1.1.1', 24, 2500] + ], + 'IPv6': [ + ['5000::1', 64, 2500], + ['4000::1', 64, 2500] + ], + }] + + +def test_dut_configuration(multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts): # noqa: F811 + """ + Configures BGP in T1, T2 Uplink and T2 Downlink + + Args: + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.test_name = "Dut Configuration" + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + else: + continue + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_dut_configuration(snappi_extra_params) + + +def test_bgp_outbound_uplink_complete_blackout(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): + """ + Gets the packet loss duration on flapping all portchannels in uplink side + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + creds (pytest fixture): DUT credentials + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "T2 Uplink Complete Blackout" + snappi_extra_params.multi_dut_params.BLACKOUT_PERCENTAGE = 100 + + if (len(t1_t2_device_hostnames) < 3) or (len(duthosts) < 3): + pytest_assert(False, "Need minimum of 3 devices : One T1 and Two T2 line cards") + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_bgp_outbound_uplink_blackout_test(api=snappi_api, + creds=creds, + snappi_extra_params=snappi_extra_params) + + +def test_bgp_outbound_uplink_partial_blackout(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): + """ + Gets the packet loss duration on flapping 50% of portchannels in uplink side + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + creds (pytest fixture): DUT credentials + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "T2 Uplink Partial Blackout" + snappi_extra_params.multi_dut_params.BLACKOUT_PERCENTAGE = 50 + + if (len(t1_t2_device_hostnames) < 3) or (len(duthosts) < 3): + pytest_assert(False, "Need minimum of 3 devices : One T1 and Two T2 line cards") + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_bgp_outbound_uplink_blackout_test(api=snappi_api, + creds=creds, + snappi_extra_params=snappi_extra_params) diff --git a/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_po_flap.py b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_po_flap.py new file mode 100755 index 0000000000..59fa935e80 --- /dev/null +++ b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_po_flap.py @@ -0,0 +1,95 @@ +import pytest +import logging +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ + snappi_api, multidut_snappi_ports_for_bgp # noqa: F401 +from tests.snappi_tests.variables import t1_t2_device_hostnames # noqa: F401 +from tests.snappi_tests.multidut.bgp.files.bgp_outbound_helper import ( + run_bgp_outbound_link_flap_test) # noqa: F401 +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams # noqa: F401 + +logger = logging.getLogger(__name__) + +pytestmark = [pytest.mark.topology('multidut-tgen')] + +FLAP_DETAILS = { + 'device_name': 'Ixia', + 'port_name': 'Snappi_Uplink_PO_1_Link_1' + } + +ITERATION = 1 +ROUTE_RANGES = [{ + 'IPv4': [ + ['100.1.1.1', 24, 500], + ['200.1.1.1', 24, 500] + ], + 'IPv6': [ + ['5000::1', 64, 500], + ['4000::1', 64, 500] + ], + }, + { + 'IPv4': [ + ['100.1.1.1', 24, 2500], + ['200.1.1.1', 24, 2500] + ], + 'IPv6': [ + ['5000::1', 64, 2500], + ['4000::1', 64, 2500] + ], + }] + + +def test_bgp_outbound_uplink_po_flap(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): + """ + Gets the packet loss duration on flapping portchannel in uplink side + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + creds (pytest fixture): DUT credentials + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "T2 Uplink Portchannel Flap" + snappi_extra_params.multi_dut_params.flap_details = FLAP_DETAILS + + if (len(t1_t2_device_hostnames) < 3) or (len(duthosts) < 3): + pytest_assert(False, "Need minimum of 3 devices : One T1 and Two T2 line cards") + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_bgp_outbound_link_flap_test(api=snappi_api, + creds=creds, + snappi_extra_params=snappi_extra_params) diff --git a/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_po_member_flap.py b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_po_member_flap.py new file mode 100755 index 0000000000..3c273641a7 --- /dev/null +++ b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_po_member_flap.py @@ -0,0 +1,94 @@ +import pytest +import logging +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ + snappi_api, multidut_snappi_ports_for_bgp # noqa: F401 +from tests.snappi_tests.variables import t1_t2_device_hostnames # noqa: F401 +from tests.snappi_tests.multidut.bgp.files.bgp_outbound_helper import ( + run_bgp_outbound_link_flap_test) # noqa: F401 +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams # noqa: F401 + +logger = logging.getLogger(__name__) + +pytestmark = [pytest.mark.topology('multidut-tgen')] + +FLAP_DETAILS = { + 'device_name': 'Ixia', + 'port_name': 'Snappi_Uplink_PO_1_Link_1' + } + +ITERATION = 1 +ROUTE_RANGES = [{ + 'IPv4': [ + ['100.1.1.1', 24, 500], + ['200.1.1.1', 24, 500] + ], + 'IPv6': [ + ['5000::1', 64, 500], + ['4000::1', 64, 500] + ], + }, + { + 'IPv4': [ + ['100.1.1.1', 24, 2500], + ['200.1.1.1', 24, 2500] + ], + 'IPv6': [ + ['5000::1', 64, 2500], + ['4000::1', 64, 2500] + ], + }] + + +def test_bgp_outbound_uplink_po_member_flap(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): + """ + Gets the packet loss duration on flapping portchannel member in uplink side + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + creds (pytest fixture): DUT credentials + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "T2 Uplink Portchannel Member Flap" + snappi_extra_params.multi_dut_params.flap_details = FLAP_DETAILS + + if (len(t1_t2_device_hostnames) < 3) or (len(duthosts) < 3): + pytest_assert(False, "Need minimum of 3 devices : One T1 and Two T2 line cards") + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + else: + continue + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_bgp_outbound_link_flap_test(api=snappi_api, + creds=creds, + snappi_extra_params=snappi_extra_params) diff --git a/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_process_crash.py b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_process_crash.py new file mode 100755 index 0000000000..d27cde536b --- /dev/null +++ b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_uplink_process_crash.py @@ -0,0 +1,94 @@ +import pytest +import logging +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ + snappi_api, multidut_snappi_ports_for_bgp # noqa: F401 +from tests.snappi_tests.variables import t1_t2_device_hostnames # noqa: F401 +from tests.snappi_tests.multidut.bgp.files.bgp_outbound_helper import ( + run_bgp_outbound_process_restart_test) # noqa: F401 +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams # noqa: F401 + +logger = logging.getLogger(__name__) + +pytestmark = [pytest.mark.topology('multidut-tgen')] + +ITERATION = 1 +ROUTE_RANGES = [{ + 'IPv4': [ + ['100.1.1.1', 24, 500], + ['200.1.1.1', 24, 500] + ], + 'IPv6': [ + ['5000::1', 64, 500], + ['4000::1', 64, 500] + ], + }, + { + 'IPv4': [ + ['100.1.1.1', 24, 2500], + ['200.1.1.1', 24, 2500] + ], + 'IPv6': [ + ['5000::1', 64, 2500], + ['4000::1', 64, 2500] + ], + }] + + +def test_bgp_outbound_uplink_process_crash(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, # noqa: F811 + creds): # noqa: F811 + """ + Gets the packet loss duration on killing certain processes in uplink + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + creds (dict): DUT credentials + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "T2 Uplink Process Crash" + snappi_extra_params.multi_dut_params.process_names = { + 'swss': "/usr/bin/orchagent", + 'syncd': "/usr/bin/syncd", + } + snappi_extra_params.multi_dut_params.host_name = t1_t2_device_hostnames[1] + if (len(t1_t2_device_hostnames) < 3) or (len(duthosts) < 3): + pytest_assert(False, "Need minimum of 3 devices : One T1 and Two T2 line cards") + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + for device_hostname in t1_t2_device_hostnames: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_bgp_outbound_process_restart_test(api=snappi_api, + creds=creds, + snappi_extra_params=snappi_extra_params) diff --git a/tests/snappi_tests/multidut/ecn/files/multidut_helper.py b/tests/snappi_tests/multidut/ecn/files/multidut_helper.py index a9bca73098..76c2703131 100644 --- a/tests/snappi_tests/multidut/ecn/files/multidut_helper.py +++ b/tests/snappi_tests/multidut/ecn/files/multidut_helper.py @@ -7,7 +7,7 @@ from tests.common.snappi_tests.common_helpers import pfc_class_enable_vector, config_wred, \ enable_ecn, config_ingress_lossless_buffer_alpha, stop_pfcwd, disable_packet_aging, \ config_capture_pkt, traffic_flow_mode, calc_pfc_pause_flow_rate # noqa: F401 -from tests.common.snappi_tests.read_pcap import get_ip_pkts +from tests.common.snappi_tests.read_pcap import get_ipv4_pkts from tests.common.snappi_tests.snappi_test_params import SnappiTestParams from tests.common.snappi_tests.traffic_generation import setup_base_traffic_config, generate_test_flows, \ generate_pause_flows, run_traffic # noqa: F401 @@ -51,29 +51,32 @@ def run_ecn_test(api, if snappi_extra_params is None: snappi_extra_params = SnappiTestParams() - duthost1 = snappi_extra_params.multi_dut_params.duthost1 + # Traffic flow: + # tx_port (TGEN) --- ingress DUT --- egress DUT --- rx_port (TGEN) + rx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[0] - duthost2 = snappi_extra_params.multi_dut_params.duthost2 + egress_duthost = rx_port['duthost'] + tx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[1] - iters = snappi_extra_params.test_iterations + ingress_duthost = tx_port['duthost'] pytest_assert(testbed_config is not None, 'Failed to get L2/3 testbed config') logger.info("Stopping PFC watchdog") - stop_pfcwd(duthost1, rx_port['asic_value']) - stop_pfcwd(duthost2, tx_port['asic_value']) + stop_pfcwd(egress_duthost, rx_port['asic_value']) + stop_pfcwd(ingress_duthost, tx_port['asic_value']) logger.info("Disabling packet aging if necessary") - disable_packet_aging(duthost1) - disable_packet_aging(duthost2) + disable_packet_aging(egress_duthost) + disable_packet_aging(ingress_duthost) # Configure WRED/ECN thresholds logger.info("Configuring WRED and ECN thresholds") - config_result = config_wred(host_ans=duthost1, + config_result = config_wred(host_ans=egress_duthost, kmin=snappi_extra_params.ecn_params["kmin"], kmax=snappi_extra_params.ecn_params["kmax"], pmax=snappi_extra_params.ecn_params["pmax"]) pytest_assert(config_result is True, 'Failed to configure WRED/ECN at the DUT') - config_result = config_wred(host_ans=duthost2, + config_result = config_wred(host_ans=ingress_duthost, kmin=snappi_extra_params.ecn_params["kmin"], kmax=snappi_extra_params.ecn_params["kmax"], pmax=snappi_extra_params.ecn_params["pmax"]) @@ -81,20 +84,20 @@ def run_ecn_test(api, # Enable ECN marking logger.info("Enabling ECN markings") - pytest_assert(enable_ecn(host_ans=duthost1, prio=lossless_prio), 'Unable to enable ecn') - pytest_assert(enable_ecn(host_ans=duthost2, prio=lossless_prio), 'Unable to enable ecn') + pytest_assert(enable_ecn(host_ans=egress_duthost, prio=lossless_prio), 'Unable to enable ecn') + pytest_assert(enable_ecn(host_ans=ingress_duthost, prio=lossless_prio), 'Unable to enable ecn') - config_result = config_ingress_lossless_buffer_alpha(host_ans=duthost1, + config_result = config_ingress_lossless_buffer_alpha(host_ans=egress_duthost, alpha_log2=3) pytest_assert(config_result is True, 'Failed to configure PFC threshold to 8') - config_result = config_ingress_lossless_buffer_alpha(host_ans=duthost2, + config_result = config_ingress_lossless_buffer_alpha(host_ans=ingress_duthost, alpha_log2=3) pytest_assert(config_result is True, 'Failed to configure PFC threshold to 8') # Get the ID of the port to test - port_id = get_dut_port_id(dut_hostname=duthost1.hostname, + port_id = get_dut_port_id(dut_hostname=egress_duthost.hostname, dut_port=dut_port, conn_data=conn_data, fanout_data=fanout_data) @@ -167,7 +170,7 @@ def run_ecn_test(api, capture_name=snappi_extra_params.packet_capture_file) logger.info("Running traffic") - run_traffic(duthost=duthost1, + run_traffic(duthost=egress_duthost, api=api, config=testbed_config, data_flow_names=data_flow_names, @@ -175,6 +178,6 @@ def run_ecn_test(api, exp_dur_sec=EXP_DURATION_SEC, snappi_extra_params=snappi_extra_params) - result.append(get_ip_pkts(snappi_extra_params.packet_capture_file + ".pcapng")) + result.append(get_ipv4_pkts(snappi_extra_params.packet_capture_file + ".pcapng")) return result diff --git a/tests/snappi_tests/multidut/ecn/test_multidut_dequeue_ecn_with_snappi.py b/tests/snappi_tests/multidut/ecn/test_multidut_dequeue_ecn_with_snappi.py index 60aab367ed..299134eee2 100644 --- a/tests/snappi_tests/multidut/ecn/test_multidut_dequeue_ecn_with_snappi.py +++ b/tests/snappi_tests/multidut/ecn/test_multidut_dequeue_ecn_with_snappi.py @@ -2,36 +2,33 @@ import random import logging -from tests.common.helpers.assertions import pytest_assert, pytest_require +from tests.common.helpers.assertions import pytest_assert, pytest_require # noqa: F401 from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ - fanout_graph_facts # noqa: F401 + fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, lossless_prio_list # noqa F401 -from tests.snappi_tests.variables import config_set, line_card_choice +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.multidut.ecn.files.multidut_helper import run_ecn_test from tests.common.snappi_tests.read_pcap import is_ecn_marked from tests.snappi_tests.files.helper import skip_ecn_tests from tests.common.snappi_tests.common_helpers import packet_capture -from tests.common.config_reload import config_reload from tests.common.snappi_tests.snappi_test_params import SnappiTestParams logger = logging.getLogger(__name__) pytestmark = [pytest.mark.topology('multidut-tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_dequeue_ecn(request, snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, - rand_one_dut_lossless_prio, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa: F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, # noqa: F811 prio_dscp_map): # noqa: F811 """ Test if the device under test (DUT) performs ECN marking at the egress @@ -40,54 +37,57 @@ def test_dequeue_ecn(request, request (pytest fixture): pytest request object snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs + lossless_prio_list (pytest fixture): list of all the lossless priorities rand_one_dut_lossless_prio (str): name of lossless priority to test, e.g., 's6100-1|3' line_card_choice: Line card choice to be mentioned in the variable.py file linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). - + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - pytest_require(False, "Need Minimum of 2 ports for the test") - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) - - _, lossless_prio = rand_one_dut_lossless_prio.split('|') - skip_ecn_tests(duthost1) - skip_ecn_tests(duthost2) + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + lossless_prio = random.sample(lossless_prio_list, 1) + skip_ecn_tests(snappi_ports[0]['duthost']) + skip_ecn_tests(snappi_ports[1]['duthost']) lossless_prio = int(lossless_prio) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 + snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports snappi_extra_params.packet_capture_type = packet_capture.IP_CAPTURE snappi_extra_params.is_snappi_ingress_port_cap = True snappi_extra_params.ecn_params = {'kmin': 50000, 'kmax': 51000, 'pmax': 100} data_flow_pkt_size = 1024 data_flow_pkt_count = 101 + num_iterations = 1 logger.info("Running ECN dequeue test with params: {}".format(snappi_extra_params.ecn_params)) snappi_extra_params.traffic_flow_config.data_flow_config = { @@ -99,11 +99,11 @@ def test_dequeue_ecn(request, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], lossless_prio=lossless_prio, prio_dscp_map=prio_dscp_map, - iters=1, + iters=num_iterations, snappi_extra_params=snappi_extra_params)[0] logger.info("Running verification for ECN dequeue test") @@ -117,8 +117,4 @@ def test_dequeue_ecn(request, # Check if the last packet is not ECN marked pytest_assert(not is_ecn_marked(ip_pkts[-1]), "The last packet should not be marked") - - # Teardown ECN config through a reload - logger.info("Reloading config to teardown ECN config") - config_reload(sonic_host=duthost1, config_source='config_db', safe_reload=True) - config_reload(sonic_host=duthost2, config_source='config_db', safe_reload=True) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/ecn/test_multidut_red_accuracy_with_snappi.py b/tests/snappi_tests/multidut/ecn/test_multidut_red_accuracy_with_snappi.py index e813e8dfbb..27a8a6906c 100644 --- a/tests/snappi_tests/multidut/ecn/test_multidut_red_accuracy_with_snappi.py +++ b/tests/snappi_tests/multidut/ecn/test_multidut_red_accuracy_with_snappi.py @@ -3,35 +3,32 @@ import random import logging from tabulate import tabulate # noqa F401 -from tests.common.helpers.assertions import pytest_assert, pytest_require -from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 +from tests.common.helpers.assertions import pytest_assert, pytest_require # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, \ lossless_prio_list # noqa F401 -from tests.snappi_tests.variables import config_set, line_card_choice +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.files.helper import skip_ecn_tests from tests.common.snappi_tests.read_pcap import is_ecn_marked from tests.snappi_tests.multidut.ecn.files.multidut_helper import run_ecn_test from tests.common.snappi_tests.common_helpers import packet_capture # noqa F401 from tests.common.snappi_tests.snappi_test_params import SnappiTestParams -from tests.common.config_reload import config_reload logger = logging.getLogger(__name__) pytestmark = [pytest.mark.topology('multidut-tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_red_accuracy(request, snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, - rand_one_dut_lossless_prio, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa: F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, # noqa: F811 prio_dscp_map): # noqa: F811 """ Measure RED/ECN marking accuracy of the device under test (DUT). @@ -43,12 +40,11 @@ def test_red_accuracy(request, conn_graph_facts (pytest fixture): connection graph fanout_graph_facts (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs - rand_one_dut_lossless_prio (str): name of lossless priority to test, e.g., 's6100-1|3' + lossless_prio_list (pytest fixture): list of all the lossless priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). - + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ @@ -56,43 +52,42 @@ def test_red_accuracy(request, # if disable_test: # pytest.skip("test_red_accuracy is disabled") - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] # noqa: E501 - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - pytest_require(False, "Need Minimum of 2 ports for the test") - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) - - _, lossless_prio = rand_one_dut_lossless_prio.split('|') - skip_ecn_tests(duthost1) or skip_ecn_tests(duthost2) - lossless_prio = int(lossless_prio) + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + skip_ecn_tests(snappi_ports[0]['duthost']) or skip_ecn_tests(snappi_ports[1]['duthost']) + lossless_prio = random.sample(lossless_prio_list, 1) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports snappi_extra_params.packet_capture_type = packet_capture.IP_CAPTURE snappi_extra_params.is_snappi_ingress_port_cap = True - snappi_extra_params.ecn_params = {'kmin': 500000, 'kmax': 2000000, 'pmax': 5} + snappi_extra_params.ecn_params = {'kmin': 500000, 'kmax': 900000, 'pmax': 5} data_flow_pkt_size = 1024 - data_flow_pkt_count = 2100 + data_flow_pkt_count = 910 num_iterations = 1 logger.info("Running ECN red accuracy test with ECN params: {}".format(snappi_extra_params.ecn_params)) @@ -107,7 +102,7 @@ def test_red_accuracy(request, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], lossless_prio=lossless_prio, prio_dscp_map=prio_dscp_map, @@ -145,8 +140,4 @@ def test_red_accuracy(request, for queue, mark_cnt in list(queue_mark_cnt.items()): output_table.append([queue, float(mark_cnt)/num_iterations]) logger.info(tabulate(output_table, headers=['Queue Length', 'ECN Marking Probability'])) - - # Teardown ECN config through a reload - logger.info("Reloading config to teardown ECN config") - config_reload(sonic_host=duthost1, config_source='config_db', safe_reload=True) - config_reload(sonic_host=duthost2, config_source='config_db', safe_reload=True) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfc/files/lossless_response_to_external_pause_storms_helper.py b/tests/snappi_tests/multidut/pfc/files/lossless_response_to_external_pause_storms_helper.py index a09c3591d4..79223c283d 100644 --- a/tests/snappi_tests/multidut/pfc/files/lossless_response_to_external_pause_storms_helper.py +++ b/tests/snappi_tests/multidut/pfc/files/lossless_response_to_external_pause_storms_helper.py @@ -5,6 +5,7 @@ # Compiled at: 2023-02-10 09:15:26 from math import ceil # noqa: F401 import logging # noqa: F401 +import random from tests.common.helpers.assertions import pytest_assert, pytest_require # noqa: F401 from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 from tests.common.snappi_tests.snappi_helpers import get_dut_port_id # noqa: F401 @@ -13,8 +14,8 @@ from tests.common.snappi_tests.port import select_ports # noqa: F401 from tests.common.snappi_tests.snappi_test_params import SnappiTestParams from tests.common.snappi_tests.traffic_generation import run_traffic, verify_pause_flow, \ - setup_base_traffic_config, verify_m2o_oversubscribtion_results # noqa: F401 - + setup_base_traffic_config # noqa: F401 +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) TEST_FLOW_NAME = 'Test Flow' @@ -27,7 +28,6 @@ PAUSE_FLOW_DELAY_SEC = 5 DATA_FLOW_DELAY_SEC = 0 SNAPPI_POLL_DELAY_SEC = 2 -TOLERANCE_THRESHOLD = 0.05 PAUSE_FLOW_RATE = 15 PAUSE_FLOW_NAME = 'PFC Traffic' @@ -65,19 +65,29 @@ def run_lossless_response_to_external_pause_storms_test(api, if snappi_extra_params is None: snappi_extra_params = SnappiTestParams() - duthost1 = snappi_extra_params.multi_dut_params.duthost1 + # Traffic flow: + # tx_port (TGEN) --- ingress DUT --- egress DUT --- rx_port (TGEN) + + # initialize the (duthost, port) set. + dut_asics_to_be_configured = set() + rx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[0] rx_port_id_list = [rx_port["port_id"]] - duthost2 = snappi_extra_params.multi_dut_params.duthost2 + egress_duthost = rx_port['duthost'] + dut_asics_to_be_configured.add((egress_duthost, rx_port['asic_value'])) + tx_port = [snappi_extra_params.multi_dut_params.multi_dut_ports[1], snappi_extra_params.multi_dut_params.multi_dut_ports[2]] tx_port_id_list = [tx_port[0]["port_id"], tx_port[1]["port_id"]] + # add ingress DUT into the set + dut_asics_to_be_configured.add((tx_port[0]['duthost'], tx_port[0]['asic_value'])) + dut_asics_to_be_configured.add((tx_port[1]['duthost'], tx_port[1]['asic_value'])) pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') - stop_pfcwd(duthost1, rx_port['asic_value']) - disable_packet_aging(duthost1) - stop_pfcwd(duthost2, tx_port[0]['asic_value']) - disable_packet_aging(duthost2) + # Disable PFC watchdog on the rx side and tx side of the DUT + for duthost, asic in dut_asics_to_be_configured: + stop_pfcwd(duthost, asic) + disable_packet_aging(duthost) test_flow_rate_percent = int(TEST_FLOW_AGGR_RATE_PERCENT) bg_flow_rate_percent = int(BG_FLOW_AGGR_RATE_PERCENT) @@ -110,29 +120,17 @@ def run_lossless_response_to_external_pause_storms_test(api, data_flow_names = [flow.name for flow in flows if PAUSE_FLOW_NAME not in flow.name] """ Run traffic """ - flow_stats, switch_flow_stats = run_traffic(duthost=duthost1, - api=api, - config=testbed_config, - data_flow_names=data_flow_names, - all_flow_names=all_flow_names, - exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC, - snappi_extra_params=snappi_extra_params) - flag = { - 'Test Flow': { - 'loss': '0' - }, - 'Background Flow': { - 'loss': '0' - } - } - - verify_m2o_oversubscribtion_results(duthost=duthost2, - rows=flow_stats, - test_flow_name=TEST_FLOW_NAME, - bg_flow_name=BG_FLOW_NAME, - rx_port=rx_port, - rx_frame_count_deviation=TOLERANCE_THRESHOLD, - flag=flag) + flow_stats, switch_flow_stats, _ = run_traffic(duthost=egress_duthost, + api=api, + config=testbed_config, + data_flow_names=data_flow_names, + all_flow_names=all_flow_names, + exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC, + snappi_extra_params=snappi_extra_params) + + verify_external_pause_storm_result(flow_stats, + tx_port, + rx_port) # Verify pause flows verify_pause_flow(flow_metrics=flow_stats, @@ -310,32 +308,44 @@ def __gen_data_flow(testbed_config, flow = testbed_config.flows.flow(name='{} {} -> {}'.format(flow_name_prefix, src_port_id, dst_port_id))[-1] flow.tx_rx.port.tx_name = testbed_config.ports[src_port_id].name flow.tx_rx.port.rx_name = testbed_config.ports[dst_port_id].name - eth, ipv4 = flow.packet.ethernet().ipv4() + eth, ipv4, udp = flow.packet.ethernet().ipv4().udp() + src_port = random.randint(5000, 6000) + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + eth.src.value = tx_mac eth.dst.value = rx_mac - if 'Background Flow' in flow.name: - eth.pfc_queue.value = 0 - elif 'Test Flow 1 -> 0' in flow.name: - eth.pfc_queue.value = 3 + if pfcQueueGroupSize == 8: + if 'Background Flow' in flow.name: + eth.pfc_queue.value = 0 + elif 'Test Flow 1 -> 0' in flow.name: + eth.pfc_queue.value = flow_prio[0] + elif 'Test Flow 2 -> 0' in flow.name: + eth.pfc_queue.value = flow_prio[1] else: - eth.pfc_queue.value = 4 + if 'Background Flow' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[1] + elif 'Test Flow 1 -> 0' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio[0]] + elif 'Test Flow 2 -> 0' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio[1]] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip ipv4.priority.choice = ipv4.priority.DSCP - flow_prio_dscp_list = [] + if 'Background Flow 1 -> 0' in flow.name: ipv4.priority.dscp.phb.values = [ ipv4.priority.dscp.phb.CS2, ] elif 'Background Flow 2 -> 0' in flow.name: ipv4.priority.dscp.phb.values = [5] - else: - for fp in flow_prio: - for val in prio_dscp_map[fp]: - flow_prio_dscp_list.append(val) - ipv4.priority.dscp.phb.values = flow_prio_dscp_list + elif 'Test Flow 1 -> 0' in flow.name: + ipv4.priority.dscp.phb.values = prio_dscp_map[flow_prio[0]] + elif 'Test Flow 2 -> 0' in flow.name: + ipv4.priority.dscp.phb.values = prio_dscp_map[flow_prio[1]] ipv4.priority.dscp.ecn.value = ipv4.priority.dscp.ecn.CAPABLE_TRANSPORT_1 flow.duration.fixed_seconds.delay.nanoseconds = int(sec_to_nanosec(DATA_FLOW_DELAY_SEC)) @@ -384,3 +394,29 @@ def __gen_data_flow(testbed_config, pause_flow.duration.fixed_seconds.seconds = PAUSE_FLOW_DURATION_SEC pause_flow.metrics.enable = True pause_flow.metrics.loss = True + + +def verify_external_pause_storm_result(rows, + tx_port, + rx_port): + """ + Verifies the required loss % from the Traffic Items Statistics + + Args: + rows (list): Traffic Item Statistics from snappi config + tx_port (list): Ingress Ports + rx_port : Egress Port + Returns: + N/A + """ + for row in rows: + if 'Test Flow 1 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss".format(row.name)) + elif 'Test Flow 2 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) + elif 'Background Flow 1 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) + elif 'Background Flow 2 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) + elif 'PFC' in row.name: + pytest_assert(int(row.loss) == 100, "{} must have 100% loss ".format(row.name)) diff --git a/tests/snappi_tests/multidut/pfc/files/lossless_response_to_throttling_pause_storms_helper.py b/tests/snappi_tests/multidut/pfc/files/lossless_response_to_throttling_pause_storms_helper.py index e7f39031ea..c77e13a692 100644 --- a/tests/snappi_tests/multidut/pfc/files/lossless_response_to_throttling_pause_storms_helper.py +++ b/tests/snappi_tests/multidut/pfc/files/lossless_response_to_throttling_pause_storms_helper.py @@ -5,6 +5,7 @@ # Compiled at: 2023-02-10 09:15:26 from math import ceil # noqa: F401 import logging # noqa: F401 +import random from tests.common.helpers.assertions import pytest_assert, pytest_require # noqa: F401 from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 from tests.common.snappi_tests.snappi_helpers import get_dut_port_id # noqa: F401 @@ -13,14 +14,14 @@ from tests.common.snappi_tests.port import select_ports # noqa: F401 from tests.common.snappi_tests.snappi_test_params import SnappiTestParams from tests.common.snappi_tests.traffic_generation import run_traffic, verify_pause_flow, \ - setup_base_traffic_config, verify_m2o_oversubscribtion_results # noqa: F401 - + setup_base_traffic_config # noqa: F401 +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) PAUSE_FLOW_NAME = 'Pause Storm' TEST_FLOW_NAME = 'Test Flow' -TEST_FLOW_AGGR_RATE_PERCENT = 30 +TEST_FLOW_AGGR_RATE_PERCENT = 25 BG_FLOW_NAME = 'Background Flow' PAUSE_FLOW_NAME = 'PFC Traffic' BG_FLOW_AGGR_RATE_PERCENT = 25 @@ -69,19 +70,30 @@ def run_lossless_response_to_throttling_pause_storms_test(api, if snappi_extra_params is None: snappi_extra_params = SnappiTestParams() - duthost1 = snappi_extra_params.multi_dut_params.duthost1 + # Traffic flow: + # tx_port (TGEN) --- ingress DUT --- egress DUT --- rx_port (TGEN) + + # initialize the (duthost, port) set. + dut_asics_to_be_configured = set() + rx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[0] rx_port_id_list = [rx_port["port_id"]] - duthost2 = snappi_extra_params.multi_dut_params.duthost2 + egress_duthost = rx_port['duthost'] + dut_asics_to_be_configured.add((egress_duthost, rx_port['asic_value'])) + tx_port = [snappi_extra_params.multi_dut_params.multi_dut_ports[1], snappi_extra_params.multi_dut_params.multi_dut_ports[2]] tx_port_id_list = [tx_port[0]["port_id"], tx_port[1]["port_id"]] + # add ingress DUT into the set + dut_asics_to_be_configured.add((tx_port[0]['duthost'], tx_port[0]['asic_value'])) + dut_asics_to_be_configured.add((tx_port[1]['duthost'], tx_port[1]['asic_value'])) pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') - stop_pfcwd(duthost1, rx_port['asic_value']) - disable_packet_aging(duthost1) - stop_pfcwd(duthost2, tx_port[0]['asic_value']) - disable_packet_aging(duthost2) + + # Disable PFC watchdog on the rx side and tx side of the DUT + for duthost, asic in dut_asics_to_be_configured: + stop_pfcwd(duthost, asic) + disable_packet_aging(duthost) test_flow_rate_percent = int(TEST_FLOW_AGGR_RATE_PERCENT) bg_flow_rate_percent = int(BG_FLOW_AGGR_RATE_PERCENT) @@ -114,29 +126,17 @@ def run_lossless_response_to_throttling_pause_storms_test(api, data_flow_names = [flow.name for flow in flows if PAUSE_FLOW_NAME not in flow.name] """ Run traffic """ - flow_stats, switch_flow_stats = run_traffic(duthost=duthost1, - api=api, - config=testbed_config, - data_flow_names=data_flow_names, - all_flow_names=all_flow_names, - exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC, - snappi_extra_params=snappi_extra_params) - flag = { - 'Test Flow': { - 'loss': '0' - }, - 'Background Flow': { - 'loss': '0' - }, - } - - verify_m2o_oversubscribtion_results(duthost=duthost2, - rows=flow_stats, - test_flow_name=TEST_FLOW_NAME, - bg_flow_name=BG_FLOW_NAME, - rx_port=rx_port, - rx_frame_count_deviation=TOLERANCE_THRESHOLD, - flag=flag) + flow_stats, switch_flow_stats, _ = run_traffic(duthost=egress_duthost, + api=api, + config=testbed_config, + data_flow_names=data_flow_names, + all_flow_names=all_flow_names, + exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC, + snappi_extra_params=snappi_extra_params) + + verify_throttling_pause_storm_result(flow_stats, + tx_port, + rx_port) # Verify pause flows verify_pause_flow(flow_metrics=flow_stats, @@ -314,21 +314,33 @@ def __gen_data_flow(testbed_config, flow = testbed_config.flows.flow(name='{} {} -> {}'.format(flow_name_prefix, src_port_id, dst_port_id))[-1] flow.tx_rx.port.tx_name = testbed_config.ports[src_port_id].name flow.tx_rx.port.rx_name = testbed_config.ports[dst_port_id].name - eth, ipv4 = flow.packet.ethernet().ipv4() + eth, ipv4, udp = flow.packet.ethernet().ipv4().udp() + src_port = random.randint(5000, 6000) + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + eth.src.value = tx_mac eth.dst.value = rx_mac - if 'Background Flow' in flow.name: - eth.pfc_queue.value = 0 - elif 'Test Flow 1 -> 0 Prio [3, 4]' in flow.name: - eth.pfc_queue.value = 3 + if pfcQueueGroupSize == 8: + if 'Background Flow' in flow.name: + eth.pfc_queue.value = 0 + elif 'Test Flow 1 -> 0' in flow.name: + eth.pfc_queue.value = flow_prio[0] + elif 'Test Flow 2 -> 0' in flow.name: + eth.pfc_queue.value = flow_prio[1] else: - eth.pfc_queue.value = 4 + if 'Background Flow' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[1] + elif 'Test Flow 1 -> 0' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio[0]] + elif 'Test Flow 2 -> 0' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio[1]] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip ipv4.priority.choice = ipv4.priority.DSCP - flow_prio_dscp_list = [] if 'Background Flow 1 -> 0' in flow.name: ipv4.priority.dscp.phb.values = [ ipv4.priority.dscp.phb.CS2, @@ -336,11 +348,10 @@ def __gen_data_flow(testbed_config, elif 'Background Flow 2 -> 0' in flow.name: ipv4.priority.dscp.phb.value = ipv4.priority.dscp.phb.DEFAULT ipv4.priority.dscp.phb.value = 5 - else: - for fp in flow_prio: - for val in prio_dscp_map[fp]: - flow_prio_dscp_list.append(val) - ipv4.priority.dscp.phb.values = flow_prio_dscp_list + elif 'Test Flow 1 -> 0' in flow.name: + ipv4.priority.dscp.phb.values = prio_dscp_map[flow_prio[0]] + elif 'Test Flow 2 -> 0' in flow.name: + ipv4.priority.dscp.phb.values = prio_dscp_map[flow_prio[1]] ipv4.priority.dscp.ecn.value = ipv4.priority.dscp.ecn.CAPABLE_TRANSPORT_1 flow.duration.fixed_seconds.delay.nanoseconds = int(sec_to_nanosec(DATA_FLOW_DELAY_SEC)) @@ -389,3 +400,29 @@ def __gen_data_flow(testbed_config, pause_flow.duration.fixed_seconds.seconds = PAUSE_FLOW_DURATION_SEC pause_flow.metrics.enable = True pause_flow.metrics.loss = True + + +def verify_throttling_pause_storm_result(rows, + tx_port, + rx_port): + """ + Verifies the required loss % from the Traffic Items Statistics + + Args: + rows (list): Traffic Item Statistics from snappi config + tx_port (list): Ingress Ports + rx_port : Egress Port + Returns: + N/A + """ + for row in rows: + if 'Test Flow 1 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss".format(row.name)) + elif 'Test Flow 2 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) + elif 'Background Flow 1 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) + elif 'Background Flow 2 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) + elif 'PFC' in row.name: + pytest_assert(int(row.loss) == 100, "{} must have 100% loss ".format(row.name)) diff --git a/tests/snappi_tests/multidut/pfc/files/m2o_fluctuating_lossless_helper.py b/tests/snappi_tests/multidut/pfc/files/m2o_fluctuating_lossless_helper.py new file mode 100644 index 0000000000..cb2d69d9c0 --- /dev/null +++ b/tests/snappi_tests/multidut/pfc/files/m2o_fluctuating_lossless_helper.py @@ -0,0 +1,378 @@ +import logging # noqa: F401 +import random +from tests.common.helpers.assertions import pytest_assert, pytest_require # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 +from tests.common.snappi_tests.snappi_helpers import get_dut_port_id # noqa: F401 +from tests.common.snappi_tests.common_helpers import pfc_class_enable_vector, stop_pfcwd, \ + disable_packet_aging, sec_to_nanosec # noqa: F401 +from tests.common.snappi_tests.port import select_ports # noqa: F401 +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams +from tests.common.snappi_tests.traffic_generation import run_traffic, \ + setup_base_traffic_config # noqa: F401 +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict +logger = logging.getLogger(__name__) + +PAUSE_FLOW_NAME = 'Pause Storm' +TEST_FLOW_NAME = 'Test Flow' +TEST_FLOW_AGGR_RATE_PERCENT = [20, 10] +BG_FLOW_NAME = 'Background Flow' +BG_FLOW_AGGR_RATE_PERCENT = [20, 20] +DATA_PKT_SIZE = 1024 +DATA_FLOW_DURATION_SEC = 20 +DATA_FLOW_DELAY_SEC = 10 +SNAPPI_POLL_DELAY_SEC = 2 + + +def run_m2o_fluctuating_lossless_test(api, + testbed_config, + port_config_list, + conn_data, + fanout_data, + dut_port, + pause_prio_list, + test_prio_list, + bg_prio_list, + prio_dscp_map, + snappi_extra_params=None): + """ + Run PFC Fluctuating Lossless Traffic Congestion with many to one traffic pattern + + Args: + api (obj): SNAPPI session + testbed_config (obj): testbed L1/L2/L3 configuration + port_config_list (list): list of port configuration + conn_data (dict): the dictionary returned by conn_graph_fact. + fanout_data (dict): the dictionary returned by fanout_graph_fact. + duthost (Ansible host instance): device under test + dut_port (str): DUT port to test + pause_prio_list (list): priorities to pause for PFC pause storm + test_prio_list (list): priorities of test flows + bg_prio_list (list): priorities of background flows + prio_dscp_map (dict): Priority vs. DSCP map (key = priority) + snappi_extra_params (SnappiTestParams obj): additional parameters for Snappi traffic + + Returns: + N/A + """ + if snappi_extra_params is None: + snappi_extra_params = SnappiTestParams() + + # Traffic flow: + # tx_port (TGEN) --- ingress DUT --- egress DUT --- rx_port (TGEN) + + # initialize the (duthost, port) set. + dut_asics_to_be_configured = set() + + rx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[0] + rx_port_id_list = [rx_port["port_id"]] + egress_duthost = rx_port['duthost'] + dut_asics_to_be_configured.add((egress_duthost, rx_port['asic_value'])) + + tx_port = [snappi_extra_params.multi_dut_params.multi_dut_ports[1], + snappi_extra_params.multi_dut_params.multi_dut_ports[2]] + tx_port_id_list = [tx_port[0]["port_id"], tx_port[1]["port_id"]] + # add ingress DUT into the set + dut_asics_to_be_configured.add((tx_port[0]['duthost'], tx_port[0]['asic_value'])) + dut_asics_to_be_configured.add((tx_port[1]['duthost'], tx_port[1]['asic_value'])) + + pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') + + # Disable PFC watchdog on the rx side and tx side of the DUT + for duthost, asic in dut_asics_to_be_configured: + stop_pfcwd(duthost, asic) + disable_packet_aging(duthost) + + port_id = 0 + # Generate base traffic config + snappi_extra_params.base_flow_config = setup_base_traffic_config(testbed_config=testbed_config, + port_config_list=port_config_list, + port_id=port_id) + __gen_traffic(testbed_config=testbed_config, + port_config_list=port_config_list, + rx_port_id_list=rx_port_id_list, + tx_port_id_list=tx_port_id_list, + pause_flow_name=PAUSE_FLOW_NAME, + pause_prio_list=pause_prio_list, + test_flow_name=TEST_FLOW_NAME, + test_flow_prio_list=test_prio_list, + test_flow_rate_percent=TEST_FLOW_AGGR_RATE_PERCENT, + bg_flow_name=BG_FLOW_NAME, + bg_flow_prio_list=bg_prio_list, + bg_flow_rate_percent=BG_FLOW_AGGR_RATE_PERCENT, + data_flow_dur_sec=DATA_FLOW_DURATION_SEC, + data_pkt_size=DATA_PKT_SIZE, + prio_dscp_map=prio_dscp_map) + + flows = testbed_config.flows + all_flow_names = [flow.name for flow in flows] + data_flow_names = [flow.name for flow in flows if PAUSE_FLOW_NAME not in flow.name] + + """ Run traffic """ + flow_stats, switch_flow_stats, _ = run_traffic(duthost=egress_duthost, + api=api, + config=testbed_config, + data_flow_names=data_flow_names, + all_flow_names=all_flow_names, + exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC, + snappi_extra_params=snappi_extra_params) + + verify_m2o_fluctuating_lossless_result(flow_stats, + tx_port, + rx_port) + + +def __gen_traffic(testbed_config, + port_config_list, + rx_port_id_list, + tx_port_id_list, + pause_flow_name, + pause_prio_list, + test_flow_name, + test_flow_prio_list, + test_flow_rate_percent, + bg_flow_name, + bg_flow_prio_list, + bg_flow_rate_percent, + data_flow_dur_sec, + data_pkt_size, + prio_dscp_map): + """ + Generate configurations of flows under all to all traffic pattern, including + test flows, background flows and pause storm. Test flows and background flows + are also known as data flows. + + Args: + testbed_config (obj): testbed L1/L2/L3 configuration + port_config_list (list): list of port configuration + port_id (int): ID of DUT port to test. + pause_flow_name (str): name of pause storm + pause_prio_list (list): priorities to pause for PFC frames + test_flow_name (str): name prefix of test flows + test_prio_list (list): priorities of test flows + test_flow_rate_percent (int): rate percentage for each test flow + bg_flow_name (str): name prefix of background flows + bg_prio_list (list): priorities of background flows + bg_flow_rate_percent (int): rate percentage for each background flow + data_flow_dur_sec (int): duration of data flows in second + pfc_storm_dur_sec (float): duration of the pause storm in second + data_pkt_size (int): packet size of data flows in byte + prio_dscp_map (dict): Priority vs. DSCP map (key = priority). + + Returns: + N/A + """ + __gen_data_flows(testbed_config=testbed_config, + port_config_list=port_config_list, + src_port_id_list=tx_port_id_list, + dst_port_id_list=rx_port_id_list, + flow_name_prefix=TEST_FLOW_NAME, + flow_prio_list=test_flow_prio_list, + flow_rate_percent=TEST_FLOW_AGGR_RATE_PERCENT, + flow_dur_sec=data_flow_dur_sec, + data_pkt_size=data_pkt_size, + prio_dscp_map=prio_dscp_map) + + __gen_data_flows(testbed_config=testbed_config, + port_config_list=port_config_list, + src_port_id_list=tx_port_id_list, + dst_port_id_list=rx_port_id_list, + flow_name_prefix=BG_FLOW_NAME, + flow_prio_list=bg_flow_prio_list, + flow_rate_percent=BG_FLOW_AGGR_RATE_PERCENT, + flow_dur_sec=data_flow_dur_sec, + data_pkt_size=data_pkt_size, + prio_dscp_map=prio_dscp_map) + + +def __gen_data_flows(testbed_config, + port_config_list, + src_port_id_list, + dst_port_id_list, + flow_name_prefix, + flow_prio_list, + flow_rate_percent, + flow_dur_sec, + data_pkt_size, + prio_dscp_map): + """ + Generate the configuration for data flows + + Args: + testbed_config (obj): testbed L1/L2/L3 configuration + port_config_list (list): list of port configuration + src_port_id_list (list): IDs of source ports + dst_port_id_list (list): IDs of destination ports + flow_name_prefix (str): prefix of flows' names + flow_prio_list (list): priorities of data flows + flow_rate_percent (int): rate percentage for each flow + flow_dur_sec (int): duration of each flow in second + data_pkt_size (int): packet size of data flows in byte + prio_dscp_map (dict): Priority vs. DSCP map (key = priority). + + Returns: + N/A + """ + if TEST_FLOW_NAME in flow_name_prefix: + for index, src_port_id in enumerate(src_port_id_list): + for dst_port_id in dst_port_id_list: + if src_port_id == dst_port_id: + continue + __gen_data_flow(testbed_config=testbed_config, + port_config_list=port_config_list, + src_port_id=src_port_id, + dst_port_id=dst_port_id, + flow_name_prefix=flow_name_prefix, + flow_prio=flow_prio_list, + flow_rate_percent=flow_rate_percent[index], + flow_dur_sec=flow_dur_sec, + data_pkt_size=data_pkt_size, + prio_dscp_map=prio_dscp_map, + index=None) + else: + index = 1 + for rate_percent in flow_rate_percent: + for src_port_id in src_port_id_list: + for dst_port_id in dst_port_id_list: + if src_port_id == dst_port_id: + continue + __gen_data_flow(testbed_config=testbed_config, + port_config_list=port_config_list, + src_port_id=src_port_id, + dst_port_id=dst_port_id, + flow_name_prefix=flow_name_prefix, + flow_prio=flow_prio_list, + flow_rate_percent=rate_percent, + flow_dur_sec=flow_dur_sec, + data_pkt_size=data_pkt_size, + prio_dscp_map=prio_dscp_map, + index=index) + index += 1 + + +def __gen_data_flow(testbed_config, + port_config_list, + src_port_id, + dst_port_id, + flow_name_prefix, + flow_prio, + flow_rate_percent, + flow_dur_sec, + data_pkt_size, + prio_dscp_map, + index): + """ + Generate the configuration for a data flow + + Args: + testbed_config (obj): testbed L1/L2/L3 configuration + port_config_list (list): list of port configuration + src_port_id (int): ID of the source port + dst_port_id (int): ID of destination port + flow_name_prefix (str): prefix of flow' name + flow_prio_list (list): priorities of the flow + flow_rate_percent (int): rate percentage for the flow + flow_dur_sec (int): duration of the flow in second + data_pkt_size (int): packet size of the flow in byte + prio_dscp_map (dict): Priority vs. DSCP map (key = priority). + + Returns: + N/A + """ + tx_port_config = next((x for x in port_config_list if x.id == src_port_id), None) + rx_port_config = next((x for x in port_config_list if x.id == dst_port_id), None) + tx_mac = tx_port_config.mac + if tx_port_config.gateway == rx_port_config.gateway and tx_port_config.prefix_len == rx_port_config.prefix_len: + rx_mac = rx_port_config.mac + else: + rx_mac = tx_port_config.gateway_mac + if 'Background Flow' in flow_name_prefix: + flow = testbed_config.flows.flow( + name='{} {} {} -> {} Rate:{}'.format(index, flow_name_prefix, + src_port_id, dst_port_id, flow_rate_percent))[-1] + else: + flow = testbed_config.flows.flow( + name='{} {} -> {} Rate:{}'.format(flow_name_prefix, + src_port_id, dst_port_id, flow_rate_percent))[-1] + flow.tx_rx.port.tx_name = testbed_config.ports[src_port_id].name + flow.tx_rx.port.rx_name = testbed_config.ports[dst_port_id].name + eth, ipv4, udp = flow.packet.ethernet().ipv4().udp() + src_port = random.randint(5000, 6000) + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + + eth.src.value = tx_mac + eth.dst.value = rx_mac + + if pfcQueueGroupSize == 8: + if 'Background Flow' in flow.name: + eth.pfc_queue.value = 1 + elif 'Test Flow 1 -> 0' in flow.name: + eth.pfc_queue.value = flow_prio[0] + elif 'Test Flow 2 -> 0' in flow.name: + eth.pfc_queue.value = flow_prio[1] + else: + if 'Background Flow' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[1] + elif 'Test Flow 1 -> 0' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio[0]] + elif 'Test Flow 2 -> 0' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio[1]] + + ipv4.src.value = tx_port_config.ip + ipv4.dst.value = rx_port_config.ip + ipv4.priority.choice = ipv4.priority.DSCP + + if '1 Background Flow 1 -> 0' in flow.name: + ipv4.priority.dscp.phb.values = [ + ipv4.priority.dscp.phb.CS2, + ] + elif '2 Background Flow 2 -> 0' in flow.name: + ipv4.priority.dscp.phb.values = [ + ipv4.priority.dscp.phb.DEFAULT, + ] + ipv4.priority.dscp.phb.value = 5 + elif '3 Background Flow 1 -> 0' in flow.name: + ipv4.priority.dscp.phb.values = [ + ipv4.priority.dscp.phb.CS6, + ] + elif '4 Background Flow 2 -> 0' in flow.name: + ipv4.priority.dscp.phb.values = [ + ipv4.priority.dscp.phb.CS1, + ] + elif 'Test Flow 1 -> 0' in flow.name: + ipv4.priority.dscp.phb.values = [flow_prio[0]] + elif 'Test Flow 2 -> 0' in flow.name: + ipv4.priority.dscp.phb.values = [ + ipv4.priority.dscp.phb.CS1, + ] + ipv4.priority.dscp.phb.value = flow_prio[1] + + ipv4.priority.dscp.ecn.value = ipv4.priority.dscp.ecn.CAPABLE_TRANSPORT_1 + flow.size.fixed = data_pkt_size + flow.rate.percentage = flow_rate_percent + flow.duration.fixed_seconds.seconds = flow_dur_sec + flow.metrics.enable = True + flow.metrics.loss = True + + +def verify_m2o_fluctuating_lossless_result(rows, + tx_port, + rx_port): + """ + Verifies the required loss % from the Traffic Items Statistics + + Args: + rows (list): Traffic Item Statistics from snappi config + tx_port (list): Ingress Ports + rx_port : Egress Port + Returns: + N/A + """ + background_loss = 0 + for row in rows: + if 'Test Flow' in row.name: + pytest_assert(int(row.loss) == 0, "FAIL: {} must have 0% loss".format(row.name)) + elif 'Background Flow' in row.name: + background_loss += float(row.loss) + pytest_assert(int(background_loss/4) == 10, "Each Background Flow must have an avg of 10% loss ") diff --git a/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossless_helper.py b/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossless_helper.py index dff14923fc..a6878d4286 100644 --- a/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossless_helper.py +++ b/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossless_helper.py @@ -5,6 +5,7 @@ # Compiled at: 2023-02-10 09:15:26 from math import ceil # noqa: F401 import logging # noqa: F401 +import random from tests.common.helpers.assertions import pytest_assert, pytest_require # noqa: F401 from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 from tests.common.snappi_tests.snappi_helpers import get_dut_port_id # noqa: F401 @@ -13,7 +14,8 @@ from tests.common.snappi_tests.port import select_ports # noqa: F401 from tests.common.snappi_tests.snappi_test_params import SnappiTestParams from tests.common.snappi_tests.traffic_generation import setup_base_traffic_config, \ - verify_m2o_oversubscribtion_results, run_traffic # noqa: F401 + run_traffic # noqa: F401 +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) PAUSE_FLOW_NAME = 'Pause Storm' @@ -61,19 +63,27 @@ def run_m2o_oversubscribe_lossless_test(api, if snappi_extra_params is None: snappi_extra_params = SnappiTestParams() - duthost1 = snappi_extra_params.multi_dut_params.duthost1 + # initialize the (duthost, port) set. + dut_asics_to_be_configured = set() + rx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[0] rx_port_id_list = [rx_port["port_id"]] - duthost2 = snappi_extra_params.multi_dut_params.duthost2 + egress_duthost = rx_port['duthost'] + dut_asics_to_be_configured.add((egress_duthost, rx_port['asic_value'])) + tx_port = [snappi_extra_params.multi_dut_params.multi_dut_ports[1], snappi_extra_params.multi_dut_params.multi_dut_ports[2]] tx_port_id_list = [tx_port[0]["port_id"], tx_port[1]["port_id"]] + # add ingress DUT into the set + dut_asics_to_be_configured.add((tx_port[0]['duthost'], tx_port[0]['asic_value'])) + dut_asics_to_be_configured.add((tx_port[1]['duthost'], tx_port[1]['asic_value'])) pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') - stop_pfcwd(duthost1, rx_port['asic_value']) - disable_packet_aging(duthost1) - stop_pfcwd(duthost2, tx_port[0]['asic_value']) - disable_packet_aging(duthost2) + + # Disable PFC watchdog on the rx side and tx side of the DUT + for duthost, asic in dut_asics_to_be_configured: + stop_pfcwd(duthost, asic) + disable_packet_aging(duthost) test_flow_rate_percent = int(TEST_FLOW_AGGR_RATE_PERCENT) bg_flow_rate_percent = int(BG_FLOW_AGGR_RATE_PERCENT) @@ -103,30 +113,17 @@ def run_m2o_oversubscribe_lossless_test(api, data_flow_names = [flow.name for flow in flows if PAUSE_FLOW_NAME not in flow.name] """ Run traffic """ - flow_stats, switch_flow_stats = run_traffic(duthost=duthost1, - api=api, - config=testbed_config, - data_flow_names=data_flow_names, - all_flow_names=all_flow_names, - exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC, - snappi_extra_params=snappi_extra_params) - - flag = { - 'Test Flow': { - 'loss': '0' - }, - 'Background Flow': { - 'loss': '0' - } - } - - verify_m2o_oversubscribtion_results(duthost=duthost2, - rows=flow_stats, - test_flow_name=TEST_FLOW_NAME, - bg_flow_name=BG_FLOW_NAME, - rx_port=rx_port, - rx_frame_count_deviation=TOLERANCE_THRESHOLD, - flag=flag) + flow_stats, switch_flow_stats, _ = run_traffic(duthost=egress_duthost, + api=api, + config=testbed_config, + data_flow_names=data_flow_names, + all_flow_names=all_flow_names, + exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC, + snappi_extra_params=snappi_extra_params) + + verify_m2o_oversubscribe_lossless_result(flow_stats, + tx_port, + rx_port) def __gen_traffic(testbed_config, @@ -275,15 +272,29 @@ def __gen_data_flow(testbed_config, flow = testbed_config.flows.flow(name='{} {} -> {}'.format(flow_name_prefix, src_port_id, dst_port_id))[-1] flow.tx_rx.port.tx_name = testbed_config.ports[src_port_id].name flow.tx_rx.port.rx_name = testbed_config.ports[dst_port_id].name - eth, ipv4 = flow.packet.ethernet().ipv4() + eth, ipv4, udp = flow.packet.ethernet().ipv4().udp() + src_port = random.randint(5000, 6000) + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + eth.src.value = tx_mac eth.dst.value = rx_mac flow.duration.fixed_seconds.delay.nanoseconds = int(sec_to_nanosec(DATA_FLOW_DELAY_SEC)) - if 'Background Flow' in flow.name: - flow.duration.fixed_seconds.delay.nanoseconds = 0 - eth.pfc_queue.value = 0 + if pfcQueueGroupSize == 8: + if 'Background Flow' in flow.name: + eth.pfc_queue.value = 1 + elif 'Test Flow 1 -> 0' in flow.name: + eth.pfc_queue.value = flow_prio[0] + elif 'Test Flow 2 -> 0' in flow.name: + eth.pfc_queue.value = flow_prio[1] else: - eth.pfc_queue.value = 3 + if 'Background Flow' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[1] + elif 'Test Flow 1 -> 0' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio[0]] + elif 'Test Flow 2 -> 0' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio[1]] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip @@ -310,7 +321,7 @@ def __gen_data_flow(testbed_config, ipv4.priority.dscp.phb.values = [ ipv4.priority.dscp.phb.DEFAULT, ] - ipv4.priority.dscp.phb.values = flow_prio_dscp_list + ipv4.priority.dscp.phb.values = prio_dscp_map[flow_prio[0]] elif 'Test Flow 2 -> 0' in flow.name: for fp in flow_prio: for val in prio_dscp_map[fp]: @@ -318,7 +329,7 @@ def __gen_data_flow(testbed_config, ipv4.priority.dscp.phb.values = [ ipv4.priority.dscp.phb.AF11, ] - ipv4.priority.dscp.phb.values = flow_prio_dscp_list + ipv4.priority.dscp.phb.values = prio_dscp_map[flow_prio[1]] else: pass @@ -328,3 +339,27 @@ def __gen_data_flow(testbed_config, flow.duration.fixed_seconds.seconds = flow_dur_sec flow.metrics.enable = True flow.metrics.loss = True + + +def verify_m2o_oversubscribe_lossless_result(rows, + tx_port, + rx_port): + """ + Verifies the required loss % from the Traffic Items Statistics + + Args: + rows (list): Traffic Item Statistics from snappi config + tx_port (list): Ingress Ports + rx_port : Egress Port + Returns: + N/A + """ + for row in rows: + if 'Test Flow 1 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss".format(row.name)) + elif 'Test Flow 2 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) + elif 'Background Flow 1 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) + elif 'Background Flow 2 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) diff --git a/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossless_lossy_helper.py b/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossless_lossy_helper.py index 80bd9b0242..6adda32474 100644 --- a/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossless_lossy_helper.py +++ b/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossless_lossy_helper.py @@ -5,6 +5,7 @@ # Compiled at: 2023-02-10 09:15:26 from math import ceil # noqa: F401 import logging # noqa: F401 +import random from tests.common.helpers.assertions import pytest_assert, pytest_require # noqa: F401 from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 from tests.common.snappi_tests.snappi_helpers import get_dut_port_id # noqa: F401 @@ -13,7 +14,8 @@ from tests.common.snappi_tests.port import select_ports # noqa: F401 from tests.common.snappi_tests.snappi_test_params import SnappiTestParams from tests.common.snappi_tests.traffic_generation import run_traffic, \ - setup_base_traffic_config, verify_m2o_oversubscribtion_results # noqa: F401 + setup_base_traffic_config # noqa: F401 +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) @@ -63,19 +65,30 @@ def run_pfc_m2o_oversubscribe_lossless_lossy_test(api, if snappi_extra_params is None: snappi_extra_params = SnappiTestParams() - duthost1 = snappi_extra_params.multi_dut_params.duthost1 + # Traffic flow: + # tx_port (TGEN) --- ingress DUT --- egress DUT --- rx_port (TGEN) + + # initialize the (duthost, port) set. + dut_asics_to_be_configured = set() + rx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[0] rx_port_id_list = [rx_port["port_id"]] - duthost2 = snappi_extra_params.multi_dut_params.duthost2 + egress_duthost = rx_port['duthost'] + dut_asics_to_be_configured.add((egress_duthost, rx_port['asic_value'])) + tx_port = [snappi_extra_params.multi_dut_params.multi_dut_ports[1], snappi_extra_params.multi_dut_params.multi_dut_ports[2]] tx_port_id_list = [tx_port[0]["port_id"], tx_port[1]["port_id"]] + # add ingress DUT into the set + dut_asics_to_be_configured.add((tx_port[0]['duthost'], tx_port[0]['asic_value'])) + dut_asics_to_be_configured.add((tx_port[1]['duthost'], tx_port[1]['asic_value'])) pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') - stop_pfcwd(duthost1, rx_port['asic_value']) - disable_packet_aging(duthost1) - stop_pfcwd(duthost2, tx_port[0]['asic_value']) - disable_packet_aging(duthost2) + + # Disable PFC watchdog on the rx side and tx side of the DUT + for duthost, asic in dut_asics_to_be_configured: + stop_pfcwd(duthost, asic) + disable_packet_aging(duthost) port_id = 0 # Generate base traffic config @@ -103,36 +116,17 @@ def run_pfc_m2o_oversubscribe_lossless_lossy_test(api, data_flow_names = [flow.name for flow in flows if PAUSE_FLOW_NAME not in flow.name] """ Run traffic """ - flow_stats, switch_flow_stats = run_traffic(duthost=duthost1, - api=api, - config=testbed_config, - data_flow_names=data_flow_names, - all_flow_names=all_flow_names, - exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC, - snappi_extra_params=snappi_extra_params) - - flag = { - 'Test Flow 1 -> 0 Rate:40': { - 'loss': '0' - }, - 'Test Flow 2 -> 0 Rate:20': { - 'loss': '0' - }, - 'Background Flow 1 -> 0 Rate:20': { - 'loss': '0' - }, - 'Background Flow 2 -> 0 Rate:40': { - 'loss': 'continuing' - }, - } - # Background Flow 2 -> 0 Rate:40 - verify_m2o_oversubscribtion_results(duthost=duthost2, - rows=flow_stats, - test_flow_name=TEST_FLOW_NAME, - bg_flow_name=BG_FLOW_NAME, - rx_port=rx_port, - rx_frame_count_deviation=TOLERANCE_THRESHOLD, - flag=flag) + flow_stats, switch_flow_stats, _ = run_traffic(duthost=egress_duthost, + api=api, + config=testbed_config, + data_flow_names=data_flow_names, + all_flow_names=all_flow_names, + exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC, + snappi_extra_params=snappi_extra_params) + + verify_m2o_oversubscribe_lossless_lossy_result(flow_stats, + tx_port, + rx_port) def __gen_traffic(testbed_config, @@ -282,7 +276,12 @@ def __gen_data_flow(testbed_config, dst_port_id, flow_rate_percent))[-1] flow.tx_rx.port.tx_name = testbed_config.ports[src_port_id].name flow.tx_rx.port.rx_name = testbed_config.ports[dst_port_id].name - eth, ipv4 = flow.packet.ethernet().ipv4() + eth, ipv4, udp = flow.packet.ethernet().ipv4().udp() + src_port = random.randint(5000, 6000) + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + eth.src.value = tx_mac eth.dst.value = rx_mac flow.duration.fixed_seconds.delay.nanoseconds = 0 @@ -291,10 +290,21 @@ def __gen_data_flow(testbed_config, else: flow.duration.fixed_seconds.delay.nanoseconds = 0 - if 'Background Flow' in flow.name: - eth.pfc_queue.value = 0 + # if 'Background Flow' in flow.name: + # eth.pfc_queue.value = 0 + # else: + # eth.pfc_queue.value = 3 + + if pfcQueueGroupSize == 8: + if 'Background Flow' in flow.name: + eth.pfc_queue.value = 1 + else: + eth.pfc_queue.value = flow_prio[0] else: - eth.pfc_queue.value = 3 + if 'Background Flow' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[1] + else: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio[0]] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip @@ -317,12 +327,12 @@ def __gen_data_flow(testbed_config, ipv4.priority.dscp.phb.values = [ ipv4.priority.dscp.phb.DEFAULT, ] - ipv4.priority.dscp.phb.values = [3] + ipv4.priority.dscp.phb.values = prio_dscp_map[flow_prio[0]] elif 'Test Flow 2 -> 0' in flow.name: ipv4.priority.dscp.phb.values = [ ipv4.priority.dscp.phb.AF11, ] - ipv4.priority.dscp.phb.values = [4] + ipv4.priority.dscp.phb.values = prio_dscp_map[flow_prio[1]] else: pass @@ -332,3 +342,27 @@ def __gen_data_flow(testbed_config, flow.duration.fixed_seconds.seconds = flow_dur_sec flow.metrics.enable = True flow.metrics.loss = True + + +def verify_m2o_oversubscribe_lossless_lossy_result(rows, + tx_port, + rx_port): + """ + Verifies the required loss % from the Traffic Items Statistics + + Args: + rows (list): Traffic Item Statistics from snappi config + tx_port (list): Ingress Ports + rx_port : Egress Port + Returns: + N/A + """ + for row in rows: + if 'Test Flow 1 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss".format(row.name)) + elif 'Test Flow 2 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) + elif 'Background Flow 1 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) + elif 'Background Flow 2 -> 0' in row.name: + pytest_assert(int(row.loss) >= 10, "{} must have loss >= 10%".format(row.name)) diff --git a/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossy_helper.py b/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossy_helper.py index 8e8427b25e..68b85b6d92 100644 --- a/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossy_helper.py +++ b/tests/snappi_tests/multidut/pfc/files/m2o_oversubscribe_lossy_helper.py @@ -5,6 +5,7 @@ # Compiled at: 2023-02-10 09:15:26 from math import ceil # noqa: F401 import logging # noqa: F401 +import random from tests.common.helpers.assertions import pytest_assert, pytest_require # noqa: F401 from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 from tests.common.snappi_tests.snappi_helpers import get_dut_port_id # noqa: F401 @@ -13,7 +14,8 @@ from tests.common.snappi_tests.port import select_ports # noqa: F401 from tests.common.snappi_tests.snappi_test_params import SnappiTestParams from tests.common.snappi_tests.traffic_generation import setup_base_traffic_config, \ - verify_m2o_oversubscribtion_results, run_traffic # noqa: F401 + run_traffic # noqa: F401 +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) PAUSE_FLOW_NAME = 'Pause Storm' @@ -62,19 +64,30 @@ def run_pfc_m2o_oversubscribe_lossy_test(api, if snappi_extra_params is None: snappi_extra_params = SnappiTestParams() - duthost1 = snappi_extra_params.multi_dut_params.duthost1 + # Traffic flow: + # tx_port (TGEN) --- ingress DUT --- egress DUT --- rx_port (TGEN) + + # initialize the (duthost, port) set. + dut_asics_to_be_configured = set() + rx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[0] rx_port_id_list = [rx_port["port_id"]] - duthost2 = snappi_extra_params.multi_dut_params.duthost2 + egress_duthost = rx_port['duthost'] + dut_asics_to_be_configured.add((egress_duthost, rx_port['asic_value'])) + tx_port = [snappi_extra_params.multi_dut_params.multi_dut_ports[1], snappi_extra_params.multi_dut_params.multi_dut_ports[2]] tx_port_id_list = [tx_port[0]["port_id"], tx_port[1]["port_id"]] + # add ingress DUT into the set + dut_asics_to_be_configured.add((tx_port[0]['duthost'], tx_port[0]['asic_value'])) + dut_asics_to_be_configured.add((tx_port[1]['duthost'], tx_port[1]['asic_value'])) pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') - stop_pfcwd(duthost1, rx_port['asic_value']) - disable_packet_aging(duthost1) - stop_pfcwd(duthost2, tx_port[0]['asic_value']) - disable_packet_aging(duthost2) + + # Disable PFC watchdog on the rx side and tx side of the DUT + for duthost, asic in dut_asics_to_be_configured: + stop_pfcwd(duthost, asic) + disable_packet_aging(duthost) test_flow_rate_percent = int(TEST_FLOW_AGGR_RATE_PERCENT) bg_flow_rate_percent = int(BG_FLOW_AGGR_RATE_PERCENT) @@ -103,31 +116,20 @@ def run_pfc_m2o_oversubscribe_lossy_test(api, flows = testbed_config.flows all_flow_names = [flow.name for flow in flows] data_flow_names = [flow.name for flow in flows if PAUSE_FLOW_NAME not in flow.name] - flag = { - 'Test Flow': { - 'loss': '16' - }, - 'Background Flow': { - 'loss': '0' - }, - } + """ Run traffic """ - flow_stats, switch_flow_stats = run_traffic(duthost=duthost1, - api=api, - config=testbed_config, - data_flow_names=data_flow_names, - all_flow_names=all_flow_names, - exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC, - snappi_extra_params=snappi_extra_params) + flow_stats, switch_flow_stats, _ = run_traffic(duthost=egress_duthost, + api=api, + config=testbed_config, + data_flow_names=data_flow_names, + all_flow_names=all_flow_names, + exp_dur_sec=DATA_FLOW_DURATION_SEC + DATA_FLOW_DELAY_SEC, + snappi_extra_params=snappi_extra_params) """ Verify Results """ - verify_m2o_oversubscribtion_results(duthost=duthost2, - rows=flow_stats, - test_flow_name=TEST_FLOW_NAME, - bg_flow_name=BG_FLOW_NAME, - rx_port=rx_port, - rx_frame_count_deviation=TOLERANCE_THRESHOLD, - flag=flag) + verify_m2o_oversubscribe_lossy_result(flow_stats, + tx_port, + rx_port) def __data_flow_name(name_prefix, src_id, dst_id, prio): @@ -293,31 +295,44 @@ def __gen_data_flow(testbed_config, flow = testbed_config.flows.flow(name='{} {} -> {}'.format(flow_name_prefix, src_port_id, dst_port_id))[-1] flow.tx_rx.port.tx_name = testbed_config.ports[src_port_id].name flow.tx_rx.port.rx_name = testbed_config.ports[dst_port_id].name - eth, ipv4 = flow.packet.ethernet().ipv4() + eth, ipv4, udp = flow.packet.ethernet().ipv4().udp() + src_port = random.randint(5000, 6000) + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + eth.src.value = tx_mac eth.dst.value = rx_mac - if 'Background Flow' in flow.name: - eth.pfc_queue.value = 3 - elif 'Test Flow 2 -> 0' in flow.name: - eth.pfc_queue.value = 1 + if pfcQueueGroupSize == 8: + if 'Test Flow' in flow.name: + eth.pfc_queue.value = 1 + elif 'Background Flow 1 -> 0' in flow.name: + eth.pfc_queue.value = flow_prio[0] + elif 'Background Flow 2 -> 0' in flow.name: + eth.pfc_queue.value = flow_prio[1] else: - eth.pfc_queue.value = 0 + if 'Test Flow' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[1] + elif 'Background Flow 1 -> 0' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio[0]] + elif 'Background Flow 2 -> 0' in flow.name: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio[1]] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip ipv4.priority.choice = ipv4.priority.DSCP - - flow_prio_dscp_list = [] flow.duration.fixed_seconds.delay.nanoseconds = 0 - if 'Background Flow' in flow_name_prefix: - for fp in flow_prio: - for val in prio_dscp_map[fp]: - flow_prio_dscp_list.append(val) + if 'Background Flow 1 -> 0' in flow.name: ipv4.priority.dscp.phb.values = [ ipv4.priority.dscp.phb.AF11, ] - ipv4.priority.dscp.phb.values = flow_prio_dscp_list + ipv4.priority.dscp.phb.values = prio_dscp_map[flow_prio[0]] + elif 'Background Flow 2 -> 0' in flow.name: + ipv4.priority.dscp.phb.values = [ + ipv4.priority.dscp.phb.AF11, + ] + ipv4.priority.dscp.phb.values = prio_dscp_map[flow_prio[1]] elif 'Test Flow 1 -> 0' in flow.name: ipv4.priority.dscp.phb.values = [ ipv4.priority.dscp.phb.CS1, @@ -342,3 +357,27 @@ def __gen_data_flow(testbed_config, flow.duration.fixed_seconds.seconds = flow_dur_sec flow.metrics.enable = True flow.metrics.loss = True + + +def verify_m2o_oversubscribe_lossy_result(rows, + tx_port, + rx_port): + """ + Verifies the required loss % from the Traffic Items Statistics + + Args: + rows (list): Traffic Item Statistics from snappi config + tx_port (list): Ingress Ports + rx_port : Egress Port + Returns: + N/A + """ + for row in rows: + if 'Test Flow 1 -> 0' in row.name: + pytest_assert(int(row.loss) == 16, "{} must have 16% loss".format(row.name)) + elif 'Test Flow 2 -> 0' in row.name: + pytest_assert(int(row.loss) == 16, "{} must have 16% loss ".format(row.name)) + elif 'Background Flow 1 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) + elif 'Background Flow 2 -> 0' in row.name: + pytest_assert(int(row.loss) == 0, "{} must have 0% loss ".format(row.name)) diff --git a/tests/snappi_tests/multidut/pfc/files/multidut_helper.py b/tests/snappi_tests/multidut/pfc/files/multidut_helper.py index 23a59c5ebd..56507c4dad 100644 --- a/tests/snappi_tests/multidut/pfc/files/multidut_helper.py +++ b/tests/snappi_tests/multidut/pfc/files/multidut_helper.py @@ -75,17 +75,25 @@ def run_pfc_test(api, if snappi_extra_params is None: snappi_extra_params = SnappiTestParams() - duthost1 = snappi_extra_params.multi_dut_params.duthost1 + # Traffic flow: + # tx_port (TGEN) --- ingress DUT --- egress DUT --- rx_port (TGEN) + + # initialize the (duthost, port) set. + dut_asics_to_be_configured = set() + rx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[0] - duthost2 = snappi_extra_params.multi_dut_params.duthost2 + egress_duthost = rx_port['duthost'] + dut_asics_to_be_configured.add((egress_duthost, rx_port['asic_value'])) + tx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[1] + ingress_duthost = tx_port['duthost'] + dut_asics_to_be_configured.add((ingress_duthost, tx_port['asic_value'])) pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') - stop_pfcwd(duthost1, rx_port['asic_value']) - disable_packet_aging(duthost1) - stop_pfcwd(duthost2, tx_port['asic_value']) - disable_packet_aging(duthost2) + for duthost, asic in dut_asics_to_be_configured: + stop_pfcwd(duthost, asic) + disable_packet_aging(duthost) global DATA_FLOW_DURATION_SEC global data_flow_delay_sec @@ -103,7 +111,7 @@ def run_pfc_test(api, port_id=port_id) speed_str = testbed_config.layer1[0].speed - speed_gbps = int(speed_str.split('_')[1]) + speed_gbps = int(float(speed_str.split('_')[1])) if snappi_extra_params.headroom_test_params is not None: DATA_FLOW_DURATION_SEC += 10 @@ -216,10 +224,10 @@ def run_pfc_test(api, data_flow_names = [flow.name for flow in flows if PAUSE_FLOW_NAME not in flow.name] # Clear PFC, queue and interface counters before traffic run - duthost = duthost1 + duthost = egress_duthost duthost.command("pfcstat -c") time.sleep(1) - duthost1.command("sonic-clear queuecounters") + duthost.command("sonic-clear queuecounters") time.sleep(1) duthost.command("sonic-clear counters") time.sleep(1) @@ -263,22 +271,26 @@ def run_pfc_test(api, snappi_extra_params=snappi_extra_params) # Verify PFC pause frame count on the DUT - verify_pause_frame_count_dut(duthost=duthost, + verify_pause_frame_count_dut(rx_dut=ingress_duthost, + tx_dut=egress_duthost, test_traffic_pause=test_traffic_pause, + global_pause=global_pause, snappi_extra_params=snappi_extra_params) # Verify in flight TX lossless packets do not leave the DUT when traffic is expected # to be paused, or leave the DUT when the traffic is not expected to be paused - verify_egress_queue_frame_count(duthost=duthost, + # Verifying the packets on DUT egress, especially for multi line card scenario + verify_egress_queue_frame_count(duthost=egress_duthost, switch_flow_stats=switch_flow_stats, test_traffic_pause=test_traffic_pause, snappi_extra_params=snappi_extra_params) if test_traffic_pause: # Verify in flight TX packets count relative to switch buffer size - verify_in_flight_buffer_pkts(duthost=duthost, + verify_in_flight_buffer_pkts(duthost=egress_duthost, flow_metrics=in_flight_flow_metrics, - snappi_extra_params=snappi_extra_params) + snappi_extra_params=snappi_extra_params, + asic_value=rx_port['asic_value']) else: # Verify zero pause frames are counted when the PFC class enable vector is not set verify_unset_cev_pause_frame_count(duthost=duthost, diff --git a/tests/snappi_tests/multidut/pfc/test_lossless_response_to_external_pause_storms.py b/tests/snappi_tests/multidut/pfc/test_lossless_response_to_external_pause_storms.py index 61f55511c3..2cb631b911 100644 --- a/tests/snappi_tests/multidut/pfc/test_lossless_response_to_external_pause_storms.py +++ b/tests/snappi_tests/multidut/pfc/test_lossless_response_to_external_pause_storms.py @@ -1,15 +1,13 @@ import pytest -import random import logging +from tests.common.helpers.assertions import pytest_assert from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ - fanout_graph_facts # noqa: F401 + fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, \ lossless_prio_list # noqa: F401 -from tests.snappi_tests.variables import config_set, line_card_choice -from tests.common.config_reload import config_reload +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.multidut.pfc.files.lossless_response_to_external_pause_storms_helper import ( run_lossless_response_to_external_pause_storms_test, ) @@ -18,18 +16,17 @@ pytestmark = [pytest.mark.topology('multidut-tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_lossless_response_to_external_pause_storms_test(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 - line_card_choice, + fanout_graph_facts_multidut, # noqa: F811 duthosts, prio_dscp_map, # noqa: F811 lossless_prio_list, # noqa: F811 - linecard_configuration_set, - get_multidut_snappi_ports,): # noqa: F811 - + tbinfo, # noqa: F811 + get_snappi_ports, # noqa: F811 + multidut_port_info, + ): # noqa: F811 """ Run PFC lossless response to external pause storm with many to one traffic pattern @@ -37,12 +34,12 @@ def test_lossless_response_to_external_pause_storms_test(snappi_api, snappi_api (pytest fixture): SNAPPI session snappi_testbed_config (pytest fixture): testbed configuration information conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs lossy_prio_list (pytest fixture): list of lossy priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Brief Description: This test uses the lossless_response_to_external_pause_storms_helper.py file and generates 2 Background @@ -58,50 +55,47 @@ def test_lossless_response_to_external_pause_storms_test(snappi_api, Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - assert False, "Invalid line_card_choice value passed in parameter" - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - assert False, "Hostname can't be an empty list" - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 3: - assert False, "Need Minimum of 3 ports for the test" - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 3) + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 2 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 3 ports defined in ansible/files/*links.csv file") - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) all_prio_list = prio_dscp_map.keys() test_prio_list = lossless_prio_list pause_prio_list = test_prio_list bg_prio_list = [x for x in all_prio_list if x not in pause_prio_list] snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_lossless_response_to_external_pause_storms_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, bg_prio_list=bg_prio_list, prio_dscp_map=prio_dscp_map, snappi_extra_params=snappi_extra_params) - - # Teardown config through a reload - logger.info("Reloading config to teardown") - config_reload(sonic_host=duthost1, config_source='config_db', safe_reload=True) - config_reload(sonic_host=duthost2, config_source='config_db', safe_reload=True) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfc/test_lossless_response_to_throttling_pause_storms.py b/tests/snappi_tests/multidut/pfc/test_lossless_response_to_throttling_pause_storms.py index af9caafc7a..1fc579899c 100644 --- a/tests/snappi_tests/multidut/pfc/test_lossless_response_to_throttling_pause_storms.py +++ b/tests/snappi_tests/multidut/pfc/test_lossless_response_to_throttling_pause_storms.py @@ -1,34 +1,31 @@ import pytest -import random import logging from tests.common.helpers.assertions import pytest_assert from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ - fanout_graph_facts # noqa: F401 + fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, \ lossless_prio_list # noqa: F401 -from tests.snappi_tests.variables import config_set, line_card_choice -from tests.common.config_reload import config_reload +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.multidut.pfc.files.lossless_response_to_throttling_pause_storms_helper import ( run_lossless_response_to_throttling_pause_storms_test) from tests.common.snappi_tests.snappi_test_params import SnappiTestParams +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict # noqa: F401 logger = logging.getLogger(__name__) pytestmark = [pytest.mark.topology('multidut-tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_lossless_response_to_throttling_pause_storms(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 - line_card_choice, + fanout_graph_facts_multidut, # noqa: F811 duthosts, prio_dscp_map, # noqa: F811 lossless_prio_list, # noqa: F811 - linecard_configuration_set, # noqa: F811 - get_multidut_snappi_ports,): # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info): # noqa: F811 """ Run PFC lossless response to throttling pause storms @@ -37,12 +34,12 @@ def test_lossless_response_to_throttling_pause_storms(snappi_api, snappi_api (pytest fixture): SNAPPI session snappi_testbed_config (pytest fixture): testbed configuration information conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs lossy_prio_list (pytest fixture): list of lossy priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Brief Description: This test uses the lossless_response_to_throttling_pause_storms_helper.py file and generates @@ -60,26 +57,30 @@ def test_lossless_response_to_throttling_pause_storms(snappi_api, Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_assert(False, "Invalid line_card_choice value passed in parameter") - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_assert(False, "Hostname can't be an empty list") - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 3: - pytest_assert(False, "Need Minimum of 3 ports for the test") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 3) + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 2 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 3 ports defined in ansible/files/*links.csv file") - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) all_prio_list = prio_dscp_map.keys() test_prio_list = lossless_prio_list @@ -87,23 +88,17 @@ def test_lossless_response_to_throttling_pause_storms(snappi_api, bg_prio_list = [x for x in all_prio_list if x not in pause_prio_list] snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_lossless_response_to_throttling_pause_storms_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, bg_prio_list=bg_prio_list, prio_dscp_map=prio_dscp_map, snappi_extra_params=snappi_extra_params) - - # Teardown config through a reload - logger.info("Reloading config to teardown") - config_reload(sonic_host=duthost1, config_source='config_db', safe_reload=True) - config_reload(sonic_host=duthost2, config_source='config_db', safe_reload=True) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfc/test_m2o_fluctuating_lossless.py b/tests/snappi_tests/multidut/pfc/test_m2o_fluctuating_lossless.py new file mode 100644 index 0000000000..c135ea3da3 --- /dev/null +++ b/tests/snappi_tests/multidut/pfc/test_m2o_fluctuating_lossless.py @@ -0,0 +1,101 @@ +import pytest +import logging +from tests.common.helpers.assertions import pytest_assert +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 +from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, \ + lossless_prio_list # noqa: F401 +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED +from tests.snappi_tests.multidut.pfc.files.m2o_fluctuating_lossless_helper import run_m2o_fluctuating_lossless_test +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams +logger = logging.getLogger(__name__) +pytestmark = [pytest.mark.topology('multidut-tgen')] + + +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) +def test_m2o_fluctuating_lossless(snappi_api, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + prio_dscp_map, # noqa: F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info): # noqa: F811 + + """ + Run PFC Fluctuating Lossless Traffic Congestion with many to one traffic pattern + + Args: + snappi_api (pytest fixture): SNAPPI session + snappi_testbed_config (pytest fixture): testbed configuration information + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + lossy_prio_list (pytest fixture): list of lossy priorities + prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list + + Brief Description: + This test uses the m2o_fluctuating_lossless_helper.py file and generates 4 Background traffic and + 2 Test flow traffic. The background traffic will include four lossy traffic streams, with any priorities + 0..2 and 5..6, each having 20% bandwidth for a total of 80% of the port line rate. The test data traffic + will include two lossless traffic flows, with the SONiC default lossless priorities of 3 and 4. Each of + lossless traffic flows will be shaped to have line rate of 20% and 10%, so that there are periods where + both lossless flows contribute a bandwidth of 30% (which should cause over-subscription on the egress port). + The __gen_traffic() generates the flows. run_traffic() starts the flows and returns the flows stats. + The verify_m2o_oversubscribtion_results() takes in the flows stats and verifies the loss criteria + mentioned in the flag. Ex: 'loss': '10' means the flows tohave 10% loss, 'loss': '0' means there shouldn't + be any loss + + Returns: + N/A + """ + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 2 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 3 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + all_prio_list = prio_dscp_map.keys() + test_prio_list = lossless_prio_list + pause_prio_list = test_prio_list + bg_prio_list = [x for x in all_prio_list if x not in pause_prio_list] + + snappi_extra_params = SnappiTestParams() + snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports + + run_m2o_fluctuating_lossless_test(api=snappi_api, + testbed_config=testbed_config, + port_config_list=port_config_list, + conn_data=conn_graph_facts, + fanout_data=fanout_graph_facts_multidut, + dut_port=snappi_ports[0]['peer_port'], + pause_prio_list=pause_prio_list, + test_prio_list=test_prio_list, + bg_prio_list=bg_prio_list, + prio_dscp_map=prio_dscp_map, + snappi_extra_params=snappi_extra_params) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossless.py b/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossless.py index 4a5a3fcc89..ffcefc461c 100644 --- a/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossless.py +++ b/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossless.py @@ -1,16 +1,13 @@ import pytest -import random import logging from tests.common.helpers.assertions import pytest_assert from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ - fanout_graph_facts # noqa: F401 + fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, \ lossless_prio_list # noqa: F401 -from tests.snappi_tests.variables import config_set, line_card_choice -from tests.common.config_reload import config_reload +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.multidut.pfc.files.m2o_oversubscribe_lossless_helper import ( run_m2o_oversubscribe_lossless_test ) @@ -19,17 +16,16 @@ pytestmark = [pytest.mark.topology('multidut-tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_m2o_oversubscribe_lossless(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 - line_card_choice, + fanout_graph_facts_multidut, # noqa: F811 duthosts, prio_dscp_map, # noqa: F811 lossless_prio_list, # noqa: F811 - linecard_configuration_set, - get_multidut_snappi_ports,): # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, + multidut_port_info): # noqa: F811 """ Run PFC oversubsription lossless for many to one traffic pattern @@ -42,8 +38,8 @@ def test_m2o_oversubscribe_lossless(snappi_api, # n duthosts (pytest fixture): list of DUTs lossy_prio_list (pytest fixture): list of lossy priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Brief Description: This test uses the m2o_oversubscribe_lossless_helper.py file and generates 2 Background traffic and @@ -58,26 +54,30 @@ def test_m2o_oversubscribe_lossless(snappi_api, # n Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_assert(False, "Invalid line_card_choice value passed in parameter") - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_assert(False, "Hostname can't be an empty list") - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 3: - pytest_assert(False, "Need Minimum of 3 ports for the test") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 3) + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 2 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 3 ports defined in ansible/files/*links.csv file") - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) all_prio_list = prio_dscp_map.keys() test_prio_list = lossless_prio_list @@ -85,23 +85,17 @@ def test_m2o_oversubscribe_lossless(snappi_api, # n bg_prio_list = [x for x in all_prio_list if x not in pause_prio_list] snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_m2o_oversubscribe_lossless_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, bg_prio_list=bg_prio_list, prio_dscp_map=prio_dscp_map, snappi_extra_params=snappi_extra_params) - - # Teardown config through a reload - logger.info("Reloading config to teardown") - config_reload(sonic_host=duthost1, config_source='config_db', safe_reload=True) - config_reload(sonic_host=duthost2, config_source='config_db', safe_reload=True) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossless_lossy.py b/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossless_lossy.py index 8c65fafa25..f681c5acda 100644 --- a/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossless_lossy.py +++ b/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossless_lossy.py @@ -1,32 +1,32 @@ import pytest -import random +import logging +from tests.common.helpers.assertions import pytest_assert from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ - fanout_graph_facts # noqa: F401 + fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, \ lossless_prio_list # noqa: F401 -from tests.snappi_tests.variables import config_set, line_card_choice +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.multidut.pfc.files.m2o_oversubscribe_lossless_lossy_helper import ( run_pfc_m2o_oversubscribe_lossless_lossy_test - ) -from tests.common.snappi_tests.snappi_test_params import SnappiTestParams - + ) # noqa: F401 +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams # noqa: F401 +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict # noqa: F401 +logger = logging.getLogger(__name__) pytestmark = [pytest.mark.topology('multidut-tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_m2o_oversubscribe_lossless_lossy(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 - line_card_choice, + fanout_graph_facts_multidut, # noqa: F811 duthosts, prio_dscp_map, # noqa: F811 lossless_prio_list, # noqa: F811 - linecard_configuration_set, - get_multidut_snappi_ports,): # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, + multidut_port_info): # noqa: F811 """ Run PFC Oversubscribe Lossless Lossy for many to one traffic pattern @@ -35,12 +35,12 @@ def test_m2o_oversubscribe_lossless_lossy(snappi_api, # noqa: snappi_api (pytest fixture): SNAPPI session snappi_testbed_config (pytest fixture): testbed configuration information conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs lossy_prio_list (pytest fixture): list of lossy priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Brief Description: This test uses the m2o_oversubscribe_lossless_lossy_helper.py file and generates 2 Background traffic and @@ -56,26 +56,30 @@ def test_m2o_oversubscribe_lossless_lossy(snappi_api, # noqa: Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - assert False, "Invalid line_card_choice value passed in parameter" - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - assert False, "Hostname can't be an empty list" - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 3: - assert False, "Need Minimum of 3 ports for the test" - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 3) + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 2 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 3 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) all_prio_list = prio_dscp_map.keys() test_prio_list = lossless_prio_list @@ -83,20 +87,17 @@ def test_m2o_oversubscribe_lossless_lossy(snappi_api, # noqa: bg_prio_list = [x for x in all_prio_list if x not in pause_prio_list] snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfc_m2o_oversubscribe_lossless_lossy_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, bg_prio_list=bg_prio_list, prio_dscp_map=prio_dscp_map, snappi_extra_params=snappi_extra_params) - - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossy.py b/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossy.py index fcea54a93e..7b7e4bfe41 100644 --- a/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossy.py +++ b/tests/snappi_tests/multidut/pfc/test_m2o_oversubscribe_lossy.py @@ -1,45 +1,41 @@ import pytest -import random import logging from tests.common.helpers.assertions import pytest_assert from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ - fanout_graph_facts # noqa: F401 + fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, \ lossless_prio_list # noqa: F401 -from tests.snappi_tests.variables import config_set, line_card_choice -from tests.common.config_reload import config_reload +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.multidut.pfc.files.m2o_oversubscribe_lossy_helper import run_pfc_m2o_oversubscribe_lossy_test from tests.common.snappi_tests.snappi_test_params import SnappiTestParams logger = logging.getLogger(__name__) pytestmark = [pytest.mark.topology('multidut-tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_m2o_oversubscribe_lossy(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 - line_card_choice, + fanout_graph_facts_multidut, # noqa: F811 duthosts, prio_dscp_map, # noqa: F811 lossless_prio_list, # noqa: F811 - linecard_configuration_set, - get_multidut_snappi_ports,): # noqa: F811o + get_snappi_ports, # noqa: F811o + tbinfo, + multidut_port_info): # noqa: F811 """ Run PFC oversubscription lossy test under many to one traffic pattern Args: snappi_api (pytest fixture): SNAPPI session snappi_testbed_config (pytest fixture): testbed configuration information conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs lossy_prio_list (pytest fixture): list of lossy priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Brief Description: This test uses the m2o_oversubscribe_lossy_helper.py file and generates 2 Background traffic and @@ -55,41 +51,44 @@ def test_m2o_oversubscribe_lossy(snappi_api, # Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_assert(False, "Invalid line_card_choice value passed in parameter") - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_assert(False, "Hostname can't be an empty list") - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 3: - pytest_assert(False, "Need Minimum of 3 ports for the test") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 3) + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 2 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 3 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) all_prio_list = prio_dscp_map.keys() bg_prio_list = lossless_prio_list pause_prio_list = bg_prio_list test_prio_list = [x for x in all_prio_list if x not in pause_prio_list] snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfc_m2o_oversubscribe_lossy_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, @@ -97,7 +96,4 @@ def test_m2o_oversubscribe_lossy(snappi_api, # prio_dscp_map=prio_dscp_map, snappi_extra_params=snappi_extra_params) - # Teardown config through a reload - logger.info("Reloading config to teardown") - config_reload(sonic_host=duthost1, config_source='config_db', safe_reload=True) - config_reload(sonic_host=duthost2, config_source='config_db', safe_reload=True) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfc/test_multidut_global_pause_with_snappi.py b/tests/snappi_tests/multidut/pfc/test_multidut_global_pause_with_snappi.py index 2f11a5fc6a..58a4421ee1 100644 --- a/tests/snappi_tests/multidut/pfc/test_multidut_global_pause_with_snappi.py +++ b/tests/snappi_tests/multidut/pfc/test_multidut_global_pause_with_snappi.py @@ -1,84 +1,89 @@ import pytest -import random import logging from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 -from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ - fanout_graph_facts # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, get_snappi_ports, is_snappi_multidut, \ + get_snappi_ports_single_dut, snappi_testbed_config, \ + get_snappi_ports_multi_dut, snappi_dut_base_config, cleanup_config, get_snappi_ports_for_rdma # noqa: F401 from tests.common.snappi_tests.qos_fixtures import lossless_prio_list, prio_dscp_map # noqa: F401 -from tests.snappi_tests.variables import config_set, line_card_choice from tests.snappi_tests.multidut.pfc.files.multidut_helper import run_pfc_test # noqa: F401 from tests.common.snappi_tests.snappi_test_params import SnappiTestParams +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED logger = logging.getLogger(__name__) -pytestmark = [pytest.mark.topology('multidut-tgen')] +pytestmark = [pytest.mark.topology('multidut-tgen', 'tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_global_pause(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + get_snappi_ports, # noqa: F811 duthosts, prio_dscp_map, # noqa: F811 lossless_prio_list, # noqa: F811 - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports # noqa: F811 - ): + tbinfo, # noqa: F811 + multidut_port_info): """ Test if IEEE 802.3X pause (a.k.a., global pause) will impact any priority Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph for multiple duts + get_snappi_ports (pytest fixture): list of snappi port and duthost information duthosts (pytest fixture): list of DUTs lossless_prio_list (pytest fixture): list of all the lossless priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ + snappi_port_list = get_snappi_ports - pytest_assert(line_card_choice in linecard_configuration_set.keys(), "Invalid line_card_choice in parameter") + tbname = tbinfo.get('conf-name', None) - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] # noqa: E501 - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_assert(False, "Hostname can't be an empty list") + tx_port_count = 1 + rx_port_count = 1 - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - pytest_assert(len(snappi_port_list) >= 2, "Need Minimum of 2 ports for the test") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) + for testbed_subtype, rdma_ports in multidut_port_info.items(): + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(tbname, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + if is_snappi_multidut(duthosts): + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + else: + snappi_ports = snappi_port_list + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) all_prio_list = prio_dscp_map.keys() test_prio_list = lossless_prio_list bg_prio_list = [x for x in all_prio_list if x not in test_prio_list] snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports - run_pfc_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, global_pause=True, pause_prio_list=None, test_prio_list=test_prio_list, @@ -87,4 +92,4 @@ def test_global_pause(snappi_api, # noqa: F811 test_traffic_pause=False, snappi_extra_params=snappi_extra_params) - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossless_with_snappi.py b/tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossless_with_snappi.py index 2150eb0e58..c283e0c672 100644 --- a/tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossless_with_snappi.py +++ b/tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossless_with_snappi.py @@ -1,91 +1,101 @@ import pytest -import random from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 -from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 -from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, \ - lossless_prio_list -from tests.snappi_tests.variables import config_set, line_card_choice + snappi_api, snappi_dut_base_config, get_snappi_ports_for_rdma, cleanup_config, get_snappi_ports_multi_dut, \ + snappi_testbed_config, get_snappi_ports_single_dut, \ + get_snappi_ports, is_snappi_multidut # noqa: F401 +from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, all_prio_list, lossless_prio_list,\ + lossy_prio_list # noqa F401 +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.multidut.pfc.files.multidut_helper import run_pfc_test from tests.common.reboot import reboot from tests.common.utilities import wait_until import logging from tests.common.snappi_tests.snappi_test_params import SnappiTestParams +from tests.snappi_tests.files.helper import skip_warm_reboot logger = logging.getLogger(__name__) -pytestmark = [pytest.mark.topology('multidut-tgen')] +pytestmark = [pytest.mark.topology('multidut-tgen', 'tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfc_pause_single_lossless_prio(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports): # noqa: F811 + enum_dut_lossless_prio, + prio_dscp_map, # noqa: F811 + lossless_prio_list, # noqa: F811 + all_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info): """ - Test if PFC can pause a single lossless priority + Test if PFC can pause a single lossless priority in multidut setup Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs enum_dut_lossless_prio (str): lossless priority to test, e.g., 's6100-1|3' all_prio_list (pytest fixture): list of all the priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + lossless_prio_list (pytest fixture): list of all the lossless priorities + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) >= 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts - if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1 = duthost2 = dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - pytest_require(False, "Need Minimum of 2 ports for the test") - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - tgen_ports = [port['location'] for port in snappi_ports] - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - tgen_ports, - snappi_ports, - snappi_api) - - all_prio_list = prio_dscp_map.keys() - test_prio_list = lossless_prio_list - pause_prio_list = test_prio_list - bg_prio_list = [x for x in all_prio_list if x not in pause_prio_list] + snappi_port_list = get_snappi_ports + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + if is_snappi_multidut(duthosts): + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + else: + snappi_ports = snappi_port_list + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + _, lossless_prio = enum_dut_lossless_prio.split('|') + lossless_prio = int(lossless_prio) + pause_prio_list = [lossless_prio] + test_prio_list = [lossless_prio] + bg_prio_list = [p for p in all_prio_list] + bg_prio_list.remove(lossless_prio) + logger.info("Snappi Ports : {}".format(snappi_ports)) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfc_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, global_pause=False, pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, @@ -93,76 +103,75 @@ def test_pfc_pause_single_lossless_prio(snappi_api, # noqa: prio_dscp_map=prio_dscp_map, test_traffic_pause=True, snappi_extra_params=snappi_extra_params) + cleanup_config(duthosts, snappi_ports) - cleanup_config(dut_list, snappi_ports) - -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfc_pause_multi_lossless_prio(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports): # noqa: F811 + prio_dscp_map, # noqa: F811 + lossy_prio_list, # noqa: F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info): # noqa: F811 """ - Test if PFC can pause multiple lossless priorities + Test if PFC can pause multiple lossless priorities in multidut setup Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + lossless_prio_list (pytest fixture): list of all the lossless priorities + lossy_prio_list (pytest fixture): list of all the lossy priorities + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - assert False, "Invalid line_card_choice value passed in parameter" - - if (len(linecard_configuration_set[line_card_choice]['hostname']) >= 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts - if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1 = duthost2 = dut_list[0] - else: - assert False, "Hostname can't be an empty list" - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - assert False, "Need Minimum of 2 ports for the test" - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - tgen_ports = [port['location'] for port in snappi_ports] - - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - tgen_ports, - snappi_ports, - snappi_api) - - all_prio_list = prio_dscp_map.keys() + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + pause_prio_list = lossless_prio_list test_prio_list = lossless_prio_list - pause_prio_list = test_prio_list - bg_prio_list = [x for x in all_prio_list if x not in pause_prio_list] + bg_prio_list = lossy_prio_list logger.info("Snappi Ports : {}".format(snappi_ports)) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfc_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, global_pause=False, pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, @@ -170,86 +179,93 @@ def test_pfc_pause_multi_lossless_prio(snappi_api, # noqa: F811 prio_dscp_map=prio_dscp_map, test_traffic_pause=True, snappi_extra_params=snappi_extra_params) - - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) @pytest.mark.disable_loganalyzer @pytest.mark.parametrize('reboot_type', ['warm', 'cold', 'fast']) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfc_pause_single_lossless_prio_reboot(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, localhost, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa: F811 - reboot_type): + enum_dut_lossless_prio, # noqa: F811 + prio_dscp_map, # noqa: F811 + lossless_prio_list, # noqa: F811 + all_prio_list, # noqa: F811 + reboot_type, + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info): """ - Test if PFC can pause a single lossless priority even after various types of reboot + Test if PFC can pause a single lossless priority even after various types of reboot in multidut setup Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs localhost (pytest fixture): localhost handle + all_prio_list (pytest fixture): list of all the priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). + lossless_prio_list (pytest fixture): list of all the lossless priorities reboot_type (str): reboot type to be issued on the DUT - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - assert False, "Invalid line_card_choice value passed in parameter" - - if (len(linecard_configuration_set[line_card_choice]['hostname']) >= 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts - if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1 = duthost2 = dut_list[0] - else: - assert False, "Hostname can't be an empty list" - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - assert False, "Need Minimum of 2 ports for the test" - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - tgen_ports = [port['location'] for port in snappi_ports] - - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - tgen_ports, - snappi_ports, - snappi_api) - - all_prio_list = prio_dscp_map.keys() - test_prio_list = lossless_prio_list - pause_prio_list = test_prio_list - bg_prio_list = [x for x in all_prio_list if x not in pause_prio_list] + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + skip_warm_reboot(snappi_ports[0]['duthost'], reboot_type) + skip_warm_reboot(snappi_ports[1]['duthost'], reboot_type) + + _, lossless_prio = enum_dut_lossless_prio.split('|') + lossless_prio = int(lossless_prio) + pause_prio_list = [lossless_prio] + test_prio_list = [lossless_prio] + bg_prio_list = [p for p in all_prio_list] + bg_prio_list.remove(lossless_prio) logger.info("Snappi Ports : {}".format(snappi_ports)) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports - logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost1.hostname)) - reboot(duthost1, localhost, reboot_type=reboot_type) - logger.info("Wait until the system is stable") - wait_until(180, 20, 0, duthost1.critical_services_fully_started) + for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]: + logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) + reboot(duthost, localhost, reboot_type=reboot_type) + logger.info("Wait until the system is stable") + wait_until(180, 20, 0, duthost.critical_services_fully_started) run_pfc_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, global_pause=False, pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, @@ -258,87 +274,87 @@ def test_pfc_pause_single_lossless_prio_reboot(snappi_api, # no test_traffic_pause=True, snappi_extra_params=snappi_extra_params) - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) @pytest.mark.disable_loganalyzer @pytest.mark.parametrize('reboot_type', ['warm', 'cold', 'fast']) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfc_pause_multi_lossless_prio_reboot(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, localhost, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa: F811 - reboot_type): + prio_dscp_map, # noqa: F811 + lossy_prio_list, # noqa: F811 + lossless_prio_list, # noqa: F811 + reboot_type, + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info): """ - Test if PFC can pause multiple lossless priorities even after various types of reboot + Test if PFC can pause multiple lossless priorities even after various types of reboot in multidut setup Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs localhost (pytest fixture): localhost handle prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). + lossless_prio_list (pytest fixture): list of all the lossless priorities + lossy_prio_list (pytest fixture): list of all the lossy priorities reboot_type (str): reboot type to be issued on the DUT - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) - + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - - if line_card_choice not in linecard_configuration_set.keys(): - assert False, "Invalid line_card_choice value passed in parameter" - - if (len(linecard_configuration_set[line_card_choice]['hostname']) >= 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts - if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1 = duthost2 = dut_list[0] - else: - assert False, "Hostname can't be an empty list" - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - assert False, "Need Minimum of 2 ports for the test" - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - tgen_ports = [port['location'] for port in snappi_ports] - - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - tgen_ports, - snappi_ports, - snappi_api) - - all_prio_list = prio_dscp_map.keys() + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + skip_warm_reboot(snappi_ports[0]['duthost'], reboot_type) + skip_warm_reboot(snappi_ports[1]['duthost'], reboot_type) + pause_prio_list = lossless_prio_list test_prio_list = lossless_prio_list - pause_prio_list = test_prio_list - bg_prio_list = [x for x in all_prio_list if x not in pause_prio_list] + bg_prio_list = lossy_prio_list logger.info("Snappi Ports : {}".format(snappi_ports)) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports - logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost1.hostname)) - reboot(duthost1, localhost, reboot_type=reboot_type) - logger.info("Wait until the system is stable") - wait_until(180, 20, 0, duthost1.critical_services_fully_started) + for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]: + logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) + reboot(duthost, localhost, reboot_type=reboot_type) + logger.info("Wait until the system is stable") + wait_until(180, 20, 0, duthost.critical_services_fully_started) run_pfc_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, global_pause=False, pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, @@ -347,4 +363,4 @@ def test_pfc_pause_multi_lossless_prio_reboot(snappi_api, # noq test_traffic_pause=True, snappi_extra_params=snappi_extra_params) - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossy_with_snappi.py b/tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossy_with_snappi.py index 25eba58eb7..a5297a3988 100644 --- a/tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossy_with_snappi.py +++ b/tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossy_with_snappi.py @@ -1,90 +1,94 @@ import pytest from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ - fanout_graph_facts # noqa: F401 + fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 -from tests.common.snappi_tests.qos_fixtures import lossy_prio_list, prio_dscp_map, \ - lossless_prio_list # noqa: F401 -from tests.snappi_tests.variables import config_set, line_card_choice + snappi_api, snappi_dut_base_config, get_snappi_ports_for_rdma, cleanup_config, \ + get_snappi_ports # noqa: F401 +from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, all_prio_list, lossless_prio_list,\ + lossy_prio_list # noqa F401 +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.multidut.pfc.files.multidut_helper import run_pfc_test from tests.common.reboot import reboot from tests.common.utilities import wait_until import logging -import random from tests.common.snappi_tests.snappi_test_params import SnappiTestParams +from tests.snappi_tests.files.helper import skip_warm_reboot logger = logging.getLogger(__name__) pytestmark = [pytest.mark.topology('multidut-tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfc_pause_single_lossy_prio(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports): # noqa: F811 + enum_dut_lossy_prio, + prio_dscp_map, # noqa: F811 + lossy_prio_list, # noqa: F811 + all_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info): # noqa: F811 """ - Test if PFC will impact a single lossy priority + Test if PFC will impact a single lossy priority in multidut setup Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs + enum_dut_lossy_prio (str): name of lossy priority to test, e.g., 's6100-1|2' prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + lossy_prio_list (pytest fixture): list of all the lossy priorities + all_prio_list (pytest fixture): list of all the priorities + lossy_prio_list (pytest fixture): list of all the lossy priorities + Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) >= 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts - if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - pytest_require(False, "Need Minimum of 2 ports for the test") - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - tgen_ports = [port['location'] for port in snappi_ports] - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - tgen_ports, - snappi_ports, - snappi_api) - all_prio_list = prio_dscp_map.keys() - lossy_prio_list = [x for x in all_prio_list if x not in lossless_prio_list] # noqa: F811 - lossy_prio = int(random.sample(lossy_prio_list, 1)[0]) + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + _, lossy_prio = enum_dut_lossy_prio.split('|') + lossy_prio = int(lossy_prio) pause_prio_list = [lossy_prio] - test_prio_list = pause_prio_list + test_prio_list = [lossy_prio] bg_prio_list = [p for p in all_prio_list] bg_prio_list.remove(lossy_prio) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfc_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, global_pause=False, pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, @@ -92,22 +96,22 @@ def test_pfc_pause_single_lossy_prio(snappi_api, # noqa: F811 prio_dscp_map=prio_dscp_map, test_traffic_pause=False, snappi_extra_params=snappi_extra_params) - - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfc_pause_multi_lossy_prio(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 fanout_graph_facts, # noqa: F811 duthosts, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports # noqa: F811 - ): + prio_dscp_map, # noqa: F811 + lossy_prio_list, # noqa: F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info): # noqa: F811 """ - Test if PFC will impact multiple lossy priorities + Test if PFC will impact multiple lossy priorities in multidut setup Args: snappi_api (pytest fixture): SNAPPI session @@ -115,46 +119,43 @@ def test_pfc_pause_multi_lossy_prio(snappi_api, # noqa: F811 fanout_graph_facts (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) - + lossless_prio_list (pytest fixture): list of all the lossless priorities + lossy_prio_list (pytest fixture): list of all the lossy priorities + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) >= 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts - if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - pytest_require(False, "Need Minimum of 2 ports for the test") - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - tgen_ports = [port['location'] for port in snappi_ports] - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - tgen_ports, - snappi_ports, - snappi_api) - - all_prio_list = prio_dscp_map.keys() - lossy_prio_list = [x for x in all_prio_list if x not in lossless_prio_list] # noqa: F811 + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + pause_prio_list = lossy_prio_list test_prio_list = lossy_prio_list bg_prio_list = lossless_prio_list snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfc_test(api=snappi_api, @@ -169,25 +170,27 @@ def test_pfc_pause_multi_lossy_prio(snappi_api, # noqa: F811 prio_dscp_map=prio_dscp_map, test_traffic_pause=False, snappi_extra_params=snappi_extra_params) - - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) @pytest.mark.disable_loganalyzer @pytest.mark.parametrize('reboot_type', ['warm', 'cold', 'fast']) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfc_pause_single_lossy_prio_reboot(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, localhost, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa: F811 - reboot_type): + enum_dut_lossy_prio, + prio_dscp_map, # noqa: F811 + lossy_prio_list, # noqa: F811 + all_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + reboot_type, + multidut_port_info): """ - Test if PFC will impact a single lossy priority after various kinds of reboots + Test if PFC will impact a single lossy priority after various kinds of reboots in multidut setup Args: snappi_api (pytest fixture): SNAPPI session @@ -195,63 +198,65 @@ def test_pfc_pause_single_lossy_prio_reboot(snappi_api, # noqa: F811 fanout_graph_facts (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs localhost (pytest fixture): localhost handle + enum_dut_lossy_prio (str): name of lossy priority to test, e.g., 's6100-1|2' prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). + lossy_prio_list (pytest fixture): list of all the lossy priorities + all_prio_list (pytest fixture): list of all the priorities reboot_type (str): reboot type to be issued on the DUT - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) - + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) >= 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts - if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - pytest_require(False, "Need Minimum of 2 ports for the test") - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - tgen_ports = [port['location'] for port in snappi_ports] - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - tgen_ports, - snappi_ports, - snappi_api) - - all_prio_list = prio_dscp_map.keys() - lossy_prio_list = [x for x in all_prio_list if x not in lossless_prio_list] # noqa: F811 - lossy_prio = int(random.sample(lossy_prio_list, 1)[0]) + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + skip_warm_reboot(snappi_ports[0]['duthost'], reboot_type) + skip_warm_reboot(snappi_ports[1]['duthost'], reboot_type) + + _, lossy_prio = enum_dut_lossy_prio.split('|') + lossy_prio = int(lossy_prio) pause_prio_list = [lossy_prio] - test_prio_list = pause_prio_list + test_prio_list = [lossy_prio] bg_prio_list = [p for p in all_prio_list] bg_prio_list.remove(lossy_prio) - logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost1.hostname)) - reboot(duthost1, localhost, reboot_type=reboot_type) - logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, 0, duthost1.critical_services_fully_started), - "Not all critical services are fully started") + for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]: + logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) + reboot(duthost, localhost, reboot_type=reboot_type) + logger.info("Wait until the system is stable") + wait_until(180, 20, 0, duthost.critical_services_fully_started) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfc_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, global_pause=False, pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, @@ -259,23 +264,24 @@ def test_pfc_pause_single_lossy_prio_reboot(snappi_api, # noqa: F811 prio_dscp_map=prio_dscp_map, test_traffic_pause=False, snappi_extra_params=snappi_extra_params) - - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) @pytest.mark.disable_loganalyzer @pytest.mark.parametrize('reboot_type', ['warm', 'cold', 'fast']) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfc_pause_multi_lossy_prio_reboot(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 fanout_graph_facts, # noqa: F811 duthosts, localhost, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa: F811 - reboot_type): + prio_dscp_map, # noqa: F811 + lossy_prio_list, # noqa: F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + reboot_type, + multidut_port_info): """ Test if PFC will impact multiple lossy priorities after various kinds of reboots @@ -287,61 +293,59 @@ def test_pfc_pause_multi_lossy_prio_reboot(snappi_api, # noqa: F811 localhost (pytest fixture): localhost handle duthosts (pytest fixture): list of DUTs prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). + lossless_prio_list (pytest fixture): list of all the lossless priorities + lossy_prio_list (pytest fixture): list of all the lossy priorities reboot_type (str): reboot type to be issued on the DUT - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) - + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + skip_warm_reboot(snappi_ports[0]['duthost'], reboot_type) + skip_warm_reboot(snappi_ports[1]['duthost'], reboot_type) - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) >= 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts - if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - pytest_require(False, "Need Minimum of 2 ports for the test") - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - tgen_ports = [port['location'] for port in snappi_ports] - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - tgen_ports, - snappi_ports, - snappi_api) - - all_prio_list = prio_dscp_map.keys() - lossy_prio_list = [x for x in all_prio_list if x not in lossless_prio_list] # noqa: F811 pause_prio_list = lossy_prio_list test_prio_list = lossy_prio_list bg_prio_list = lossless_prio_list - logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost1.hostname)) - reboot(duthost1, localhost, reboot_type=reboot_type) - logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, 0, duthost1.critical_services_fully_started), - "Not all critical services are fully started") + for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]: + logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) + reboot(duthost, localhost, reboot_type=reboot_type) + logger.info("Wait until the system is stable") + wait_until(180, 20, 0, duthost.critical_services_fully_started) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfc_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, global_pause=False, pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, @@ -349,5 +353,4 @@ def test_pfc_pause_multi_lossy_prio_reboot(snappi_api, # noqa: F811 prio_dscp_map=prio_dscp_map, test_traffic_pause=False, snappi_extra_params=snappi_extra_params) - - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_basic_helper.py b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_basic_helper.py index 46fec194b6..93995f72bd 100644 --- a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_basic_helper.py +++ b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_basic_helper.py @@ -1,23 +1,26 @@ import time from math import ceil import logging +import random from tests.common.helpers.assertions import pytest_assert from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 from tests.common.snappi_tests.snappi_helpers import get_dut_port_id # noqa: F401 from tests.common.snappi_tests.common_helpers import pfc_class_enable_vector, \ get_pfcwd_poll_interval, get_pfcwd_detect_time, get_pfcwd_restore_time, \ - enable_packet_aging, start_pfcwd, sec_to_nanosec # noqa: F401 + enable_packet_aging, start_pfcwd, sec_to_nanosec, get_pfcwd_stats # noqa: F401 from tests.common.snappi_tests.port import select_ports, select_tx_port # noqa: F401 from tests.common.snappi_tests.snappi_helpers import wait_for_arp # noqa: F401 from tests.common.snappi_tests.snappi_test_params import SnappiTestParams +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) PAUSE_FLOW_NAME = "Pause Storm" DATA_FLOW1_NAME = "Data Flow 1" DATA_FLOW2_NAME = "Data Flow 2" -PAUSE_TRAFFIC_DUR = 1 +WARM_UP_TRAFFIC_NAME = "Warm Up Traffic" +WARM_UP_TRAFFIC_DUR = 1 DATA_PKT_SIZE = 1024 SNAPPI_POLL_DELAY_SEC = 2 DEVIATION = 0.3 @@ -55,68 +58,93 @@ def run_pfcwd_basic_test(api, if snappi_extra_params is None: snappi_extra_params = SnappiTestParams() - duthost1 = snappi_extra_params.multi_dut_params.duthost1 + # Traffic flow: + # tx_port (TGEN) --- ingress DUT --- egress DUT --- rx_port (TGEN) + rx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[0] - rx_port_id = rx_port["port_id"] - duthost2 = snappi_extra_params.multi_dut_params.duthost2 + egress_duthost = rx_port['duthost'] + tx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[1] - tx_port_id = tx_port["port_id"] + ingress_duthost = tx_port["duthost"] pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') - start_pfcwd(duthost1, rx_port['asic_value']) - enable_packet_aging(duthost1) - start_pfcwd(duthost2, tx_port['asic_value']) - enable_packet_aging(duthost2) + if (egress_duthost.is_multi_asic): + enable_packet_aging(egress_duthost, rx_port['asic_value']) + enable_packet_aging(ingress_duthost, tx_port['asic_value']) + start_pfcwd(egress_duthost, rx_port['asic_value']) + start_pfcwd(ingress_duthost, tx_port['asic_value']) + else: + enable_packet_aging(egress_duthost) + enable_packet_aging(ingress_duthost) + start_pfcwd(egress_duthost) + start_pfcwd(ingress_duthost) + + ini_stats = {} + for prio in prio_list: + ini_stats.update(get_stats(egress_duthost, rx_port['peer_port'], prio)) # Set appropriate pfcwd loss deviation - these values are based on empirical testing - DEVIATION = 0.35 if duthost1.facts['asic_type'] in ["broadcom"] or \ - duthost2.facts['asic_type'] in ["broadcom"] else 0.3 + DEVIATION = 0.35 if egress_duthost.facts['asic_type'] in ["broadcom"] or \ + ingress_duthost.facts['asic_type'] in ["broadcom"] else 0.3 - poll_interval_sec = get_pfcwd_poll_interval(duthost1, rx_port['asic_value']) / 1000.0 - detect_time_sec = get_pfcwd_detect_time(host_ans=duthost1, intf=dut_port, + poll_interval_sec = get_pfcwd_poll_interval(egress_duthost, rx_port['asic_value']) / 1000.0 + detect_time_sec = get_pfcwd_detect_time(host_ans=egress_duthost, intf=dut_port, asic_value=rx_port['asic_value']) / 1000.0 - restore_time_sec = get_pfcwd_restore_time(host_ans=duthost1, intf=dut_port, + restore_time_sec = get_pfcwd_restore_time(host_ans=egress_duthost, intf=dut_port, asic_value=rx_port['asic_value']) / 1000.0 + """ Warm up traffic is initially sent before any other traffic to prevent pfcwd + fake alerts caused by idle links (non-incremented packet counters) during pfcwd detection periods """ + warm_up_traffic_dur_sec = WARM_UP_TRAFFIC_DUR + warm_up_traffic_delay_sec = 0 + if trigger_pfcwd: """ Large enough to trigger PFC watchdog """ pfc_storm_dur_sec = ceil(detect_time_sec + poll_interval_sec + 0.1) - flow1_delay_sec = restore_time_sec / 2 + flow1_delay_sec = restore_time_sec / 2 + WARM_UP_TRAFFIC_DUR flow1_dur_sec = pfc_storm_dur_sec """ Start data traffic 2 after PFC is restored """ - flow2_delay_sec = pfc_storm_dur_sec + restore_time_sec + poll_interval_sec + flow2_delay_sec = pfc_storm_dur_sec + restore_time_sec + \ + poll_interval_sec + WARM_UP_TRAFFIC_DUR flow2_dur_sec = 1 + flow1_max_loss_rate = 1 flow1_min_loss_rate = 1 - DEVIATION else: pfc_storm_dur_sec = detect_time_sec * 0.5 - flow1_delay_sec = pfc_storm_dur_sec * 0.1 + flow1_delay_sec = pfc_storm_dur_sec * 0.1 + WARM_UP_TRAFFIC_DUR flow1_dur_sec = ceil(pfc_storm_dur_sec) """ Start data traffic 2 after the completion of data traffic 1 """ - flow2_delay_sec = flow1_delay_sec + flow1_dur_sec + 0.1 + flow2_delay_sec = flow1_delay_sec + flow1_dur_sec + WARM_UP_TRAFFIC_DUR + 0.1 flow2_dur_sec = 1 + flow1_max_loss_rate = 0 flow1_min_loss_rate = 0 exp_dur_sec = flow2_delay_sec + flow2_dur_sec + 1 + cisco_platform = "Cisco" in egress_duthost.facts['hwsku'] """ Generate traffic config """ __gen_traffic(testbed_config=testbed_config, port_config_list=port_config_list, - tx_port_id=tx_port_id, - rx_port_id=rx_port_id, + port_id=0, pause_flow_name=PAUSE_FLOW_NAME, pause_flow_dur_sec=pfc_storm_dur_sec, - data_flow_name_list=[DATA_FLOW1_NAME, DATA_FLOW2_NAME], - data_flow_delay_sec_list=[flow1_delay_sec, flow2_delay_sec], - data_flow_dur_sec_list=[flow1_dur_sec, flow2_dur_sec], + data_flow_name_list=[WARM_UP_TRAFFIC_NAME, + DATA_FLOW1_NAME, DATA_FLOW2_NAME], + data_flow_delay_sec_list=[ + warm_up_traffic_delay_sec, flow1_delay_sec, flow2_delay_sec], + data_flow_dur_sec_list=[ + warm_up_traffic_dur_sec, flow1_dur_sec, flow2_dur_sec], data_pkt_size=DATA_PKT_SIZE, prio_list=prio_list, - prio_dscp_map=prio_dscp_map) + prio_dscp_map=prio_dscp_map, + traffic_rate=99.98 if cisco_platform else 100.0, + number_of_streams=2 if cisco_platform else 1) flows = testbed_config.flows @@ -127,15 +155,52 @@ def run_pfcwd_basic_test(api, all_flow_names=all_flow_names, exp_dur_sec=exp_dur_sec) + fin_stats = {} + for prio in prio_list: + fin_stats.update(get_stats(egress_duthost, rx_port['peer_port'], prio)) + + loss_packets = 0 + for k in fin_stats.keys(): + logger.info('Parameter:{}, Initial Value:{}, Final Value:{}'.format(k, ini_stats[k], fin_stats[k])) + if 'DROP' in k: + loss_packets += (int(fin_stats[k]) - int(ini_stats[k])) + + logger.info('Total PFCWD drop packets before and after the test:{}'.format(loss_packets)) + __verify_results(rows=flow_stats, data_flow_name_list=[DATA_FLOW1_NAME, DATA_FLOW2_NAME], - data_flow_min_loss_rate_list=[flow1_min_loss_rate, 0]) + data_flow_min_loss_rate_list=[flow1_min_loss_rate, 0], + data_flow_max_loss_rate_list=[flow1_max_loss_rate, 0], + loss_packets=loss_packets) + + +def get_stats(duthost, port, prio): + """ + Returns the PFCWD stats for Tx Ok, Tx drop, Storm detected and restored. + + Args: + duthost (obj): DUT + port (string): Port on the DUT + prio (int): Priority + + Returns: + Dictionary with prio_'parameter' as key and associated value. + + """ + my_dict = {} + new_dict = {} + init_pfcwd = get_pfcwd_stats(duthost, port, prio) + key_list = ['TX_OK/DROP', 'STORM_DETECTED/RESTORED'] + for keys in key_list: + my_dict[keys] = init_pfcwd[keys] + new_dict = {str(prio)+'_'+k: v for key, value in my_dict.items() for k, v in zip(key.split('/'), value.split('/'))} + + return new_dict def __gen_traffic(testbed_config, port_config_list, - tx_port_id, - rx_port_id, + port_id, pause_flow_name, pause_flow_dur_sec, data_flow_name_list, @@ -143,15 +208,16 @@ def __gen_traffic(testbed_config, data_flow_dur_sec_list, data_pkt_size, prio_list, - prio_dscp_map): + prio_dscp_map, + traffic_rate, + number_of_streams): """ Generate configurations of flows, including data flows and pause storm. Args: testbed_config (obj): testbed L1/L2/L3 configuration port_config_list (list): list of port configuration - tx_port_id: ID of tx port - rx_port_id: ID of rx port + port_id (int): ID of DUT port to test. pause_flow_name (str): name of pause storm pause_flow_dur_sec (float): duration of pause storm in second data_flow_name_list (list): list of data flow names @@ -160,12 +226,26 @@ def __gen_traffic(testbed_config, data_pkt_size (int): size of data packets in byte prio_list (list): priorities of data flows and pause storm prio_dscp_map (dict): Priority vs. DSCP map (key = priority). + traffic_rate: Total rate of traffic for all streams together. + number_of_streams: The number of UDP streams needed. Returns: N/A """ - tx_port_config = next((x for x in port_config_list if x.id == tx_port_id), None) - rx_port_config = next((x for x in port_config_list if x.id == rx_port_id), None) + + rx_port_id = port_id + tx_port_id_list, rx_port_id_list = select_ports(port_config_list=port_config_list, + pattern="many to one", + rx_port_id=rx_port_id) + pytest_assert(len(tx_port_id_list) > 0, "Cannot find any TX ports") + tx_port_id = select_tx_port(tx_port_id_list=tx_port_id_list, + rx_port_id=rx_port_id) + pytest_assert(tx_port_id is not None, "Cannot find a suitable TX port") + + tx_port_config = next( + (x for x in port_config_list if x.id == tx_port_id), None) + rx_port_config = next( + (x for x in port_config_list if x.id == rx_port_id), None) tx_mac = tx_port_config.mac if tx_port_config.gateway == rx_port_config.gateway and \ @@ -213,14 +293,14 @@ def __gen_traffic(testbed_config, pause_flow.size.fixed = 64 pause_flow.duration.fixed_packets.packets = int(pause_pkt_cnt) pause_flow.duration.fixed_packets.delay.nanoseconds = int( - sec_to_nanosec(PAUSE_TRAFFIC_DUR)) + sec_to_nanosec(WARM_UP_TRAFFIC_DUR)) pause_flow.metrics.enable = True pause_flow.metrics.loss = True tx_port_name = testbed_config.ports[tx_port_id].name rx_port_name = testbed_config.ports[rx_port_id].name - data_flow_rate_percent = int(20 / len(prio_list)) + data_flow_rate_percent = int(traffic_rate / len(prio_list)) """ For each data flow """ for i in range(len(data_flow_name_list)): @@ -232,10 +312,18 @@ def __gen_traffic(testbed_config, data_flow.tx_rx.port.tx_name = tx_port_name data_flow.tx_rx.port.rx_name = rx_port_name - eth, ipv4 = data_flow.packet.ethernet().ipv4() + eth, ipv4, udp = data_flow.packet.ethernet().ipv4().udp() + src_port = random.randint(5000, 6000) + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = number_of_streams + eth.src.value = tx_mac eth.dst.value = rx_mac - eth.pfc_queue.value = prio + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = prio + else: + eth.pfc_queue.value = pfcQueueValueDict[prio] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip @@ -271,9 +359,9 @@ def __run_traffic(api, config, all_flow_names, exp_dur_sec): api.set_config(config) logger.info('Wait for Arp to Resolve ...') - wait_for_arp(api, max_attempts=10, poll_interval_sec=2) + wait_for_arp(api, max_attempts=30, poll_interval_sec=2) - logger.info('Starting transmiting on all flows ...') + logger.info('Starting transmit on all flows ...') ts = api.transmit_state() ts.state = ts.START api.set_transmit_state(ts) @@ -315,7 +403,9 @@ def __run_traffic(api, config, all_flow_names, exp_dur_sec): def __verify_results(rows, data_flow_name_list, - data_flow_min_loss_rate_list): + data_flow_min_loss_rate_list, + data_flow_max_loss_rate_list, + loss_packets): """ Verify if we get expected experiment results @@ -323,6 +413,7 @@ def __verify_results(rows, rows (list): per-flow statistics data_flow_name_list (list): list of data flow names data_flow_min_loss_rate_list (list): list of data flow min loss rates + data_flow_max_loss_rate_list (list): list of data flow max loss rates Returns: N/A @@ -335,17 +426,22 @@ def __verify_results(rows, flow_name = row.name tx_frames = row.frames_tx rx_frames = row.frames_rx - logger.info('Flow Name : {} , Tx Frames : {}, Rx Frames : {}'.format(flow_name, tx_frames, rx_frames)) for i in range(num_data_flows): if data_flow_name_list[i] in flow_name: data_flow_tx_frames_list[i] += tx_frames data_flow_rx_frames_list[i] += rx_frames + tgen_loss_packets = 0 for i in range(num_data_flows): - loss_rate = 1 - float(data_flow_rx_frames_list[i]) / data_flow_tx_frames_list[i] + tgen_loss_packets += data_flow_tx_frames_list[i] - data_flow_rx_frames_list[i] + loss_rate = 1 - \ + float(data_flow_rx_frames_list[i]) / data_flow_tx_frames_list[i] min_loss_rate = data_flow_min_loss_rate_list[i] - logger.info('Flow Name : {}, Loss Rate : {}'.format(data_flow_name_list[i], loss_rate)) - pytest_assert(loss_rate >= float(min_loss_rate), - '{} has loss rate less than {}'. - format(data_flow_name_list[i], min_loss_rate)) + max_loss_rate = data_flow_max_loss_rate_list[i] + + pytest_assert(loss_rate <= max_loss_rate and loss_rate >= min_loss_rate, + 'Loss rate of {} ({}) should be in [{}, {}]'.format( + data_flow_name_list[i], loss_rate, min_loss_rate, max_loss_rate)) + + logger.info('TGEN Loss packets:{}'.format(tgen_loss_packets)) diff --git a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py index 7c97502eda..9b8b02e925 100644 --- a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py +++ b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py @@ -1,6 +1,7 @@ import time from math import ceil import logging +import random from tests.common.helpers.assertions import pytest_assert from tests.common.snappi_tests.snappi_helpers import get_dut_port_id # noqa: F401 from tests.common.snappi_tests.common_helpers import pfc_class_enable_vector, \ @@ -9,6 +10,7 @@ from tests.common.snappi_tests.port import select_ports, select_tx_port # noqa: F401 from tests.common.snappi_tests.snappi_helpers import wait_for_arp from tests.common.snappi_tests.snappi_test_params import SnappiTestParams +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) @@ -51,22 +53,27 @@ def run_pfcwd_burst_storm_test(api, if snappi_extra_params is None: snappi_extra_params = SnappiTestParams() - duthost1 = snappi_extra_params.multi_dut_params.duthost1 + # Traffic flow: + # tx_port (TGEN) --- ingress DUT --- egress DUT --- rx_port (TGEN) + rx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[0] rx_port_id = rx_port["port_id"] - duthost2 = snappi_extra_params.multi_dut_params.duthost2 + egress_duthost = rx_port['duthost'] + tx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[1] tx_port_id = tx_port["port_id"] + ingress_duthost = tx_port['duthost'] pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') - start_pfcwd(duthost1, rx_port['asic_value']) - enable_packet_aging(duthost1) - start_pfcwd(duthost2, tx_port['asic_value']) - enable_packet_aging(duthost2) - poll_interval_sec = get_pfcwd_poll_interval(duthost1, rx_port['asic_value']) / 1000.0 - detect_time_sec = get_pfcwd_detect_time(host_ans=duthost1, intf=dut_port, asic_value=rx_port['asic_value']) / 1000.0 # noqa: E501 - restore_time_sec = get_pfcwd_restore_time(host_ans=duthost1, intf=dut_port, asic_value=rx_port['asic_value']) / 1000.0 # noqa: E501 + start_pfcwd(egress_duthost, rx_port['asic_value']) + enable_packet_aging(egress_duthost) + start_pfcwd(ingress_duthost, tx_port['asic_value']) + enable_packet_aging(ingress_duthost) + + poll_interval_sec = get_pfcwd_poll_interval(egress_duthost, rx_port['asic_value']) / 1000.0 + detect_time_sec = get_pfcwd_detect_time(host_ans=egress_duthost, intf=rx_port['peer_port'], asic_value=rx_port['asic_value']) / 1000.0 # noqa: E501 + restore_time_sec = get_pfcwd_restore_time(host_ans=egress_duthost, intf=rx_port['peer_port'], asic_value=rx_port['asic_value']) / 1000.0 # noqa: E501 burst_cycle_sec = poll_interval_sec + detect_time_sec + restore_time_sec + 0.1 data_flow_dur_sec = ceil(burst_cycle_sec * BURST_EVENTS) pause_flow_dur_sec = poll_interval_sec * 0.5 @@ -108,7 +115,7 @@ def run_pfcwd_burst_storm_test(api, __verify_results(rows=flow_stats, data_flow_prefix=DATA_FLOW_PREFIX, pause_flow_prefix=PAUSE_FLOW_PREFIX, - duthosts=[duthost1, duthost2]) + duthosts=[egress_duthost, ingress_duthost]) def __gen_traffic(testbed_config, @@ -174,10 +181,18 @@ def __gen_traffic(testbed_config, data_flow.tx_rx.port.tx_name = tx_port_name data_flow.tx_rx.port.rx_name = rx_port_name - eth, ipv4 = data_flow.packet.ethernet().ipv4() + eth, ipv4, udp = data_flow.packet.ethernet().ipv4().udp() + src_port = random.randint(5000, 6000) + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + eth.src.value = tx_mac eth.dst.value = rx_mac - eth.pfc_queue.value = prio + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = prio + else: + eth.pfc_queue.value = pfcQueueValueDict[prio] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip diff --git a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py index 737a65f179..f1e4fd6f2c 100644 --- a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py +++ b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py @@ -1,6 +1,7 @@ import time from math import ceil import logging +import random from tests.common.helpers.assertions import pytest_assert, pytest_require from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 @@ -11,6 +12,7 @@ from tests.common.snappi_tests.port import select_ports # noqa: F401 from tests.common.snappi_tests.snappi_helpers import wait_for_arp # noqa: F401 from tests.common.snappi_tests.snappi_test_params import SnappiTestParams +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) @@ -67,25 +69,37 @@ def run_pfcwd_multi_node_test(api, if snappi_extra_params is None: snappi_extra_params = SnappiTestParams() - duthost1 = snappi_extra_params.multi_dut_params.duthost1 + # Traffic flow: + # tx_port (TGEN) --- ingress DUT --- egress DUT --- rx_port (TGEN) + + # initialize the (duthost, port) set. + # The final list will have all the asics which needs to be configured for PFC + pfcwd_to_be_configured = set() + rx_port = snappi_extra_params.multi_dut_params.multi_dut_ports[0] rx_port_id_list = [rx_port["port_id"]] - duthost2 = snappi_extra_params.multi_dut_params.duthost2 + egress_duthost = rx_port['duthost'] + # Add the port to the set of ports to be configured for PFC + pfcwd_to_be_configured.add((egress_duthost, rx_port['asic_value'])) + tx_port = [snappi_extra_params.multi_dut_params.multi_dut_ports[1], snappi_extra_params.multi_dut_params.multi_dut_ports[2]] tx_port_id_list = [tx_port[0]["port_id"], tx_port[1]["port_id"]] + # add ingress DUT into the set + pfcwd_to_be_configured.add((tx_port[0]['duthost'], tx_port[0]['asic_value'])) + pfcwd_to_be_configured.add((tx_port[1]['duthost'], tx_port[1]['asic_value'])) pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') num_ports = len(port_config_list) pytest_require(num_ports >= 3, "This test requires at least 3 ports") - start_pfcwd(duthost1, rx_port['asic_value']) - enable_packet_aging(duthost1) - start_pfcwd(duthost2, tx_port[0]['asic_value']) - enable_packet_aging(duthost2) + # Enable PFC watchdog on the rx side and tx side of the DUT without duplication. + for duthost, asic in pfcwd_to_be_configured: + start_pfcwd(duthost, asic) + enable_packet_aging(duthost) - poll_interval_sec = get_pfcwd_poll_interval(duthost1, rx_port['asic_value']) / 1000.0 - detect_time_sec = get_pfcwd_detect_time(host_ans=duthost1, intf=dut_port, + poll_interval_sec = get_pfcwd_poll_interval(egress_duthost, rx_port['asic_value']) / 1000.0 + detect_time_sec = get_pfcwd_detect_time(host_ans=egress_duthost, intf=rx_port['peer_port'], asic_value=rx_port['asic_value']) / 1000.0 if trigger_pfcwd: @@ -94,6 +108,16 @@ def run_pfcwd_multi_node_test(api, pfc_storm_dur_sec = 0.5 * detect_time_sec exp_dur_sec = ceil(pfc_storm_dur_sec + 1) + cisco_platform = "Cisco" in egress_duthost.facts['hwsku'] + + speed_str = testbed_config.layer1[0].speed + speed_gbps = int(speed_str.split('_')[1]) + # Backplane is 200G in Cisco platforms. + if speed_gbps > 200 and cisco_platform: + global TEST_FLOW_AGGR_RATE_PERCENT + global BG_FLOW_AGGR_RATE_PERCENT + TEST_FLOW_AGGR_RATE_PERCENT = TEST_FLOW_AGGR_RATE_PERCENT * 200 / speed_gbps + BG_FLOW_AGGR_RATE_PERCENT = BG_FLOW_AGGR_RATE_PERCENT * 200 / speed_gbps """ Generate traffic config """ test_flow_rate_percent = int(TEST_FLOW_AGGR_RATE_PERCENT / @@ -131,9 +155,6 @@ def run_pfcwd_multi_node_test(api, all_flow_names=all_flow_names, exp_dur_sec=exp_dur_sec) - speed_str = testbed_config.layer1[0].speed - speed_gbps = int(speed_str.split('_')[1]) - __verify_results(rows=flow_stats, speed_gbps=speed_gbps, pause_flow_name=PAUSE_FLOW_NAME, @@ -396,10 +417,18 @@ def __gen_data_flow(testbed_config, flow.tx_rx.port.tx_name = testbed_config.ports[src_port_id].name flow.tx_rx.port.rx_name = testbed_config.ports[dst_port_id].name - eth, ipv4 = flow.packet.ethernet().ipv4() + eth, ipv4, udp = flow.packet.ethernet().ipv4().udp() + src_port = random.randint(5000, 6000) + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + eth.src.value = tx_mac eth.dst.value = rx_mac - eth.pfc_queue.value = flow_prio + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = flow_prio + else: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip diff --git a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py index 24a9180512..8e97d4f62d 100644 --- a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py +++ b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py @@ -1,5 +1,6 @@ import time import logging +import random from tests.common.helpers.assertions import pytest_assert from tests.common.snappi_tests.snappi_helpers import get_dut_port_id # noqa: F401 @@ -7,6 +8,7 @@ from tests.common.snappi_tests.port import select_ports, select_tx_port # noqa: F401 from tests.common.snappi_tests.snappi_helpers import wait_for_arp from tests.common.snappi_tests.snappi_test_params import SnappiTestParams +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict DATA_FLOW_NAME = "Data Flow" DATA_PKT_SIZE = 1024 @@ -48,16 +50,20 @@ def run_pfcwd_runtime_traffic_test(api, if snappi_extra_params is None: snappi_extra_params = SnappiTestParams() - duthost1 = snappi_extra_params.duthost1 + # Traffic flow: + # tx_port (TGEN) --- ingress DUT --- egress DUT --- rx_port (TGEN) + rx_port = snappi_extra_params.rx_port - duthost2 = snappi_extra_params.duthost2 - tx_port = snappi_extra_params.tx_port + egress_duthost = rx_port['duthost'] rx_port_id = snappi_extra_params.rx_port_id + + tx_port = snappi_extra_params.tx_port + ingress_duthost = tx_port['duthost'] tx_port_id = snappi_extra_params.tx_port_id pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') - stop_pfcwd(duthost1, rx_port['asic_value']) - stop_pfcwd(duthost2, tx_port['asic_value']) + stop_pfcwd(egress_duthost, rx_port['asic_value']) + stop_pfcwd(ingress_duthost, tx_port['asic_value']) __gen_traffic(testbed_config=testbed_config, port_config_list=port_config_list, @@ -75,7 +81,7 @@ def run_pfcwd_runtime_traffic_test(api, flow_stats = __run_traffic(api=api, config=testbed_config, - duthost=duthost1, + duthost=egress_duthost, port=rx_port, all_flow_names=all_flow_names, pfcwd_start_delay_sec=PFCWD_START_DELAY_SEC, @@ -139,10 +145,18 @@ def __gen_traffic(testbed_config, data_flow.tx_rx.port.tx_name = tx_port_name data_flow.tx_rx.port.rx_name = rx_port_name - eth, ipv4 = data_flow.packet.ethernet().ipv4() + eth, ipv4, udp = data_flow.packet.ethernet().ipv4().udp() + src_port = random.randint(5000, 6000) + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + eth.src.value = tx_mac eth.dst.value = rx_mac - eth.pfc_queue.value = prio + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = prio + else: + eth.pfc_queue.value = pfcQueueValueDict[prio] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip diff --git a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py index db61d74ed0..bb9237fcbc 100644 --- a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py +++ b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py @@ -1,30 +1,29 @@ import pytest import random -from tests.common.helpers.assertions import pytest_require -from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 +import logging +from tests.common.helpers.assertions import pytest_assert # noqa F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, all_prio_list,\ lossless_prio_list, lossy_prio_list # noqa F401 -from tests.snappi_tests.variables import config_set, line_card_choice +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.multidut.pfcwd.files.pfcwd_multidut_multi_node_helper import run_pfcwd_multi_node_test from tests.common.snappi_tests.snappi_test_params import SnappiTestParams - +logger = logging.getLogger(__name__) pytestmark = [pytest.mark.topology('multidut-tgen')] @pytest.mark.parametrize("trigger_pfcwd", [False]) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_multidut_pfcwd_all_to_all(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, - line_card_choice, - rand_one_dut_lossless_prio, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa: F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, # noqa: F811 trigger_pfcwd, prio_dscp_map, # noqa: F811 lossy_prio_list): # noqa: F811 @@ -36,51 +35,51 @@ def test_multidut_pfcwd_all_to_all(snappi_api, # noqa: F811 snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph fanout_graph_facts (pytest fixture): fanout graph - duthosts (pytest fixture): list of DUTs - rand_one_dut_lossless_prio (str): lossless priority to test, e.g., 's6100-1|3' + duthosts (pytest fixture): list of DUTs. + lossless_prio_list (pytest fixture): list of all the lossless priorities lossy_prio_list (pytest fixture): list of lossy priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) trigger_pfcwd (bool): if PFC watchdog is expected to be triggered - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 2 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 3 ports defined in ansible/files/*links.csv file") - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 3: - pytest_require(False, "Need Minimum of 3 ports for the test") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 3) + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) - _, lossless_prio = rand_one_dut_lossless_prio.split('|') - lossless_prio = int(lossless_prio) + lossless_prio = random.sample(lossless_prio_list, 1) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfcwd_multi_node_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], pause_prio_list=[lossless_prio], test_prio_list=[lossless_prio], @@ -90,4 +89,4 @@ def test_multidut_pfcwd_all_to_all(snappi_api, # noqa: F811 pattern="all to all", snappi_extra_params=snappi_extra_params) - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py index ca222b97e8..f039f9b897 100644 --- a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py +++ b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py @@ -1,35 +1,33 @@ import pytest import random import logging -from tests.common.helpers.assertions import pytest_require, pytest_assert -from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 +import re +from collections import defaultdict +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, lossless_prio_list # noqa F401 -from tests.snappi_tests.variables import config_set, line_card_choice +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.common.reboot import reboot # noqa: F401 from tests.common.utilities import wait_until # noqa: F401 from tests.snappi_tests.multidut.pfcwd.files.pfcwd_multidut_basic_helper import run_pfcwd_basic_test from tests.common.snappi_tests.snappi_test_params import SnappiTestParams -from tests.snappi_tests.files.helper import skip_warm_reboot, skip_pfcwd_test -from tests.common.config_reload import config_reload - +from tests.snappi_tests.files.helper import skip_warm_reboot, skip_pfcwd_test # noqa: F401 logger = logging.getLogger(__name__) pytestmark = [pytest.mark.topology('multidut-tgen')] @pytest.mark.parametrize("trigger_pfcwd", [True, False]) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfcwd_basic_single_lossless_prio(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa: F811 - enum_dut_lossless_prio, + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, prio_dscp_map, # noqa F811 trigger_pfcwd): """ @@ -38,75 +36,71 @@ def test_pfcwd_basic_single_lossless_prio(snappi_api, # noqa: Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs - enum_dut_lossless_prio (str): name of lossless priority to test, e.g., 's6100-1|3' prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) trigger_pfcwd (bool): if PFC watchdog is expected to be triggered - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) - get_multidut_snappi_ports: Populates tgen and connected DUT ports info of T0 testbed and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - pytest_assert(len(snappi_port_list) >= 2, "Need Minimum of 2 ports for the test") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) - skip_pfcwd_test(duthost=duthost1, trigger_pfcwd=trigger_pfcwd) - skip_pfcwd_test(duthost=duthost2, trigger_pfcwd=trigger_pfcwd) - - _, lossless_prio = enum_dut_lossless_prio.split('|') + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + skip_pfcwd_test(duthost=snappi_ports[0]['duthost'], trigger_pfcwd=trigger_pfcwd) + skip_pfcwd_test(duthost=snappi_ports[1]['duthost'], trigger_pfcwd=trigger_pfcwd) + + lossless_prio = random.sample(lossless_prio_list, 1) lossless_prio = int(lossless_prio) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfcwd_basic_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], prio_list=[lossless_prio], prio_dscp_map=prio_dscp_map, trigger_pfcwd=trigger_pfcwd, snappi_extra_params=snappi_extra_params) - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) @pytest.mark.parametrize("trigger_pfcwd", [True, False]) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfcwd_basic_multi_lossless_prio(snappi_api, # noqa F811 conn_graph_facts, # noqa F811 - fanout_graph_facts, # noqa F811 + fanout_graph_facts_multidut, # noqa F811 duthosts, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa F811 - lossless_prio_list, # noqa F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, prio_dscp_map, # noqa F811 trigger_pfcwd): """ @@ -115,74 +109,70 @@ def test_pfcwd_basic_multi_lossless_prio(snappi_api, # noqa F811 Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs lossless_prio_list (pytest fixture): list of all the lossless priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) trigger_pfcwd (bool): if PFC watchdog is expected to be triggered - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) - get_multidut_snappi_ports: Populates tgen and connected DUT ports info of T0 testbed and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - pytest_assert(len(snappi_port_list) >= 2, "Need Minimum of 2 ports for the test") - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfcwd_basic_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], prio_list=lossless_prio_list, prio_dscp_map=prio_dscp_map, trigger_pfcwd=trigger_pfcwd, snappi_extra_params=snappi_extra_params) - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) @pytest.mark.disable_loganalyzer @pytest.mark.parametrize('reboot_type', ['warm', 'cold', 'fast']) @pytest.mark.parametrize("trigger_pfcwd", [True, False]) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfcwd_basic_single_lossless_prio_reboot(snappi_api, # noqa F811 conn_graph_facts, # noqa F811 - fanout_graph_facts, # noqa F811 + fanout_graph_facts_multidut, # noqa F811 localhost, duthosts, - rand_one_dut_lossless_prio, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, prio_dscp_map, # noqa F811 reboot_type, trigger_pfcwd): @@ -192,52 +182,48 @@ def test_pfcwd_basic_single_lossless_prio_reboot(snappi_api, # no Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph localhost (pytest fixture): localhost handle duthosts (pytest fixture): list of DUTs - rand_one_dut_lossless_prio (str): name of lossless priority to test, e.g., 's6100-1|3' prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) reboot_type (str): reboot type to be issued on the DUT trigger_pfcwd (bool): if PFC watchdog is expected to be triggered - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) - get_multidut_snappi_ports: Populates tgen and connected DUT ports info of T0 testbed and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - skip_warm_reboot(duthost=duthost1, reboot_type=reboot_type) - skip_warm_reboot(duthost=duthost2, reboot_type=reboot_type) - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - pytest_assert(len(snappi_port_list) >= 2, "Need Minimum of 2 ports for the test") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) - - _, lossless_prio = rand_one_dut_lossless_prio.split('|') + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + lossless_prio = random.sample(lossless_prio_list, 1) lossless_prio = int(lossless_prio) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports - for duthost in dut_list: + for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]: logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True) logger.info("Wait until the system is stable") @@ -248,30 +234,29 @@ def test_pfcwd_basic_single_lossless_prio_reboot(snappi_api, # no testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], prio_list=[lossless_prio], prio_dscp_map=prio_dscp_map, trigger_pfcwd=trigger_pfcwd, snappi_extra_params=snappi_extra_params) - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) @pytest.mark.disable_loganalyzer @pytest.mark.parametrize('reboot_type', ['warm', 'cold', 'fast']) @pytest.mark.parametrize("trigger_pfcwd", [True, False]) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfcwd_basic_multi_lossless_prio_reboot(snappi_api, # noqa F811 conn_graph_facts, # noqa F811 - fanout_graph_facts, # noqa F811 + fanout_graph_facts_multidut, # noqa F811 localhost, duthosts, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa F811 - lossless_prio_list, # noqa F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, prio_dscp_map, # noqa F811 reboot_type, trigger_pfcwd): @@ -281,46 +266,43 @@ def test_pfcwd_basic_multi_lossless_prio_reboot(snappi_api, # no Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph localhost (pytest fixture): localhost handle duthosts (pytest fixture): list of DUTs lossless_prio_list (pytest fixture): list of all the lossless priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) reboot_type (str): reboot type to be issued on the DUT trigger_pfcwd (bool): if PFC watchdog is expected to be triggered - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) - get_multidut_snappi_ports: Populates tgen and connected DUT ports info of T0 testbed and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - skip_warm_reboot(duthost=duthost1, reboot_type=reboot_type) - skip_warm_reboot(duthost=duthost2, reboot_type=reboot_type) - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - pytest_assert(len(snappi_port_list) >= 2, "Need Minimum of 2 ports for the test") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) - - for duthost in dut_list: + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]: logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True) logger.info("Wait until the system is stable") @@ -328,37 +310,34 @@ def test_pfcwd_basic_multi_lossless_prio_reboot(snappi_api, # no "Not all critical services are fully started") snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfcwd_basic_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], prio_list=lossless_prio_list, prio_dscp_map=prio_dscp_map, trigger_pfcwd=trigger_pfcwd, snappi_extra_params=snappi_extra_params) - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) @pytest.mark.disable_loganalyzer @pytest.mark.parametrize('restart_service', ['swss']) @pytest.mark.parametrize("trigger_pfcwd", [True, False]) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfcwd_basic_single_lossless_prio_service_restart(snappi_api, # noqa F811 conn_graph_facts, # noqa F811 - fanout_graph_facts, # noqa F811 + fanout_graph_facts_multidut, # noqa F811 duthosts, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa F811 - rand_one_dut_lossless_prio, + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, prio_dscp_map, # noqa F811 restart_service, trigger_pfcwd): @@ -368,83 +347,100 @@ def test_pfcwd_basic_single_lossless_prio_service_restart(snappi_api, Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs - rand_one_dut_lossless_prio (str): name of lossless priority to test, e.g., 's6100-1|3' prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) restart_service (str): service to restart on the DUT. Only 'swss' affects pfcwd currently trigger_pfcwd (bool): if PFC watchdog is expected to be triggered - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) - get_multidut_snappi_ports: Populates tgen and connected DUT ports info of T0 testbed and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - pytest_assert(len(snappi_port_list) >= 2, "Need Minimum of 2 ports for the test") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) - _, lossless_prio = rand_one_dut_lossless_prio.split('|') + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + lossless_prio = random.sample(lossless_prio_list, 1) lossless_prio = int(lossless_prio) - for duthost in dut_list: - logger.info("Issuing a restart of service {} on the dut {}".format(restart_service, duthost.hostname)) - duthost.command("systemctl reset-failed {}".format(restart_service)) - duthost.command("systemctl restart {}".format(restart_service)) - logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), - "Not all critical services are fully started") + if (snappi_ports[0]['duthost'].is_multi_asic): + ports_dict = defaultdict(list) + for port in snappi_ports: + ports_dict[port['peer_device']].append(port['asic_value']) + + for k in ports_dict.keys(): + ports_dict[k] = list(set(ports_dict[k])) + + logger.info('Port dictionary:{}'.format(ports_dict)) + for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]: + asic_list = ports_dict[duthost.hostname] + for asic in asic_list: + asic_id = re.match(r"(asic)(\d+)", asic).group(2) + proc = 'swss@' + asic_id + logger.info("Issuing a restart of service {} on the dut {}".format(proc, duthost.hostname)) + duthost.command("sudo systemctl reset-failed {}".format(proc)) + duthost.command("sudo systemctl restart {}".format(proc)) + logger.info("Wait until the system is stable") + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), + "Not all critical services are fully started") + else: + for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]: + logger.info("Issuing a restart of service {} on the dut {}".format(restart_service, duthost.hostname)) + duthost.command("systemctl reset-failed {}".format(restart_service)) + duthost.command("systemctl restart {}".format(restart_service)) + logger.info("Wait until the system is stable") + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), + "Not all critical services are fully started") snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfcwd_basic_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], prio_list=[lossless_prio], prio_dscp_map=prio_dscp_map, trigger_pfcwd=trigger_pfcwd, snappi_extra_params=snappi_extra_params) - config_reload(sonic_host=duthost, config_source='config_db', safe_reload=True) + cleanup_config(duthosts, snappi_ports) @pytest.mark.disable_loganalyzer @pytest.mark.parametrize('restart_service', ['swss']) @pytest.mark.parametrize("trigger_pfcwd", [True, False]) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfcwd_basic_multi_lossless_prio_restart_service(snappi_api, # noqa F811 conn_graph_facts, # noqa F811 - fanout_graph_facts, # noqa F811 + fanout_graph_facts_multidut, # noqa F811 duthosts, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa F811 - lossless_prio_list, # noqa F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, prio_dscp_map, # noqa F811 restart_service, trigger_pfcwd): @@ -454,62 +450,81 @@ def test_pfcwd_basic_multi_lossless_prio_restart_service(snappi_api, Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs lossless_prio_list (pytest fixture): list of all the lossless priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) restart_service (str): service to restart on the DUT. Only 'swss' affects pfcwd currently trigger_pfcwd (bool): if PFC watchdog is expected to be triggered - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) - get_multidut_snappi_ports: Populates tgen and connected DUT ports info of T0 testbed and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + if (snappi_ports[0]['duthost'].is_multi_asic): + ports_dict = defaultdict(list) + for port in snappi_ports: + ports_dict[port['peer_device']].append(port['asic_value']) + + for k in ports_dict.keys(): + ports_dict[k] = list(set(ports_dict[k])) + + logger.info('Port dictionary:{}'.format(ports_dict)) + for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]: + asic_list = ports_dict[duthost.hostname] + for asic in asic_list: + asic_id = re.match(r"(asic)(\d+)", asic).group(2) + proc = 'swss@' + asic_id + logger.info("Issuing a restart of service {} on the dut {}".format(proc, duthost.hostname)) + duthost.command("sudo systemctl reset-failed {}".format(proc)) + duthost.command("sudo systemctl restart {}".format(proc)) + logger.info("Wait until the system is stable") + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), + "Not all critical services are fully started") else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - pytest_assert(len(snappi_port_list) >= 2, "Need Minimum of 2 ports for the test") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) - - for duthost in dut_list: - logger.info("Issuing a restart of service {} on the dut {}".format(restart_service, duthost.hostname)) - duthost.command("systemctl reset-failed {}".format(restart_service)) - duthost.command("systemctl restart {}".format(restart_service)) - logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), - "Not all critical services are fully started") + for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]: + logger.info("Issuing a restart of service {} on the dut {}".format(restart_service, duthost.hostname)) + duthost.command("systemctl reset-failed {}".format(restart_service)) + duthost.command("systemctl restart {}".format(restart_service)) + logger.info("Wait until the system is stable") + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), + "Not all critical services are fully started") snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfcwd_basic_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], prio_list=lossless_prio_list, prio_dscp_map=prio_dscp_map, trigger_pfcwd=trigger_pfcwd, snappi_extra_params=snappi_extra_params) - config_reload(sonic_host=duthost, config_source='config_db', safe_reload=True) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_burst_storm_with_snappi.py b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_burst_storm_with_snappi.py index 99891cce11..612db2059c 100644 --- a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_burst_storm_with_snappi.py +++ b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_burst_storm_with_snappi.py @@ -1,31 +1,30 @@ import pytest import random -from tests.common.helpers.assertions import pytest_require -from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 +import logging +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, all_prio_list,\ lossless_prio_list, lossy_prio_list # noqa F401 -from tests.snappi_tests.variables import config_set, line_card_choice +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.multidut.pfcwd.files.pfcwd_multidut_burst_storm_helper import run_pfcwd_burst_storm_test from tests.common.snappi_tests.snappi_test_params import SnappiTestParams - +logger = logging.getLogger(__name__) pytestmark = [pytest.mark.topology('multidut-tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfcwd_burst_storm_single_lossless_prio(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, - rand_one_dut_lossless_prio, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa: F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, prio_dscp_map, # noqa: F811 - lossy_prio_list): # noqa: F811 + ): """ Test PFC watchdog under bursty PFC storms on a single lossless priority @@ -33,57 +32,54 @@ def test_pfcwd_burst_storm_single_lossless_prio(snappi_api, # noqa: Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs rand_one_dut_lossless_prio (str): name of lossless priority to test, e.g., 's6100-1|3' prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) lossy_prio_list (pytest fixture): list of lossy priorities - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) - get_multidut_snappi_ports: Populates tgen and connected DUT ports info of T0 testbed and returns as a list + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - pytest_require(False, "Need Minimum of 2 ports for the test") + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) - _, lossless_prio = rand_one_dut_lossless_prio.split('|') - lossless_prio = int(lossless_prio) + lossless_prio = random.sample(lossless_prio_list, 1) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfcwd_burst_storm_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], prio_list=[lossless_prio], prio_dscp_map=prio_dscp_map, snappi_extra_params=snappi_extra_params) - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py index 73dc70f5c8..a6bbbf4446 100644 --- a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py +++ b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py @@ -1,30 +1,29 @@ import pytest import random -from tests.common.helpers.assertions import pytest_require -from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 +import logging +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports, \ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map, all_prio_list,\ lossless_prio_list, lossy_prio_list # noqa F401 -from tests.snappi_tests.variables import config_set, line_card_choice +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED from tests.snappi_tests.multidut.pfcwd.files.pfcwd_multidut_multi_node_helper import run_pfcwd_multi_node_test from tests.common.snappi_tests.snappi_test_params import SnappiTestParams - +logger = logging.getLogger(__name__) pytestmark = [pytest.mark.topology('multidut-tgen')] @pytest.mark.parametrize("trigger_pfcwd", [True]) -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfcwd_many_to_one(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, - rand_one_dut_lossless_prio, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports, # noqa: F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, trigger_pfcwd, prio_dscp_map, # noqa: F811 lossy_prio_list,): # noqa: F811 @@ -35,55 +34,53 @@ def test_pfcwd_many_to_one(snappi_api, # noqa: F811 Args: snappi_api (pytest fixture): SNAPPI session conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs rand_one_dut_lossless_prio (str): lossless priority to test, e.g., 's6100-1|3' lossy_prio_list (pytest fixture): list of lossy priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) trigger_pfcwd (bool): if PFC watchdog is expected to be triggered - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - pytest_require(False, "Invalid line_card_choice value passed in parameter") - - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if - linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - pytest_require(False, "Hostname can't be an empty list") - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 3: - pytest_require(False, "Need Minimum of 3 ports for the test") + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 2 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 3 ports defined in ansible/files/*links.csv file") - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 3) + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - snappi_ports, - snappi_api) + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) - _, lossless_prio = rand_one_dut_lossless_prio.split('|') - lossless_prio = int(lossless_prio) + lossless_prio = random.sample(lossless_prio_list, 1) snappi_extra_params = SnappiTestParams() - snappi_extra_params.multi_dut_params.duthost1 = duthost1 - snappi_extra_params.multi_dut_params.duthost2 = duthost2 snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports run_pfcwd_multi_node_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], pause_prio_list=[lossless_prio], test_prio_list=[lossless_prio], @@ -93,4 +90,4 @@ def test_pfcwd_many_to_one(snappi_api, # noqa: F811 pattern="many to one", snappi_extra_params=snappi_extra_params) - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_runtime_traffic_with_snappi.py b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_runtime_traffic_with_snappi.py index ca27c1a893..fbe603b109 100644 --- a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_runtime_traffic_with_snappi.py +++ b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_runtime_traffic_with_snappi.py @@ -1,79 +1,74 @@ import pytest -import random +import logging from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 -from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts_multidut # noqa: F401 from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port,\ - snappi_api, snappi_dut_base_config, get_tgen_peer_ports, get_multidut_snappi_ports,\ - get_multidut_tgen_peer_port_set, cleanup_config # noqa: F401 + snappi_api, snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.qos_fixtures import prio_dscp_map # noqa: F401 -from tests.snappi_tests.variables import config_set, line_card_choice -from tests.snappi_tests.multidut.pfcwd.files.pfcwd_multidut_runtime_traffic_helper import run_pfcwd_runtime_traffic_test +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED +from tests.snappi_tests.multidut.pfcwd.files.\ + pfcwd_multidut_runtime_traffic_helper import run_pfcwd_runtime_traffic_test from tests.common.snappi_tests.snappi_test_params import SnappiTestParams +logger = logging.getLogger(__name__) -pytestmark = [pytest.mark.topology('snappi')] +pytestmark = [pytest.mark.topology('multidut-tgen')] -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfcwd_runtime_traffic(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 - fanout_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 duthosts, - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports # noqa: F811 + prio_dscp_map, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, # noqa: F811 ): """ Test PFC watchdog's impact on runtime traffic Args: snappi_api (pytest fixture): SNAPPI session - snappi_testbed_config (pytest fixture): testbed configuration information conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph + fanout_graph_facts_multidut (pytest fixture): fanout graph duthosts (pytest fixture): list of DUTs - rand_one_dut_hostname (str): hostname of DUT - rand_one_dut_portname_oper_up (str): port to test, e.g., 's6100-1|Ethernet0' - all_prio_list (pytest fixture): list of all the priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) - line_card_choice: Line card choice to be mentioned in the variable.py file - linecard_configuration_set : Line card classification, (min 1 or max 2 hostnames and asics to be given) + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - if line_card_choice not in linecard_configuration_set.keys(): - assert False, "Invalid line_card_choice value passed in parameter" + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") - if (len(linecard_configuration_set[line_card_choice]['hostname']) == 2): - dut_list = random.sample(duthosts, 2) - duthost1, duthost2 = dut_list - elif (len(linecard_configuration_set[line_card_choice]['hostname']) == 1): - dut_list = [dut for dut in duthosts if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] # noqa: E501 - duthost1, duthost2 = dut_list[0], dut_list[0] - else: - assert False, "Hostname can't be an empty list" + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - assert False, "Need Minimum of 2 ports for the test" - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, snappi_port_list, config_set, 2) - tgen_ports = [port['location'] for port in snappi_ports] - - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - tgen_ports, - snappi_ports, - snappi_api) + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) all_prio_list = prio_dscp_map.keys() snappi_extra_params = SnappiTestParams() - snappi_extra_params.duthost1 = duthost1 snappi_extra_params.rx_port = snappi_ports[0] snappi_extra_params.rx_port_id = snappi_ports[0]["port_id"] - snappi_extra_params.duthost2 = duthost2 snappi_extra_params.tx_port = snappi_ports[1] snappi_extra_params.tx_port_id = snappi_ports[1]["port_id"] @@ -81,10 +76,10 @@ def test_pfcwd_runtime_traffic(snappi_api, # noqa: F811 testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, dut_port=snappi_ports[0]['peer_port'], prio_list=all_prio_list, prio_dscp_map=prio_dscp_map, snappi_extra_params=snappi_extra_params) - cleanup_config(dut_list, snappi_ports) + cleanup_config(duthosts, snappi_ports) diff --git a/tests/snappi_tests/pfc/files/helper.py b/tests/snappi_tests/pfc/files/helper.py index 533d2fc52b..154e5fd17b 100644 --- a/tests/snappi_tests/pfc/files/helper.py +++ b/tests/snappi_tests/pfc/files/helper.py @@ -94,6 +94,11 @@ def run_pfc_test(api, pytest_assert(port_id is not None, 'Fail to get ID for port {}'.format(dut_port)) + # Single linecard and hence rx_dut and tx_dut are the same. + # rx_dut and tx_dut are used to verify_pause_frame_count + rx_dut = duthost + tx_dut = duthost + # Rate percent must be an integer bg_flow_rate_percent = int(BG_FLOW_AGGR_RATE_PERCENT / len(bg_prio_list)) test_flow_rate_percent = int(TEST_FLOW_AGGR_RATE_PERCENT / len(test_prio_list)) @@ -263,7 +268,10 @@ def run_pfc_test(api, snappi_extra_params=snappi_extra_params) # Verify PFC pause frame count on the DUT - verify_pause_frame_count_dut(duthost=duthost, + # rx_dut is Ingress DUT receiving traffic. + # tx_dut is Egress DUT sending traffic to IXIA and also receiving PFCs. + verify_pause_frame_count_dut(rx_dut=rx_dut, + tx_dut=tx_dut, test_traffic_pause=test_traffic_pause, global_pause=global_pause, snappi_extra_params=snappi_extra_params) diff --git a/tests/snappi_tests/pfcwd/files/pfcwd_basic_helper.py b/tests/snappi_tests/pfcwd/files/pfcwd_basic_helper.py index cd868d31f1..da163989f5 100644 --- a/tests/snappi_tests/pfcwd/files/pfcwd_basic_helper.py +++ b/tests/snappi_tests/pfcwd/files/pfcwd_basic_helper.py @@ -11,6 +11,7 @@ enable_packet_aging, start_pfcwd, sec_to_nanosec from tests.common.snappi_tests.port import select_ports, select_tx_port from tests.common.snappi_tests.snappi_helpers import wait_for_arp +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) @@ -255,7 +256,10 @@ def __gen_traffic(testbed_config, eth, ipv4 = data_flow.packet.ethernet().ipv4() eth.src.value = tx_mac eth.dst.value = rx_mac - eth.pfc_queue.value = prio + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = prio + else: + eth.pfc_queue.value = pfcQueueValueDict[prio] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip diff --git a/tests/snappi_tests/pfcwd/files/pfcwd_burst_storm_helper.py b/tests/snappi_tests/pfcwd/files/pfcwd_burst_storm_helper.py index e00eaa0aee..d6f9a18f2d 100644 --- a/tests/snappi_tests/pfcwd/files/pfcwd_burst_storm_helper.py +++ b/tests/snappi_tests/pfcwd/files/pfcwd_burst_storm_helper.py @@ -9,6 +9,7 @@ enable_packet_aging, start_pfcwd, sec_to_nanosec from tests.common.snappi_tests.port import select_ports, select_tx_port from tests.common.snappi_tests.snappi_helpers import wait_for_arp +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) @@ -186,7 +187,10 @@ def __gen_traffic(testbed_config, eth, ipv4 = data_flow.packet.ethernet().ipv4() eth.src.value = tx_mac eth.dst.value = rx_mac - eth.pfc_queue.value = prio + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = prio + else: + eth.pfc_queue.value = pfcQueueValueDict[prio] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip diff --git a/tests/snappi_tests/pfcwd/files/pfcwd_multi_node_helper.py b/tests/snappi_tests/pfcwd/files/pfcwd_multi_node_helper.py index c923ea8406..4cdfe5b722 100644 --- a/tests/snappi_tests/pfcwd/files/pfcwd_multi_node_helper.py +++ b/tests/snappi_tests/pfcwd/files/pfcwd_multi_node_helper.py @@ -10,6 +10,7 @@ start_pfcwd, enable_packet_aging, get_pfcwd_poll_interval, get_pfcwd_detect_time, sec_to_nanosec from tests.common.snappi_tests.port import select_ports from tests.common.snappi_tests.snappi_helpers import wait_for_arp +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict logger = logging.getLogger(__name__) @@ -405,7 +406,10 @@ def __gen_data_flow(testbed_config, eth, ipv4 = flow.packet.ethernet().ipv4() eth.src.value = tx_mac eth.dst.value = rx_mac - eth.pfc_queue.value = flow_prio + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = flow_prio + else: + eth.pfc_queue.value = pfcQueueValueDict[flow_prio] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip diff --git a/tests/snappi_tests/pfcwd/files/pfcwd_runtime_traffic_helper.py b/tests/snappi_tests/pfcwd/files/pfcwd_runtime_traffic_helper.py index 6d7bfef6d9..14452d6cc4 100644 --- a/tests/snappi_tests/pfcwd/files/pfcwd_runtime_traffic_helper.py +++ b/tests/snappi_tests/pfcwd/files/pfcwd_runtime_traffic_helper.py @@ -6,6 +6,7 @@ from tests.common.snappi_tests.common_helpers import start_pfcwd, stop_pfcwd, sec_to_nanosec from tests.common.snappi_tests.port import select_ports, select_tx_port from tests.common.snappi_tests.snappi_helpers import wait_for_arp +from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict DATA_FLOW_NAME = "Data Flow" WARM_UP_TRAFFIC_NAME = "Warm Up Traffic" @@ -166,7 +167,10 @@ def __gen_traffic(testbed_config, eth, ipv4 = data_flow.packet.ethernet().ipv4() eth.src.value = tx_mac eth.dst.value = rx_mac - eth.pfc_queue.value = prio + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = prio + else: + eth.pfc_queue.value = pfcQueueValueDict[prio] ipv4.src.value = tx_port_config.ip ipv4.dst.value = rx_port_config.ip diff --git a/tests/snappi_tests/test_multidut_snappi.py b/tests/snappi_tests/test_multidut_snappi.py index 46eb5c6fe9..016a222634 100644 --- a/tests/snappi_tests/test_multidut_snappi.py +++ b/tests/snappi_tests/test_multidut_snappi.py @@ -1,23 +1,24 @@ import time import pytest import random +import logging from tests.common.helpers.assertions import pytest_assert, pytest_require -from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts # noqa: F401 -from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, snappi_api,\ - snappi_dut_base_config, get_multidut_snappi_ports, get_multidut_tgen_peer_port_set # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts_multidut # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, snappi_api, \ + snappi_dut_base_config, get_snappi_ports, get_snappi_ports_for_rdma, cleanup_config # noqa: F401 from tests.common.snappi_tests.snappi_helpers import wait_for_arp from tests.common.snappi_tests.port import select_ports from tests.common.snappi_tests.qos_fixtures import prio_dscp_map # noqa: F401 -from tests.snappi_tests.variables import config_set, line_card_choice - +from tests.snappi_tests.variables import MULTIDUT_PORT_INFO, MULTIDUT_TESTBED +logger = logging.getLogger(__name__) SNAPPI_POLL_DELAY_SEC = 2 +pytestmark = [pytest.mark.topology('multidut-tgen')] + -@pytest.mark.topology("snappi") @pytest.mark.disable_loganalyzer def __gen_all_to_all_traffic(testbed_config, port_config_list, - dut_hostname, conn_data, fanout_data, priority, @@ -55,7 +56,12 @@ def __gen_all_to_all_traffic(testbed_config, flow.tx_rx.port.tx_name = tx_port_name flow.tx_rx.port.rx_name = rx_port_name - eth, ipv4 = flow.packet.ethernet().ipv4() + eth, ipv4, udp = flow.packet.ethernet().ipv4().udp() + src_port = random.randint(5000, 6000) + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + eth.src.value = tx_mac eth.dst.value = rx_mac eth.pfc_queue.value = priority @@ -78,16 +84,17 @@ def __gen_all_to_all_traffic(testbed_config, return testbed_config -@pytest.mark.parametrize('line_card_choice', [line_card_choice]) -@pytest.mark.parametrize('linecard_configuration_set', [config_set]) +@pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_snappi(request, duthosts, - snappi_api, conn_graph_facts, fanout_graph_facts, # noqa: F811 - rand_one_dut_lossless_prio, + snappi_api, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + lossless_prio_list, # noqa: F811 + get_snappi_ports, # noqa: F811 + tbinfo, # noqa: F811 + multidut_port_info, prio_dscp_map, # noqa: F811 - line_card_choice, - linecard_configuration_set, - get_multidut_snappi_ports # noqa: F811 ): """ @@ -97,49 +104,47 @@ def test_snappi(request, snappi_api (pytest fixture): Snappi session snappi_testbed_config (pytest fixture): testbed configuration information conn_graph_facts (pytest fixture): connection graph - fanout_graph_facts (pytest fixture): fanout graph - rand_one_dut_lossless_prio (str): name of lossless priority to test + fanout_graph_facts_multidut (pytest fixture): fanout graph prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority) - + tbinfo (pytest fixture): fixture provides information about testbed + get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list Returns: N/A """ - # - if line_card_choice not in linecard_configuration_set.keys(): - assert False, "Invalid line_card_choice value passed in parameter" - - if len(linecard_configuration_set[line_card_choice]['hostname']) > 1: - dut_list = random.sample(duthosts, 2) - elif len(linecard_configuration_set[line_card_choice]['hostname']) == 1: - dut_list = [dut for dut in duthosts - if linecard_configuration_set[line_card_choice]['hostname'] == [dut.hostname]] - else: - assert False, "Hostname can't be an empty list" - - snappi_port_list = get_multidut_snappi_ports(line_card_choice=line_card_choice, - line_card_info=linecard_configuration_set[line_card_choice]) - if len(snappi_port_list) < 2: - assert False, "Need Minimum of 2 ports for the test" - - snappi_ports = get_multidut_tgen_peer_port_set(line_card_choice, - snappi_port_list, - config_set, - 2) - tgen_ports = [port['location'] for port in snappi_ports] - testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(dut_list, - tgen_ports, - snappi_ports, - snappi_api) - - dut_hostname, lossless_prio = rand_one_dut_lossless_prio.split('|') + for testbed_subtype, rdma_ports in multidut_port_info.items(): + tx_port_count = 1 + rx_port_count = 1 + snappi_port_list = get_snappi_ports + pytest_assert(MULTIDUT_TESTBED == tbinfo['conf-name'], + "The testbed name from testbed file doesn't match with MULTIDUT_TESTBED in variables.py ") + pytest_assert(len(snappi_port_list) >= tx_port_count + rx_port_count, + "Need Minimum of 2 ports defined in ansible/files/*links.csv file") + + pytest_assert(len(rdma_ports['tx_ports']) >= tx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Tx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + + pytest_assert(len(rdma_ports['rx_ports']) >= rx_port_count, + 'MULTIDUT_PORT_INFO doesn\'t have the required Rx ports defined for \ + testbed {}, subtype {} in variables.py'. + format(MULTIDUT_TESTBED, testbed_subtype)) + logger.info('Running test for testbed subtype: {}'.format(testbed_subtype)) + snappi_ports = get_snappi_ports_for_rdma(snappi_port_list, rdma_ports, + tx_port_count, rx_port_count, MULTIDUT_TESTBED) + testbed_config, port_config_list, snappi_ports = snappi_dut_base_config(duthosts, + snappi_ports, + snappi_api) + + lossless_prio = random.sample(lossless_prio_list, 1) + lossless_prio = int(lossless_prio) pytest_require(len(port_config_list) >= 2, "This test requires at least 2 ports") config = __gen_all_to_all_traffic(testbed_config=testbed_config, port_config_list=port_config_list, - dut_hostname=dut_hostname, conn_data=conn_graph_facts, - fanout_data=fanout_graph_facts, + fanout_data=fanout_graph_facts_multidut, priority=int(lossless_prio), prio_dscp_map=prio_dscp_map) diff --git a/tests/snappi_tests/test_snappi.py b/tests/snappi_tests/test_snappi.py index c89f349061..c8c2bcb40c 100644 --- a/tests/snappi_tests/test_snappi.py +++ b/tests/snappi_tests/test_snappi.py @@ -12,8 +12,9 @@ SNAPPI_POLL_DELAY_SEC = 2 +pytestmark = [pytest.mark.topology('snappi')] + -@pytest.mark.topology("snappi") @pytest.mark.disable_loganalyzer def __gen_all_to_all_traffic(testbed_config, port_config_list, diff --git a/tests/snappi_tests/variables.py b/tests/snappi_tests/variables.py index a2e9040379..c862c0c2f3 100644 --- a/tests/snappi_tests/variables.py +++ b/tests/snappi_tests/variables.py @@ -1,25 +1,52 @@ +import sys +import ipaddress +from ipaddress import ip_address, IPv4Address, IPv6Address + +# NOTE: Ensure the ports are mapped correctly to the respective duts in ansible/files/*links.csv +# NOTE: The MULTIDUT_TESTBED must match with the conf-name defined in testbed.yml/testbed.csv file +MULTIDUT_TESTBED = 'vms-snappi-sonic-multidut' +MULTIDUT_PORT_INFO = {MULTIDUT_TESTBED: ( + ({ + 'multi-dut-single-asic': { + 'rx_ports': [ + {'port_name': 'Ethernet72', 'hostname': "sonic-s6100-dut1"}, + {'port_name': 'Ethernet76', 'hostname': "sonic-s6100-dut1"} + ], + 'tx_ports': [ + {'port_name': 'Ethernet64', 'hostname': "sonic-s6100-dut2"}, + {'port_name': 'Ethernet68', 'hostname': "sonic-s6100-dut2"} + ] + } + }), + ({ + 'single-dut-single-asic': { + 'rx_ports': [ + {'port_name': 'Ethernet72', 'hostname': "sonic-s6100-dut1"}, + {'port_name': 'Ethernet76', 'hostname': "sonic-s6100-dut1"} + ], + 'tx_ports': [ + {'port_name': 'Ethernet64', 'hostname': "sonic-s6100-dut1"}, + {'port_name': 'Ethernet68', 'hostname': "sonic-s6100-dut1"} + ] + } + }) +)} + ''' In this file user can modify the line_card_choice and it chooses the corresponding hostname and asic values from the config_set hostnames can be modified according to the dut hostname mentioned in the snappi_sonic_devices.csv and asic values based on if its a chassis based dut - chassis_single_line_card_single_asic : this option selects the ports form the hostname and its respective asic value - chassis_single_line_card_multi_asic : this option selects the ports from the hostname and minimum of 1 port from each of the asic values - chassis_multi_line_card_single_asic : this option selects min 1 port from each of the hostnames and its asic value - chassis_multi_line_card_multi_asic : this option selects min of 1 port from hostname1 and asic1 and 1 port from hostname2 and asic2 - non_chassis_multi_line_card : this option selects min of 1 port from hostname1 and 1 port from hostname2 - non_chassis_single_line_card : this option selects all the ports from the hostname - ''' line_card_choice = 'chassis_multi_line_card_multi_asic' config_set = { @@ -49,6 +76,194 @@ } } -dut_ip_start = '20.0.1.1' -snappi_ip_start = '20.0.1.2' -prefix_length = 24 +dut_ip_start = '20.1.1.0' +snappi_ip_start = '20.1.1.1' +prefix_length = 31 + +dut_ipv6_start = '2000:1::1' +snappi_ipv6_start = '2000:1::2' +v6_prefix_length = 126 + +pfcQueueGroupSize = 8 # can have values 4 or 8 +pfcQueueValueDict = {0: 0, + 1: 1, + 2: 0, + 3: 3, + 4: 2, + 5: 0, + 6: 1, + 7: 0} + + +def create_ip_list(value, count, mask=32, incr=0): + ''' + Create a list of ips based on the count provided + Parameters: + value: start value of the list + count: number of ips required + mask: subnet mask for the ips to be created + incr: increment value of the ip + ''' + if sys.version_info.major == 2: + value = unicode(value) # noqa: F821 + + ip_list = [value] + for i in range(1, count): + if ip_address(value).version == 4: + incr1 = pow(2, (32 - int(mask))) + incr + value = (IPv4Address(value) + incr1).compressed + elif ip_address(value).version == 6: + if mask == 32: + mask = 64 + incr1 = pow(2, (128 - int(mask))) + incr + value = (IPv6Address(value) + incr1).compressed + ip_list.append(value) + + return ip_list + + +def get_host_addresses(subnet, count): + try: + # Create an IPv4Network object + network = ipaddress.ip_network(subnet, strict=False) + + # Generate all possible host addresses + all_hosts = list(network.hosts()) + + # Check if the requested count is within the available host range + if count > len(all_hosts): + raise ValueError("Requested count exceeds the number of available hosts in the subnet.") + + # Return the list of host addresses up to the specified count + return all_hosts[:count] + + except ValueError as e: + return str(e) + + +ip = [] +peer_ip = [] +ipv6 = [] +peer_ipv6 = [] +# START --------------------- T2 BGP Case ------------------- +''' + PRE-REQUISITE : The DUT ports must be Administratively Up and configured as Routed ports before starting the test +''' +# *********** Common variables for Performance and Outbound **************** +T2_SNAPPI_AS_NUM = 65400 +T2_DUT_AS_NUM = 65100 +BGP_TYPE = 'ebgp' +t1_t2_device_hostnames = ["sonic-t1", "sonic-t2-uplink", "sonic-t2-downlink"] +SNAPPI_TRIGGER = 60 # timeout value for snappi operation +DUT_TRIGGER = 180 # timeout value for dut operation + +ipv4_subnet = '20.0.1.1/31' +ipv6_subnet = '2000:1:1:1::1/126' +v4_prefix_length = int(ipv4_subnet.split('/')[1]) +v6_prefix_length = int(ipv6_subnet.split('/')[1]) + +# *********** Performance case variables **************** +# asic_value is None if it's non-chassis based or single line card +PERFORMANCE_PORTS = { + 'Traffic_Tx_Ports': [ + {'port_name': 'Ethernet0', 'hostname': t1_t2_device_hostnames[1], 'asic_value': 'asic0'}, + {'port_name': 'Ethernet88', 'hostname': t1_t2_device_hostnames[1], 'asic_value': 'asic0'}, + ], + 'Uplink BGP Session': [ + {'port_name': 'Ethernet192', 'hostname': t1_t2_device_hostnames[1], 'asic_value': 'asic1'}, + {'port_name': 'Ethernet144', 'hostname': t1_t2_device_hostnames[1], 'asic_value': 'asic1'}, + ] + } +# *********** Outbound case variables **************** +# Expect the T1 and T2 ports to be routed ports and not part of any portchannel. +T1_SNAPPI_AS_NUM = 65300 +T1_DUT_AS_NUM = 65200 +AS_PATHS = [65002] + +snappi_community_for_t1 = ["8075:54000"] +snappi_community_for_t2 = ["8075:316", "8075:10400"] +fanout_presence = True +# Note: Increase the MaxSessions in /etc/ssh/sshd_config if the number of fanout ports used is more than 10 +t2_uplink_fanout_info = [ + { + 'fanout_ip': '152.148.150.143', + 'port_mapping': [{'fanout_port': 'Ethernet0', 'uplink_port': 'Ethernet0'}, + {'fanout_port': 'Ethernet88', 'uplink_port': 'Ethernet88'}, + {'fanout_port': 'Ethernet192', 'uplink_port': 'Ethernet192'}, + {'fanout_port': 'Ethernet144', 'uplink_port': 'Ethernet144'}] + }, + { + 'fanout_ip': '152.148.150.142', + 'port_mapping': [{'fanout_port': 'Ethernet2', 'uplink_port': 'Ethernet2'}, + {'fanout_port': 'Ethernet3', 'uplink_port': 'Ethernet3'}, + {'fanout_port': 'Ethernet4', 'uplink_port': 'Ethernet4'}, + {'fanout_port': 'Ethernet5', 'uplink_port': 'Ethernet5'}] + } + ] +# The order of hostname is very important for the outbound test (T1, T2 Uplink, T2 Downlink and Supervisor) +t1_t2_device_hostnames = ["sonic-t1", "sonic-t2-uplink", "sonic-t2-downlink", "sonic-t2-supervisor"] +t1_ports = { + t1_t2_device_hostnames[0]: + [ + 'Ethernet8', + 'Ethernet16' + ] + } + +# asic_value is None if it's non-chassis based or single line card +t2_uplink_portchannel_members = { + t1_t2_device_hostnames[1]: + { + 'asic0': + { + 'PortChannel0': ['Ethernet0', 'Ethernet88'] + }, + 'asic1': + { + 'PortChannel1': ['Ethernet192', 'Ethernet144'] + } + } + } +# TODO: Multiple interconnected ports scenario +t1_side_interconnected_port = 'Ethernet120' +t2_side_interconnected_port = {'port_name': 'Ethernet272', 'asic_value': 'asic1'} + +routed_port_count = 1+len(t1_ports[t1_t2_device_hostnames[0]]) +portchannel_count = sum([len(portchannel_info) for _, portchannel_info in + t2_uplink_portchannel_members[t1_t2_device_hostnames[1]].items()]) + + +def generate_ips_for_bgp_case(ipv4_subnet, ipv6_subnet): + v4_start_ips = create_ip_list(ipv4_subnet.split('/')[0], routed_port_count+portchannel_count, mask=16) + v6_start_ips = create_ip_list(ipv6_subnet.split('/')[0], routed_port_count+portchannel_count, mask=64) + count = 2 # Note: count is always 2 + + for index in range(0, routed_port_count+portchannel_count): + v4_host_addresses = get_host_addresses(str(v4_start_ips[index])+'/'+str(ipv4_subnet.split('/')[1]), count) + v6_host_addresses = get_host_addresses(str(v6_start_ips[index])+'/'+str(ipv6_subnet.split('/')[1]), count) + ip.append(str(v4_host_addresses[0])) + peer_ip.append(str(v4_host_addresses[1])) + ipv6.append(str(v6_host_addresses[0])) + peer_ipv6.append(str(v6_host_addresses[1])) + + +generate_ips_for_bgp_case(ipv4_subnet, ipv6_subnet) +router_ids = create_ip_list('100.0.0.1', routed_port_count+portchannel_count, mask=32) +t1_t2_dut_ipv4_list = ip[:routed_port_count] +t1_t2_snappi_ipv4_list = peer_ip[:routed_port_count] + +t2_dut_portchannel_ipv4_list = ip[routed_port_count:] +snappi_portchannel_ipv4_list = peer_ip[routed_port_count:] + +t1_t2_dut_ipv6_list = ipv6[:routed_port_count] +t1_t2_snappi_ipv6_list = peer_ipv6[:routed_port_count] + +t2_dut_portchannel_ipv6_list = ipv6[routed_port_count:] +snappi_portchannel_ipv6_list = peer_ipv6[routed_port_count:] + +t2_dut_ipv4_list = ip[:len(PERFORMANCE_PORTS['Traffic_Tx_Ports'] + PERFORMANCE_PORTS['Uplink BGP Session'])] +t2_dut_ipv6_list = ipv6[:len(PERFORMANCE_PORTS['Traffic_Tx_Ports'] + PERFORMANCE_PORTS['Uplink BGP Session'])] +t2_snappi_ipv4_list = peer_ip[:len(PERFORMANCE_PORTS['Traffic_Tx_Ports'] + PERFORMANCE_PORTS['Uplink BGP Session'])] +t2_snappi_ipv6_list = peer_ipv6[:len(PERFORMANCE_PORTS['Traffic_Tx_Ports'] + PERFORMANCE_PORTS['Uplink BGP Session'])] + +# END --------------------- T2 BGP Case ------------------- diff --git a/tests/snmp/test_snmp_fdb.py b/tests/snmp/test_snmp_fdb.py index 7e325f4599..22a09087d5 100644 --- a/tests/snmp/test_snmp_fdb.py +++ b/tests/snmp/test_snmp_fdb.py @@ -71,6 +71,26 @@ def build_icmp_packet(vlan_id, src_mac="00:22:00:00:00:02", dst_mac="ff:ff:ff:ff return pkt +def is_port_channel_up(duthost, config_portchannels): + portchannel_status = duthost.show_and_parse("show int po") + portchannel_status_obj = {} + for item in portchannel_status: + all_members_up = True + for port in item["ports"].split(" "): + all_members_up = all_members_up and port.endswith("(S)") + portchannel_status_obj[item["team dev"]] = { + "pc_up": True if item["protocol"].endswith("(Up)") else False, + "all_members_up": all_members_up + } + for portchannel in config_portchannels.keys(): + if portchannel not in portchannel_status_obj: + return False + if not (portchannel_status_obj[portchannel]["pc_up"] and + portchannel_status_obj[portchannel]["all_members_up"]): + return False + return True + + @pytest.mark.bsl @pytest.mark.po2vlan def test_snmp_fdb_send_tagged(ptfadapter, duthosts, rand_one_dut_hostname, # noqa F811 @@ -85,6 +105,7 @@ def test_snmp_fdb_send_tagged(ptfadapter, duthosts, rand_one_dut_hostname, cfg_facts = duthost.config_facts(host=duthost.hostname, source="running")[ 'ansible_facts'] config_portchannels = cfg_facts.get('PORTCHANNEL', {}) + assert wait_until(60, 2, 0, is_port_channel_up, duthost, config_portchannels), "Portchannel is not up" send_cnt = 0 send_portchannels_cnt = 0 vlan_ports_list = running_vlan_ports_list(duthosts, rand_one_dut_hostname, rand_selected_dut, tbinfo, ports_list) diff --git a/tests/snmp/test_snmp_link_local.py b/tests/snmp/test_snmp_link_local.py index a9a8eae750..7f8f58ff68 100644 --- a/tests/snmp/test_snmp_link_local.py +++ b/tests/snmp/test_snmp_link_local.py @@ -22,7 +22,8 @@ def test_snmp_link_local_ip(duthosts, nbrhosts, tbinfo, localhost, creds_all_duts): """ Test SNMP query to DUT over link local IP - - Send SNMP query over link local IP from one of the BGP Neighbors + - configure eth0's link local IP as snmpagentaddress + - Query over linklocal IP from within snmp docker - Get SysDescr from snmpfacts - compare result from snmp query over link local IP and snmpfacts """ @@ -44,13 +45,9 @@ def test_snmp_link_local_ip(duthosts, link_local_ip = ip.split()[1] break # configure link local IP in config_db - duthost.shell( - 'sonic-db-cli CONFIG_DB hset "MGMT_INTERFACE|eth0|{}" \ - "gwaddr" "fe80::1"' - .format(link_local_ip)) # Restart snmp service to regenerate snmpd.conf with # link local IP configured in MGMT_INTERFACE - duthost.shell("systemctl restart snmp") + duthost.shell("config snmpagentaddress add {}%eth0".format(link_local_ip)) stdout_lines = duthost.shell("docker exec snmp snmpget \ -v2c -c {} {}%eth0 {}" .format(creds_all_duts[duthost.hostname] diff --git a/tests/snmp/test_snmp_lldp.py b/tests/snmp/test_snmp_lldp.py index 764ad3d26f..7b51fdf112 100644 --- a/tests/snmp/test_snmp_lldp.py +++ b/tests/snmp/test_snmp_lldp.py @@ -65,12 +65,14 @@ def test_snmp_lldp(duthosts, enum_rand_one_per_hwsku_hostname, localhost, creds_ assert "No Such Object currently exists" not in v[oid] # Check if lldpLocManAddrTable is present - for k in ['lldpLocManAddrLen', - 'lldpLocManAddrIfSubtype', - 'lldpLocManAddrIfId', - 'lldpLocManAddrOID']: - assert snmp_facts['snmp_lldp'][k] - assert "No Such Object currently exists" not in snmp_facts['snmp_lldp'][k] + if not duthost.facts['modular_chassis']: + # Modular Chassis LCs do not run global lldp service + for k in ['lldpLocManAddrLen', + 'lldpLocManAddrIfSubtype', + 'lldpLocManAddrIfId', + 'lldpLocManAddrOID']: + assert snmp_facts['snmp_lldp'][k] + assert "No Such Object currently exists" not in snmp_facts['snmp_lldp'][k] minigraph_lldp_nei = [] for k, v in list(mg_facts.items()): diff --git a/tests/snmp/test_snmp_phy_entity.py b/tests/snmp/test_snmp_phy_entity.py index 67e18f35f3..dfc6490fe4 100644 --- a/tests/snmp/test_snmp_phy_entity.py +++ b/tests/snmp/test_snmp_phy_entity.py @@ -7,7 +7,7 @@ from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_require from tests.common.helpers.snmp_helpers import get_snmp_facts -from tests.platform_tests.thermal_control_test_helper import mocker_factory # noqa F401 +from tests.common.helpers.thermal_control_test_helper import mocker_factory # noqa F401 pytestmark = [ pytest.mark.topology('any'), @@ -525,7 +525,7 @@ def test_transceiver_info(duthosts, enum_rand_one_per_hwsku_hostname, snmp_physi keys = redis_get_keys(duthost, STATE_DB, XCVR_KEY_TEMPLATE.format('*')) # Ignore the test if the platform does not have interfaces (e.g Supervisor) if not keys: - pytest.skip('Fan information does not exist in DB, skipping this test') + pytest.skip('Transceiver information does not exist in DB, skipping this test') name_to_snmp_facts = {} for oid, values in list(snmp_physical_entity_info.items()): values['oid'] = oid @@ -786,11 +786,23 @@ def redis_get_keys(duthost, db_id, pattern): :param pattern: Redis key pattern :return: A list of key name in string """ + totalOutput = [] + + def run_cmd_store_output(cmd): + logging.debug('Getting keys from redis by command: {}'.format(cmd)) + output = duthost.shell(cmd)['stdout'].strip() + if output: + totalOutput.extend(output.split('\n')) + + if duthost.is_multi_asic: + # Search the namespaces as well on LCs + for asic in duthost.frontend_asics: + cmd = 'sonic-db-cli -n {} {} KEYS \"{}\"'.format(asic.namespace, db_id, pattern) + run_cmd_store_output(cmd) + cmd = 'sonic-db-cli {} KEYS \"{}\"'.format(db_id, pattern) - logging.debug('Getting keys from redis by command: {}'.format(cmd)) - output = duthost.shell(cmd) - content = output['stdout'].strip() - return content.split('\n') if content else None + run_cmd_store_output(cmd) + return totalOutput if totalOutput else None def redis_hgetall(duthost, db_id, key): @@ -801,14 +813,28 @@ def redis_hgetall(duthost, db_id, key): :param key: Redis Key :return: A dictionary, key is field name, value is field value """ + + def run_cmd(cmd): + output = duthost.shell(cmd)['stdout'].strip() + if not output: + return {} + # fix to make literal_eval() work with nested dictionaries + content = output.replace("'{", '"{').replace("}'", '}"') + return ast.literal_eval(content) + + if duthost.is_multi_asic: + # Search the namespaces as well on LCs + for asic in duthost.frontend_asics: + cmd = 'sonic-db-cli -n {} {} HGETALL \"{}\"'.format(asic.namespace, db_id, key) + output = run_cmd(cmd) + if output: + return output + cmd = 'sonic-db-cli {} HGETALL \"{}\"'.format(db_id, key) - output = duthost.shell(cmd) - content = output['stdout'].strip() - if not content: - return {} - # fix to make literal_eval() work with nested dictionaries - content = content.replace("'{", '"{').replace("}'", '}"') - return ast.literal_eval(content) + output = run_cmd(cmd) + if output: + return output + return {} def is_null_str(value): diff --git a/tests/snmp/test_snmp_psu.py b/tests/snmp/test_snmp_psu.py index ca3f87cfab..05d2ce4942 100644 --- a/tests/snmp/test_snmp_psu.py +++ b/tests/snmp/test_snmp_psu.py @@ -1,4 +1,5 @@ import pytest +import logging from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.snmp_helpers import get_snmp_facts @@ -21,7 +22,13 @@ def test_snmp_numpsu(duthosts, enum_supervisor_dut_hostname, localhost, creds_al snmp_facts = get_snmp_facts( localhost, host=hostip, version="v2c", community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts'] - res = duthost.shell("psuutil numpsus") + res = duthost.shell("psuutil numpsus", module_ignore_errors=True) + + # For kvm testbed, we will get the expected return code 2 because of no chassis + if duthost.facts["asic_type"] == "vs" and res['rc'] == 2: + logging.info("Get expected return code 2 on kvm testbed.") + return + assert int(res['rc']) == 0, "Failed to get number of PSUs" numpsus = int(res['stdout']) @@ -40,6 +47,11 @@ def test_snmp_psu_status(duthosts, enum_supervisor_dut_hostname, localhost, cred psus_on = 0 msg = "Unexpected operstatus results {} != {} for PSU {}" + # For kvm testbed, there is no snmp psu info + if duthost.facts["asic_type"] == "vs": + logging.info("No snmp psu info on kvm testbed.") + return + for psu_indx, operstatus in list(snmp_facts['snmp_psu'].items()): get_presence = duthost.shell( "redis-cli -n 6 hget 'PSU_INFO|PSU {}' presence".format(psu_indx)) diff --git a/tests/snmp/test_snmp_queue_counters.py b/tests/snmp/test_snmp_queue_counters.py index 5098ccb3e6..07a2fdbf4a 100644 --- a/tests/snmp/test_snmp_queue_counters.py +++ b/tests/snmp/test_snmp_queue_counters.py @@ -24,15 +24,47 @@ def get_queue_ctrs(duthost, cmd): return len(duthost.shell(cmd)["stdout_lines"]) +def get_queue_cntrs_oid(interface): + """ + @summary: Returns queue_cntrs_oid value based on the interface chosen + for single/multi-asic sonic host. + Args: + interface: Asic interface selected + Returns: + queue_cntrs_oid + """ + intf_num = interface.split('Ethernet')[1] + queue_cntrs_oid = '1.3.6.1.4.1.9.9.580.1.5.5.1.4.{}'.format(int(intf_num) + 1) + return queue_cntrs_oid + + +def get_asic_interface(inter_facts): + """ + @summary: Returns interface dynamically based on the asic chosen + for single/multi-asic sonic host. + """ + ansible_inter_facts = inter_facts['ansible_interface_facts'] + interface = None + for key, v in ansible_inter_facts.items(): + # Exclude internal interfaces + if 'IB' in key or 'Rec' in key or 'BP' in key: + continue + if 'Ether' in key and v['active']: + interface = key + break + + return interface + + def test_snmp_queue_counters(duthosts, - enum_rand_one_per_hwsku_frontend_hostname, - creds_all_duts): + enum_rand_one_per_hwsku_frontend_hostname, enum_frontend_asic_index, + creds_all_duts, teardown): """ Test SNMP queue counters - Set "create_only_config_db_buffers" to true in config db, to create only relevant counters - - Remove one of the buffer queues, Ethernet0|3-4 is chosen arbitrary - - Using snmpwalk compare number of queue counters on Ethernet0, assuming + - Remove one of the buffer queues for asic interface chosen, |3-4 is chosen arbitrary + - Using snmpwalk compare number of queue counters on the interface, assuming there will be 8 less after removing the buffer. (Assuming unicast only, 4 counters for each queue in this case) This test covers the issue: 'The feature "polling only configured ports @@ -41,17 +73,42 @@ def test_snmp_queue_counters(duthosts, """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + global ORIG_CFG_DB, CFG_DB_PATH hostip = duthost.host.options['inventory_manager'].get_host( duthost.hostname).vars['ansible_host'] - Ethernet0_queue_cntrs_oid = '1.3.6.1.4.1.9.9.580.1.5.5.1.4.1' + asic = duthost.asic_instance(enum_frontend_asic_index) + int_facts = asic.interface_facts()['ansible_facts'] + interface = get_asic_interface(int_facts) + if interface is None: + pytest.skip("No active interface present on the asic {}".format(asic)) + queue_cntrs_oid = get_queue_cntrs_oid(interface) get_bfr_queue_cntrs_cmd \ = "docker exec snmp snmpwalk -v2c -c {} {} {}".format( creds_all_duts[duthost.hostname]['snmp_rocommunity'], hostip, - Ethernet0_queue_cntrs_oid) + queue_cntrs_oid) + # Generate sonic-cfggen commands for multi-asic and single-asic duts + if duthost.sonichost.is_multi_asic and asic is not None: + ORIG_CFG_DB = "/etc/sonic/orig_config_db{}.json".format(asic.asic_index) + CFG_DB_PATH = "/etc/sonic/config_db{}.json".format(asic.asic_index) + cmd = "sonic-cfggen -n {} -d --print-data > {}".format(asic.namespace, ORIG_CFG_DB) + else: + cmd = "sonic-cfggen -d --print-data > {}".format(ORIG_CFG_DB) - duthost.shell("sonic-cfggen -d --print-data > {}".format(ORIG_CFG_DB)) + duthost.shell(cmd) data = json.loads(duthost.shell("cat {}".format(ORIG_CFG_DB), verbose=False)['stdout']) + buffer_queue_to_del = None + # Get appropriate buffer queue value to delete in case of multi-asic + if duthost.sonichost.is_multi_asic: + buffer_queues = list(data['BUFFER_QUEUE'].keys()) + iface_to_check = buffer_queues[0].split('|')[0] + iface_buffer_queues = [bq for bq in buffer_queues if any(val in iface_to_check for val in bq.split('|'))] + for queue in iface_buffer_queues: + if asic.namespace in queue and queue.split('|')[-1] == '3-4' and queue.split('|')[-2] == interface: + buffer_queue_to_del = queue + break + else: + buffer_queue_to_del = "{}|3-4".format(interface) # Add create_only_config_db_buffers entry to device metadata to enable # counters optimization and get number of queue counters of Ethernet0 prior @@ -63,16 +120,26 @@ def test_snmp_queue_counters(duthosts, # Remove buffer queue and reload and get number of queue counters of # Ethernet0 after removing two buffer queues - del data['BUFFER_QUEUE']["Ethernet0|3-4"] + del data['BUFFER_QUEUE'][buffer_queue_to_del] load_new_cfg(duthost, data) queue_counters_cnt_post = get_queue_ctrs(duthost, get_bfr_queue_cntrs_cmd) - unicast_expected_diff = BUFFER_QUEUES_REMOVED * UNICAST_CTRS - multicast_expected_diff = unicast_expected_diff + (BUFFER_QUEUES_REMOVED - * MULTICAST_CTRS) - pytest_assert((queue_counters_cnt_pre - queue_counters_cnt_post) - in [unicast_expected_diff, multicast_expected_diff], - "Queue counters count differs from expected") + # For broadcom-dnx voq chassis, number of voq are fixed (static), which cannot be modified dynamically + # Hence, make sure the queue counters before deletion and after deletion are same for broadcom-dnx voq chassis + if duthost.facts.get("platform_asic") == "broadcom-dnx" and duthost.sonichost.is_multi_asic: + pytest_assert((queue_counters_cnt_pre == queue_counters_cnt_post), + "Queue counters actual count {} differs from expected values {}". + format(queue_counters_cnt_post, queue_counters_cnt_pre)) + # check for other duts + else: + unicast_expected_diff = BUFFER_QUEUES_REMOVED * UNICAST_CTRS + multicast_expected_diff = unicast_expected_diff + (BUFFER_QUEUES_REMOVED + * MULTICAST_CTRS) + pytest_assert((queue_counters_cnt_pre - queue_counters_cnt_post) + in [unicast_expected_diff, multicast_expected_diff], + "Queue counters actual count {} differs from expected values {}, {}". + format(queue_counters_cnt_post, (queue_counters_cnt_pre - unicast_expected_diff), + (queue_counters_cnt_pre - multicast_expected_diff))) @pytest.fixture(scope="module") @@ -83,5 +150,5 @@ def teardown(duthost): """ yield # Cleanup - duthost.copy(src=ORIG_CFG_DB, dest=CFG_DB_PATH) + duthost.copy(src=ORIG_CFG_DB, dest=CFG_DB_PATH, remote_src=True) config_reload(duthost, config_source='config_db', safe_reload=True) diff --git a/tests/span/span_helpers.py b/tests/span/span_helpers.py index a85c2b8d30..28dc2f351b 100644 --- a/tests/span/span_helpers.py +++ b/tests/span/span_helpers.py @@ -5,7 +5,7 @@ import ptf.testutils as testutils -def send_and_verify_mirrored_packet(ptfadapter, src_port, monitor): +def send_and_verify_mirrored_packet(ptfadapter, src_port, monitor, skip_traffic_test=False): ''' Send packet from ptf and verify it on monitor port @@ -18,6 +18,8 @@ def send_and_verify_mirrored_packet(ptfadapter, src_port, monitor): pkt = testutils.simple_icmp_packet(eth_src=src_mac, eth_dst='ff:ff:ff:ff:ff:ff') + if skip_traffic_test is True: + return ptfadapter.dataplane.flush() testutils.send(ptfadapter, src_port, pkt) testutils.verify_packet(ptfadapter, pkt, monitor) diff --git a/tests/span/test_port_mirroring.py b/tests/span/test_port_mirroring.py index 3fcc024b81..646f699264 100644 --- a/tests/span/test_port_mirroring.py +++ b/tests/span/test_port_mirroring.py @@ -4,7 +4,8 @@ import pytest -from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from span_helpers import send_and_verify_mirrored_packet pytestmark = [ @@ -12,7 +13,7 @@ ] -def test_mirroring_rx(ptfadapter, setup_session): +def test_mirroring_rx(ptfadapter, setup_session, skip_traffic_test): # noqa F811 ''' Test case #1 Verify ingress direction session @@ -26,10 +27,11 @@ def test_mirroring_rx(ptfadapter, setup_session): ''' send_and_verify_mirrored_packet(ptfadapter, setup_session['source1_index'], - setup_session['destination_index']) + setup_session['destination_index'], + skip_traffic_test=skip_traffic_test) -def test_mirroring_tx(ptfadapter, setup_session): +def test_mirroring_tx(ptfadapter, setup_session, skip_traffic_test): # noqa F811 ''' Test case #2 Verify egress direction session @@ -43,10 +45,11 @@ def test_mirroring_tx(ptfadapter, setup_session): ''' send_and_verify_mirrored_packet(ptfadapter, setup_session['source2_index'], - setup_session['destination_index']) + setup_session['destination_index'], + skip_traffic_test=skip_traffic_test) -def test_mirroring_both(ptfadapter, setup_session): +def test_mirroring_both(ptfadapter, setup_session, skip_traffic_test): # noqa F811 ''' Test case #3 Verify bidirectional session @@ -63,14 +66,16 @@ def test_mirroring_both(ptfadapter, setup_session): ''' send_and_verify_mirrored_packet(ptfadapter, setup_session['source1_index'], - setup_session['destination_index']) + setup_session['destination_index'], + skip_traffic_test=skip_traffic_test) send_and_verify_mirrored_packet(ptfadapter, setup_session['source2_index'], - setup_session['destination_index']) + setup_session['destination_index'], + skip_traffic_test=skip_traffic_test) -def test_mirroring_multiple_source(ptfadapter, setup_session): +def test_mirroring_multiple_source(ptfadapter, setup_session, skip_traffic_test): # noqa F811 ''' Test case #4 Verify ingress direction session with multiple source ports @@ -87,8 +92,10 @@ def test_mirroring_multiple_source(ptfadapter, setup_session): ''' send_and_verify_mirrored_packet(ptfadapter, setup_session['source1_index'], - setup_session['destination_index']) + setup_session['destination_index'], + skip_traffic_test=skip_traffic_test) send_and_verify_mirrored_packet(ptfadapter, setup_session['source2_index'], - setup_session['destination_index']) + setup_session['destination_index'], + skip_traffic_test=skip_traffic_test) diff --git a/tests/ssh/test_ssh_limit.py b/tests/ssh/test_ssh_limit.py index 9c4734fa15..4f97dfab81 100755 --- a/tests/ssh/test_ssh_limit.py +++ b/tests/ssh/test_ssh_limit.py @@ -3,9 +3,10 @@ import pytest import time from tests.common.helpers.assertions import pytest_assert, pytest_require -from tests.tacacs.conftest import tacacs_creds # noqa F401 -from tests.tacacs.utils import setup_local_user +from tests.common.fixtures.tacacs import tacacs_creds # noqa F401 +from tests.common.helpers.tacacs.tacacs_helper import setup_local_user from tests.common.utilities import paramiko_ssh +from tests.common.fixtures.tacacs import get_aaa_sub_options_value pytestmark = [ pytest.mark.disable_loganalyzer, @@ -43,17 +44,19 @@ def get_device_type(duthost): return dut_type -def modify_template(admin_session, template_path, additional_content, hwsku, type): - admin_session.exec_command(TEMPLATE_BACKUP_COMMAND.format(template_path)) - admin_session.exec_command(TEMPLATE_CREATE_COMMAND.format(template_path)) - admin_session.exec_command( - LIMITS_CONF_TEMPLATE_TO_HOME.format(hwsku, type, additional_content)) - admin_session.exec_command(TEMPLATE_MOVE_COMMAND.format(template_path)) +def exec_command(admin_session, command): + stdin, stdout, stderr = admin_session.exec_command(command) + outstr = stdout.readlines() + errstr = stderr.readlines() + logging.info("Command: '{}' stdout: {} stderr: {}".format(command, outstr, errstr)) + - stdin, stdout, stderr = admin_session.exec_command( - 'sudo cat {0}'.format(template_path)) - config_file_content = stdout.readlines() - logging.info("Updated template file: {0}".format(config_file_content)) +def modify_template(admin_session, template_path, additional_content, hwsku, type): + exec_command(admin_session, TEMPLATE_BACKUP_COMMAND.format(template_path)) + exec_command(admin_session, TEMPLATE_CREATE_COMMAND.format(template_path)) + exec_command(admin_session, LIMITS_CONF_TEMPLATE_TO_HOME.format(hwsku, type, additional_content)) + exec_command(admin_session, TEMPLATE_MOVE_COMMAND.format(template_path)) + exec_command(admin_session, 'sudo cat {0}'.format(template_path)) def modify_templates(duthost, tacacs_creds, creds): # noqa F811 @@ -101,7 +104,13 @@ def setup_limit(duthosts, rand_one_dut_hostname, tacacs_creds, creds): # no # if template file not exist on duthost, ignore this UT # However still need yield, if not yield, UT will failed with StopIteration error. template_file_exist = limit_template_exist(duthost) + aaa_login_disabled = False if template_file_exist: + # If AAA authentication enabled, disable it to allow local user login + if get_aaa_sub_options_value(duthost, "authentication", "login") == "tacacs+": + duthost.shell("sudo config aaa authentication login default") + aaa_login_disabled = True + setup_local_user(duthost, tacacs_creds) # Modify templates and restart hostcfgd to render config files @@ -111,6 +120,9 @@ def setup_limit(duthosts, rand_one_dut_hostname, tacacs_creds, creds): # no yield if template_file_exist: + if aaa_login_disabled: + duthost.shell("sudo config aaa authentication login tacacs+") + # Restore SSH session limit restore_templates(duthost) restart_hostcfgd(duthost) diff --git a/tests/stress/conftest.py b/tests/stress/conftest.py index b392113c8b..e430d452cb 100644 --- a/tests/stress/conftest.py +++ b/tests/stress/conftest.py @@ -1,9 +1,11 @@ import logging -import pytest import time -from tests.common.utilities import wait_until -from utils import get_crm_resources, check_queue_status, sleep_to_wait + +import pytest + from tests.common import config_reload +from tests.common.utilities import wait_until +from utils import get_crm_resource_status, check_queue_status, sleep_to_wait CRM_POLLING_INTERVAL = 1 CRM_DEFAULT_POLL_INTERVAL = 300 @@ -13,12 +15,13 @@ @pytest.fixture(scope='module') -def get_function_conpleteness_level(pytestconfig): +def get_function_completeness_level(pytestconfig): return pytestconfig.getoption("--completeness_level") @pytest.fixture(scope="module", autouse=True) -def set_polling_interval(duthost): +def set_polling_interval(duthosts, enum_rand_one_per_hwsku_frontend_hostname): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] wait_time = 2 duthost.command("crm config polling interval {}".format(CRM_POLLING_INTERVAL)) logger.info("Waiting {} sec for CRM counters to become updated".format(wait_time)) @@ -31,8 +34,23 @@ def set_polling_interval(duthost): time.sleep(wait_time) +@pytest.fixture(scope="module") +def cleanup_neighbors_dualtor(duthosts, ptfhost, tbinfo): + """Cleanup neighbors on dualtor testbed.""" + if "dualtor" in tbinfo["topo"]["name"]: + ptfhost.shell("supervisorctl stop garp_service", module_ignore_errors=True) + ptfhost.shell("supervisorctl stop arp_responder", module_ignore_errors=True) + duthosts.shell("sonic-clear arp") + duthosts.shell("sonic-clear ndp") + + @pytest.fixture(scope='module') -def withdraw_and_announce_existing_routes(duthost, localhost, tbinfo): +def withdraw_and_announce_existing_routes(duthosts, localhost, tbinfo, enum_rand_one_per_hwsku_frontend_hostname, + enum_rand_one_frontend_asic_index, cleanup_neighbors_dualtor): # noqa F811 + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + asichost = duthost.asic_instance(enum_rand_one_frontend_asic_index) + namespace = asichost.namespace + ptf_ip = tbinfo["ptf_ip"] topo_name = tbinfo["topo"]["name"] @@ -41,8 +59,8 @@ def withdraw_and_announce_existing_routes(duthost, localhost, tbinfo): wait_until(MAX_WAIT_TIME, CRM_POLLING_INTERVAL, 0, lambda: check_queue_status(duthost, "inq") is True) sleep_to_wait(CRM_POLLING_INTERVAL * 100) - ipv4_route_used_before = get_crm_resources(duthost, "ipv4_route", "used") - ipv6_route_used_before = get_crm_resources(duthost, "ipv6_route", "used") + ipv4_route_used_before = get_crm_resource_status(duthost, "ipv4_route", "used", namespace) + ipv6_route_used_before = get_crm_resource_status(duthost, "ipv6_route", "used", namespace) logger.info("ipv4 route used {}".format(ipv4_route_used_before)) logger.info("ipv6 route used {}".format(ipv6_route_used_before)) @@ -53,12 +71,13 @@ def withdraw_and_announce_existing_routes(duthost, localhost, tbinfo): wait_until(MAX_WAIT_TIME, CRM_POLLING_INTERVAL, 0, lambda: check_queue_status(duthost, "outq") is True) sleep_to_wait(CRM_POLLING_INTERVAL * 5) - logger.info("ipv4 route used {}".format(get_crm_resources(duthost, "ipv4_route", "used"))) - logger.info("ipv6 route used {}".format(get_crm_resources(duthost, "ipv6_route", "used"))) + logger.info("ipv4 route used {}".format(get_crm_resource_status(duthost, "ipv4_route", "used", namespace))) + logger.info("ipv6 route used {}".format(get_crm_resource_status(duthost, "ipv6_route", "used", namespace))) @pytest.fixture(scope="module", autouse=True) -def check_system_memmory(duthost): +def check_system_memmory(duthosts, enum_rand_one_per_hwsku_frontend_hostname): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] for index in range(1, 4): cmd = 'echo {} > /proc/sys/vm/drop_caches'.format(index) duthost.shell(cmd, module_ignore_errors=True) diff --git a/tests/stress/test_stress_routes.py b/tests/stress/test_stress_routes.py index cb1fbf75ba..78ddbb6e09 100644 --- a/tests/stress/test_stress_routes.py +++ b/tests/stress/test_stress_routes.py @@ -1,11 +1,12 @@ #!/usr/bin/env python import logging + import pytest from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until -from utils import get_crm_resources, check_queue_status, sleep_to_wait, LOOP_TIMES_LEVEL_MAP +from utils import get_crm_resource_status, check_queue_status, sleep_to_wait, LOOP_TIMES_LEVEL_MAP ALLOW_ROUTES_CHANGE_NUMS = 5 CRM_POLLING_INTERVAL = 1 @@ -14,70 +15,75 @@ logger = logging.getLogger(__name__) pytestmark = [ - pytest.mark.topology('t0', 't1', 'm0', 'mx') + pytest.mark.topology('t0', 't1', 'm0', 'mx', 't2') ] -def announce_withdraw_routes(duthost, localhost, ptf_ip, topo_name): +def announce_withdraw_routes(duthost, namespace, localhost, ptf_ip, topo_name): logger.info("announce ipv4 and ipv6 routes") - localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="announce", path="../ansible/") + localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="announce", path="../ansible/", + log_path="logs") wait_until(MAX_WAIT_TIME, CRM_POLLING_INTERVAL, 0, lambda: check_queue_status(duthost, "outq") is True) - logger.info("ipv4 route used {}".format(get_crm_resources(duthost, "ipv4_route", "used"))) - logger.info("ipv6 route used {}".format(get_crm_resources(duthost, "ipv6_route", "used"))) + logger.info("ipv4 route used {}".format(get_crm_resource_status(duthost, "ipv4_route", "used", namespace))) + logger.info("ipv6 route used {}".format(get_crm_resource_status(duthost, "ipv6_route", "used", namespace))) sleep_to_wait(CRM_POLLING_INTERVAL * 5) logger.info("withdraw ipv4 and ipv6 routes") - localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="withdraw", path="../ansible/") + localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="withdraw", path="../ansible/", + log_path="logs") wait_until(MAX_WAIT_TIME, CRM_POLLING_INTERVAL, 0, lambda: check_queue_status(duthost, "inq") is True) sleep_to_wait(CRM_POLLING_INTERVAL * 5) - logger.info("ipv4 route used {}".format(get_crm_resources(duthost, "ipv4_route", "used"))) - logger.info("ipv6 route used {}".format(get_crm_resources(duthost, "ipv6_route", "used"))) + logger.info("ipv4 route used {}".format(get_crm_resource_status(duthost, "ipv4_route", "used", namespace))) + logger.info("ipv6 route used {}".format(get_crm_resource_status(duthost, "ipv6_route", "used", namespace))) -def test_announce_withdraw_route(duthosts, localhost, tbinfo, get_function_conpleteness_level, +def test_announce_withdraw_route(duthosts, localhost, tbinfo, get_function_completeness_level, withdraw_and_announce_existing_routes, loganalyzer, - enum_rand_one_per_hwsku_frontend_hostname): + enum_rand_one_per_hwsku_frontend_hostname, enum_rand_one_frontend_asic_index, + rotate_syslog): ptf_ip = tbinfo["ptf_ip"] topo_name = tbinfo["topo"]["name"] duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] - - if loganalyzer: - ignoreRegex = [ - ".*ERR route_check.py:.*", - ".*ERR.* 'routeCheck' status failed.*", - ".*Process \'orchagent\' is stuck in namespace \'host\'.*", - ".*ERR rsyslogd: .*" - ] - - hwsku = duthost.facts['hwsku'] - if hwsku in ['Arista-7050-QX-32S', 'Arista-7050QX32S-Q32', 'Arista-7050-QX32', 'Arista-7050QX-32S-S4Q31']: - ignoreRegex.append(".*ERR memory_threshold_check:.*") - ignoreRegex.append(".*ERR monit.*memory_check.*") - ignoreRegex.append(".*ERR monit.*mem usage of.*matches resource limit.*") - - # Ignore errors in ignoreRegex for *all* DUTs - for dut in duthosts: + asichost = duthost.asic_instance(enum_rand_one_frontend_asic_index) + namespace = asichost.namespace + + ignoreRegex = [ + ".*ERR route_check.py:.*", + ".*ERR.* 'routeCheck' status failed.*", + ".*Process \'orchagent\' is stuck in namespace \'host\'.*", + ".*ERR rsyslogd: .*" + ] + + hwsku = duthost.facts['hwsku'] + if hwsku in ['Arista-7050-QX-32S', 'Arista-7050QX32S-Q32', 'Arista-7050-QX32', 'Arista-7050QX-32S-S4Q31']: + ignoreRegex.append(".*ERR memory_threshold_check:.*") + ignoreRegex.append(".*ERR monit.*memory_check.*") + ignoreRegex.append(".*ERR monit.*mem usage of.*matches resource limit.*") + + # Ignore errors in ignoreRegex for *all* DUTs + for dut in duthosts.frontend_nodes: + if dut.loganalyzer: loganalyzer[dut.hostname].ignore_regex.extend(ignoreRegex) - normalized_level = get_function_conpleteness_level + normalized_level = get_function_completeness_level if normalized_level is None: - normalized_level = "basic" + normalized_level = "debug" ipv4_route_used_before, ipv6_route_used_before = withdraw_and_announce_existing_routes loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level] while loop_times > 0: - announce_withdraw_routes(duthost, localhost, ptf_ip, topo_name) + announce_withdraw_routes(duthost, namespace, localhost, ptf_ip, topo_name) loop_times -= 1 sleep_to_wait(CRM_POLLING_INTERVAL * 120) - ipv4_route_used_after = get_crm_resources(duthost, "ipv4_route", "used") - ipv6_route_used_after = get_crm_resources(duthost, "ipv6_route", "used") + ipv4_route_used_after = get_crm_resource_status(duthost, "ipv4_route", "used", namespace) + ipv6_route_used_after = get_crm_resource_status(duthost, "ipv6_route", "used", namespace) pytest_assert(abs(ipv4_route_used_after - ipv4_route_used_before) < ALLOW_ROUTES_CHANGE_NUMS, "ipv4 route used after is not equal to it used before") diff --git a/tests/stress/utils.py b/tests/stress/utils.py index d899b303a1..dc02e5fbbc 100644 --- a/tests/stress/utils.py +++ b/tests/stress/utils.py @@ -1,6 +1,8 @@ import re import time +from tests.common.helpers.constants import DEFAULT_NAMESPACE + TOPO_FILENAME_TEMPLATE = 'topo_{}.yml' SHOW_BGP_SUMMARY_CMD = "show ip bgp summary" LOOP_TIMES_LEVEL_MAP = { @@ -12,15 +14,19 @@ } -def get_crm_resources(duthost, resource, status): - return duthost.get_crm_resources().get("main_resources").get(resource).get(status) +def get_crm_resource_status(duthost, resource, status, namespace=DEFAULT_NAMESPACE): + return duthost.get_crm_resources(namespace).get("main_resources").get(resource).get(status) def check_queue_status(duthost, queue): bgp_neighbors = duthost.show_and_parse(SHOW_BGP_SUMMARY_CMD) bgp_neighbor_addr_regex = re.compile(r"^([0-9]{1,3}\.){3}[0-9]{1,3}") for neighbor in bgp_neighbors: - if bgp_neighbor_addr_regex.match(neighbor["neighbhor"]) and int(neighbor[queue]) != 0: + if "neighbhor" in neighbor: + neigh = neighbor["neighbhor"] + else: + neigh = neighbor["neighbor"] + if bgp_neighbor_addr_regex.match(neigh) and int(neighbor[queue]) != 0: return False return True diff --git a/tests/sub_port_interfaces/sub_ports_helpers.py b/tests/sub_port_interfaces/sub_ports_helpers.py index 3732e60c9c..f92730b54b 100644 --- a/tests/sub_port_interfaces/sub_ports_helpers.py +++ b/tests/sub_port_interfaces/sub_ports_helpers.py @@ -2,6 +2,7 @@ import time import random import ipaddress +import logging from collections import OrderedDict @@ -20,6 +21,7 @@ from tests.common.pkt_filter.filter_pkt_in_buffer import FilterPktBuffer from tests.common import constants +logger = logging.getLogger(__name__) BASE_DIR = os.path.dirname(os.path.realpath(__file__)) DUT_TMP_DIR = os.path.join('tmp', os.path.basename(BASE_DIR)) @@ -84,7 +86,7 @@ def create_packet(eth_dst, eth_src, ip_dst, ip_src, vlan_vid, tr_type, ttl, dl_v def generate_and_verify_traffic(duthost, ptfadapter, src_port, dst_port, ptfhost=None, ip_src='', ip_dst='', pkt_action=None, type_of_traffic='ICMP', ttl=64, pktlen=100, ip_tunnel=None, - **kwargs): + skip_traffic_test=False, **kwargs): """ Send packet from PTF to DUT and verify that DUT sends/doesn't packet to PTF. @@ -103,6 +105,9 @@ def generate_and_verify_traffic(duthost, ptfadapter, src_port, dst_port, ptfhost pktlen: packet length ip_tunnel: Tunnel IP address of DUT """ + if skip_traffic_test is True: + logger.info("Skipping traffic test") + return type_of_traffic = [type_of_traffic] if not isinstance(type_of_traffic, list) else type_of_traffic for tr_type in type_of_traffic: diff --git a/tests/sub_port_interfaces/test_sub_port_interfaces.py b/tests/sub_port_interfaces/test_sub_port_interfaces.py index 1818fc805e..c3bba0e73d 100644 --- a/tests/sub_port_interfaces/test_sub_port_interfaces.py +++ b/tests/sub_port_interfaces/test_sub_port_interfaces.py @@ -15,7 +15,7 @@ from sub_ports_helpers import check_sub_port from sub_ports_helpers import remove_sub_port from sub_ports_helpers import create_sub_port_on_dut - +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 pytestmark = [ pytest.mark.topology('t0', 't1') @@ -346,7 +346,8 @@ def test_routing_between_sub_ports_and_port(self, request, type_of_traffic, duth ttl=63, pktlen=pktlen) - def test_tunneling_between_sub_ports(self, duthost, ptfadapter, apply_tunnel_table_to_dut, apply_route_config): + def test_tunneling_between_sub_ports(self, duthost, ptfadapter, apply_tunnel_table_to_dut, + apply_route_config, skip_traffic_test): # noqa F811 """ Validates that packets are routed between sub-ports. @@ -379,9 +380,11 @@ def test_tunneling_between_sub_ports(self, duthost, ptfadapter, apply_tunnel_tab ip_tunnel=sub_ports[src_port]['ip'], pkt_action='fwd', type_of_traffic='decap', - ttl=63) + ttl=63, + skip_traffic_test=skip_traffic_test) - def test_balancing_sub_ports(self, duthost, ptfhost, ptfadapter, apply_balancing_config): + def test_balancing_sub_ports(self, duthost, ptfhost, ptfadapter, + apply_balancing_config, skip_traffic_test): # noqa F811 """ Validates load-balancing when sub-port is part of ECMP Test steps: @@ -414,12 +417,13 @@ def test_balancing_sub_ports(self, duthost, ptfhost, ptfadapter, apply_balancing dst_port=dst_ports, ip_dst=ip_dst, type_of_traffic='balancing', - ttl=63) + ttl=63, + skip_traffic_test=skip_traffic_test) class TestSubPortsNegative(object): def test_packet_routed_with_invalid_vlan(self, duthost, ptfadapter, apply_config_on_the_dut, - apply_config_on_the_ptf): + apply_config_on_the_ptf, skip_traffic_test): # noqa F811 """ Validates that packet aren't routed if sub-ports have invalid VLAN ID. @@ -443,11 +447,13 @@ def test_packet_routed_with_invalid_vlan(self, duthost, ptfadapter, apply_config ip_src=value['neighbor_ip'], dst_port=sub_port, ip_dst=value['ip'], - pkt_action='drop') + pkt_action='drop', + skip_traffic_test=skip_traffic_test) class TestSubPortStress(object): - def test_max_numbers_of_sub_ports(self, duthost, ptfadapter, apply_config_on_the_dut, apply_config_on_the_ptf): + def test_max_numbers_of_sub_ports(self, duthost, ptfadapter, apply_config_on_the_dut, + apply_config_on_the_ptf, skip_traffic_test): # noqa F811 """ Validates that 256 sub-ports can be created per port or LAG @@ -480,4 +486,5 @@ def test_max_numbers_of_sub_ports(self, duthost, ptfadapter, apply_config_on_the ip_src=value['neighbor_ip'], dst_port=sub_port, ip_dst=value['ip'], - pkt_action='fwd') + pkt_action='fwd', + skip_traffic_test=skip_traffic_test) diff --git a/tests/syslog/syslog_utils.py b/tests/syslog/syslog_utils.py index d173c591ec..08e254b579 100644 --- a/tests/syslog/syslog_utils.py +++ b/tests/syslog/syslog_utils.py @@ -144,15 +144,3 @@ def capture_syslog_packets(dut, tcpdump_cmd, logger_flags='', logger_msg=''): dut.fetch(src=pcap_file_full_path, dest=DOCKER_TMP_PATH) filepath = os.path.join(DOCKER_TMP_PATH, dut.hostname, pcap_file_full_path.lstrip(os.path.sep)) return filepath - - -def is_mgmt_vrf_enabled(dut): - """ - Check mgmt vrf is enabled or not - - Args: - dut (SonicHost): The target device - Return: True or False - """ - show_mgmt_vrf = dut.command("show mgmt-vrf")["stdout"] - return "ManagementVRF : Disabled" not in show_mgmt_vrf diff --git a/tests/syslog/test_logrotate.py b/tests/syslog/test_logrotate.py index 9b309452ab..7914536826 100644 --- a/tests/syslog/test_logrotate.py +++ b/tests/syslog/test_logrotate.py @@ -5,6 +5,7 @@ from tests.common.plugins.loganalyzer.loganalyzer import DisableLogrotateCronContext from tests.common import config_reload from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import wait_until logger = logging.getLogger(__name__) @@ -40,9 +41,6 @@ def backup_syslog(rand_selected_dut): logger.info('Recover syslog file to syslog') duthost.shell('sudo mv /var/log/syslog_bk /var/log/syslog') - logger.info('Remove temp file /var/log/syslog.1') - duthost.shell('sudo rm -f /var/log/syslog.1') - logger.info('Restart rsyslog service') duthost.shell('sudo service rsyslog restart') @@ -140,7 +138,7 @@ def multiply_with_unit(logrotate_threshold, num): return str(int(logrotate_threshold[:-1]) * num) + logrotate_threshold[-1] -def validate_logrotate_function(duthost, logrotate_threshold): +def validate_logrotate_function(duthost, logrotate_threshold, small_size): """ Validate logrotate function :param duthost: DUT host object @@ -153,7 +151,10 @@ def validate_logrotate_function(duthost, logrotate_threshold): logrotate_threshold)): syslog_number_origin = get_syslog_file_count(duthost) logger.info('There are {} syslog gz files'.format(syslog_number_origin)) - create_temp_syslog_file(duthost, multiply_with_unit(logrotate_threshold, 0.9)) + if small_size: + create_temp_syslog_file(duthost, multiply_with_unit(logrotate_threshold, 0.5)) + else: + create_temp_syslog_file(duthost, multiply_with_unit(logrotate_threshold, 0.9)) run_logrotate(duthost) syslog_number_no_rotate = get_syslog_file_count(duthost) logger.info('There are {} syslog gz files after running logrotate'.format(syslog_number_no_rotate)) @@ -207,7 +208,7 @@ def test_logrotate_normal_size(rand_selected_dut): if get_var_log_size(duthost) < 200 * 1024: pytest.skip('{} size is lower than 200MB, skip this test'.format(LOG_FOLDER)) rotate_large_threshold = get_threshold_based_on_memory(duthost) - validate_logrotate_function(duthost, rotate_large_threshold) + validate_logrotate_function(duthost, rotate_large_threshold, False) @pytest.mark.disable_loganalyzer @@ -219,7 +220,7 @@ def test_logrotate_small_size(rand_selected_dut, simulate_small_var_log_partitio Execute config reload to active the mount Stop logrotate cron job, make sure no logrotate executes during this test Check current syslog.x file number and save it - Create a temp file with size of rotate_size * 90%, and rename it as 'syslog', run logrotate command + Create a temp file with size of rotate_size * 50%, and rename it as 'syslog', run logrotate command There would be no logrotate happens - by checking the 'syslog.x' file number not increased Create a temp file with size of rotate_size * 110%, and rename it as 'syslog', run logrotate command There would be logrotate happens - by checking the 'syslog.x' file number increased by 1 @@ -230,11 +231,10 @@ def test_logrotate_small_size(rand_selected_dut, simulate_small_var_log_partitio """ duthost = rand_selected_dut rotate_small_threshold = get_threshold_based_on_memory(duthost) - validate_logrotate_function(duthost, rotate_small_threshold) + validate_logrotate_function(duthost, rotate_small_threshold, True) def get_pending_entries(duthost, ignore_list=None): - # grep returns error code when there is no match, add 'true' so the ansible module doesn't fail pending_entries = set(duthost.shell('sonic-db-cli APPL_DB keys "_*"')['stdout'].split()) if ignore_list: @@ -243,7 +243,9 @@ def get_pending_entries(duthost, ignore_list=None): pending_entries.remove(entry) except ValueError: continue - return list(pending_entries) + pending_entries = list(pending_entries) + logger.info('Pending entries in APPL_DB: {}'.format(pending_entries)) + return pending_entries def clear_pending_entries(duthost): @@ -254,34 +256,63 @@ def clear_pending_entries(duthost): duthost.shell('sonic-db-cli APPL_DB publish "NEIGH_TABLE_CHANNEL" ""') +def no_pending_entries(duthost, ignore_list=None): + return not bool(get_pending_entries(duthost, ignore_list=ignore_list)) + + @pytest.fixture -def orch_logrotate_setup(rand_selected_dut): - clear_pending_entries(rand_selected_dut) - rand_selected_dut.shell('sudo ip neigh flush {}'.format(FAKE_IP)) - target_port = rand_selected_dut.get_up_ip_ports()[0] +def orch_logrotate_setup(duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo, + enum_rand_one_frontend_asic_index): + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + if duthost.sonichost.is_multi_asic: + asic_id = enum_rand_one_frontend_asic_index + else: + asic_id = '' + clear_pending_entries(duthost) + duthost.shell('sudo ip neigh flush {}'.format(FAKE_IP)) + if duthost.sonichost.is_multi_asic: + target_asic = duthost.asics[enum_rand_one_frontend_asic_index] + target_port = next(iter(target_asic.get_active_ip_interfaces(tbinfo))) + else: + target_port = duthost.get_up_ip_ports()[0] - permanent_pending_entries = get_pending_entries(rand_selected_dut) + permanent_pending_entries = get_pending_entries(duthost) yield permanent_pending_entries, target_port - rand_selected_dut.shell('sudo ip neigh del {} dev {}'.format(FAKE_IP, target_port)) + if duthost.sonichost.is_multi_asic: + duthost.shell('sudo ip -n asic{} neigh del {} dev {}'.format(asic_id, FAKE_IP, target_port)) + else: + duthost.shell('sudo ip neigh del {} dev {}'.format(FAKE_IP, target_port)) # Unpause orchagent in case the test gets interrupted - rand_selected_dut.control_process('orchagent', pause=False) - clear_pending_entries(rand_selected_dut) + duthost.control_process('orchagent', pause=False, namespace=asic_id) + clear_pending_entries(duthost) # Sometimes other activity on the DUT can flush the missed notification during the test, # leading to a false positive pass. Repeat the test multiple times to make sure that it's # not a false positive @pytest.mark.repeat(5) -def test_orchagent_logrotate(orch_logrotate_setup, rand_selected_dut): +def test_orchagent_logrotate(orch_logrotate_setup, duthosts, enum_rand_one_per_hwsku_frontend_hostname, + enum_rand_one_frontend_asic_index): """ Tests for the issue where an orchagent logrotate can cause a missed APPL_DB notification """ + duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] + if duthost.sonichost.is_multi_asic: + asic_id = enum_rand_one_frontend_asic_index + else: + asic_id = '' ignore_entries, target_port = orch_logrotate_setup - rand_selected_dut.control_process('orchagent', pause=True) - rand_selected_dut.control_process('orchagent', signal='SIGHUP') - rand_selected_dut.shell('sudo ip neigh add {} lladdr {} dev {}'.format(FAKE_IP, FAKE_MAC, target_port)) - rand_selected_dut.control_process('orchagent', pause=False) - pending_entries = get_pending_entries(rand_selected_dut, ignore_list=ignore_entries) - pytest_assert(not pending_entries, "Found pending entries in APPL_DB: {}".format(pending_entries)) + duthost.control_process('orchagent', pause=True, namespace=asic_id) + duthost.control_process('orchagent', namespace=asic_id, signal='SIGHUP') + if duthost.sonichost.is_multi_asic: + duthost.shell('sudo ip -n asic{} neigh add {} lladdr {} dev {}'.format( + asic_id, FAKE_IP, FAKE_MAC, target_port)) + else: + duthost.shell('sudo ip neigh add {} lladdr {} dev {}'.format(FAKE_IP, FAKE_MAC, target_port)) + duthost.control_process('orchagent', pause=False, namespace=asic_id) + pytest_assert( + wait_until(30, 1, 0, no_pending_entries, duthost, ignore_list=ignore_entries), + "Found pending entries in APPL_DB" + ) diff --git a/tests/syslog/test_syslog.py b/tests/syslog/test_syslog.py index 7189c46822..c61156f3c3 100644 --- a/tests/syslog/test_syslog.py +++ b/tests/syslog/test_syslog.py @@ -1,9 +1,6 @@ import logging import pytest -import os -import time - -from scapy.all import rdpcap +from tests.common.helpers.syslog_helpers import run_syslog, check_default_route # noqa F401 logger = logging.getLogger(__name__) @@ -11,181 +8,6 @@ pytest.mark.topology("any") ] -DUT_PCAP_FILEPATH = "/tmp/test_syslog_tcpdump.pcap" -DOCKER_TMP_PATH = "/tmp/" - - -# If any dummy IP type doesn't have a matching default route, skip test for this parametrize -def check_dummy_addr_and_default_route(dummy_ip_a, dummy_ip_b, has_v4_default_route, has_v6_default_route): - skip_v4 = False - skip_v6 = False - - if dummy_ip_a is not None and ":" not in dummy_ip_a and not has_v4_default_route: - skip_v4 = True - if dummy_ip_a is not None and ":" in dummy_ip_a and not has_v6_default_route: - skip_v6 = True - - if dummy_ip_b is not None and ":" not in dummy_ip_b and not has_v4_default_route: - skip_v4 = True - if dummy_ip_b is not None and ":" in dummy_ip_b and not has_v6_default_route: - skip_v6 = True - - if skip_v4 | skip_v6: - proto = "IPv4" if skip_v4 else "IPv6" - pytest.skip("DUT has no matching default route for dummy syslog ips: ({}, {}), has no {} default route" - .format(dummy_ip_a, dummy_ip_b, proto)) - - -# Check pcap file for the destination IPs -def _check_pcap(dummy_ip_a, dummy_ip_b, filepath): - is_ok_a = False - is_ok_b = False - - if dummy_ip_a is None: - is_ok_a = True - if dummy_ip_b is None: - is_ok_b = True - - packets = rdpcap(filepath) - for data in packets: - proto = "IPv6" if "IPv6" in data else "IP" - if is_ok_a is False and data[proto].dst == dummy_ip_a: - is_ok_a = True - if is_ok_b is False and data[proto].dst == dummy_ip_b: - is_ok_b = True - if is_ok_a and is_ok_b: - return True - - missed_ip = [] - if not is_ok_a: - missed_ip.append(dummy_ip_a) - if not is_ok_b: - missed_ip.append(dummy_ip_b) - logger.error("Pcap file doesn't contain dummy syslog ips: ({})".format(", ".join(missed_ip))) - return False - - -# Before real test, check default route on DUT: -# If DUT has no IPv4 and IPv6 default route, skip syslog test. If DUT has at least one type default route, -# tell test_syslog function to do further check -@pytest.fixture(scope="module") -def check_default_route(rand_selected_dut): - duthost = rand_selected_dut - ret = {'IPv4': False, 'IPv6': False} - - logger.info("Checking DUT default route") - result = duthost.shell("ip route show default table default | grep via", module_ignore_errors=True)['rc'] - if result == 0: - neigh_ip = duthost.shell( - "ip route show default table default | cut -d ' ' -f 3", module_ignore_errors=True)['stdout'] - result = duthost.shell( - "ip -4 neigh show {} | grep REACHABLE".format(neigh_ip), module_ignore_errors=True)['rc'] - if result == 0: - ret['IPv4'] = True - result = duthost.shell("ip -6 route show default table default | grep via", module_ignore_errors=True)['rc'] - if result == 0: - neigh_ip = duthost.shell( - "ip -6 route show default table default | cut -d ' ' -f 3", module_ignore_errors=True)['stdout'] - result = duthost.shell( - "ip -6 neigh show {} | grep REACHABLE".format(neigh_ip), module_ignore_errors=True)['rc'] - if result == 0: - ret['IPv6'] = True - - if not ret['IPv4'] and not ret['IPv6']: - pytest.skip("DUT has no default route, skiped") - - yield ret - - -def run_syslog(rand_selected_dut, dummy_syslog_server_ip_a, dummy_syslog_server_ip_b, check_default_route): - duthost = rand_selected_dut - logger.info("Starting syslog tests") - test_message = "Basic Test Message" - - check_dummy_addr_and_default_route(dummy_syslog_server_ip_a, dummy_syslog_server_ip_b, - check_default_route['IPv4'], check_default_route['IPv6']) - - if dummy_syslog_server_ip_a: - if ":" not in dummy_syslog_server_ip_a: - duthost.command( - "sudo ip -4 rule add from all to {} pref 1 lookup default".format(dummy_syslog_server_ip_a)) - else: - duthost.command( - "sudo ip -6 rule add from all to {} pref 1 lookup default".format(dummy_syslog_server_ip_a)) - - if dummy_syslog_server_ip_b: - if ":" not in dummy_syslog_server_ip_b: - duthost.command( - "sudo ip -4 rule add from all to {} pref 2 lookup default".format(dummy_syslog_server_ip_b)) - else: - duthost.command( - "sudo ip -6 rule add from all to {} pref 2 lookup default".format(dummy_syslog_server_ip_b)) - - logger.info("Configuring the DUT") - # Add dummy rsyslog destination for testing - if dummy_syslog_server_ip_a is not None: - if "201911" in duthost.os_version and ":" in dummy_syslog_server_ip_a: - pytest.skip("IPv6 syslog server IP not supported on 201911") - duthost.shell("sudo config syslog add {}".format(dummy_syslog_server_ip_a)) - logger.debug("Added new rsyslog server IP {}".format(dummy_syslog_server_ip_a)) - if dummy_syslog_server_ip_b is not None: - if "201911" in duthost.os_version and ":" in dummy_syslog_server_ip_b: - pytest.skip("IPv6 syslog server IP not supported on 201911") - duthost.shell("sudo config syslog add {}".format(dummy_syslog_server_ip_b)) - logger.debug("Added new rsyslog server IP {}".format(dummy_syslog_server_ip_b)) - - logger.info("Start tcpdump") - # Make sure that the DUT_PCAP_FILEPATH dose not exist - duthost.shell("sudo rm -f {}".format(DUT_PCAP_FILEPATH)) - # Scapy doesn't support LINUX_SLL2 (Linux cooked v2), and tcpdump on Bullseye - # defaults to writing in that format when listening on any interface. Therefore, - # have it use LINUX_SLL (Linux cooked) instead. - tcpdump_task, tcpdump_result = duthost.shell( - "sudo timeout 20 tcpdump -y LINUX_SLL -i any -s0 -A -w {} \"udp and port 514\"" - .format(DUT_PCAP_FILEPATH), module_async=True) - # wait for starting tcpdump - time.sleep(5) - - logger.debug("Generating log message from DUT") - # Generate a syslog from the DUT - duthost.shell("logger --priority INFO {}".format(test_message)) - - # wait for stoping tcpdump - tcpdump_task.close() - tcpdump_task.join() - - # Remove the syslog configuration - if dummy_syslog_server_ip_a is not None: - duthost.shell("sudo config syslog del {}".format(dummy_syslog_server_ip_a)) - if ":" not in dummy_syslog_server_ip_a: - duthost.command( - "sudo ip -4 rule del from all to {} pref 1 lookup default".format(dummy_syslog_server_ip_a)) - else: - duthost.command( - "sudo ip -6 rule del from all to {} pref 1 lookup default".format(dummy_syslog_server_ip_a)) - - if dummy_syslog_server_ip_b is not None: - duthost.shell("sudo config syslog del {}".format(dummy_syslog_server_ip_b)) - if ":" not in dummy_syslog_server_ip_b: - duthost.command( - "sudo ip -4 rule del from all to {} pref 2 lookup default".format(dummy_syslog_server_ip_b)) - else: - duthost.command( - "sudo ip -6 rule del from all to {} pref 2 lookup default".format(dummy_syslog_server_ip_b)) - - duthost.fetch(src=DUT_PCAP_FILEPATH, dest=DOCKER_TMP_PATH) - filepath = os.path.join(DOCKER_TMP_PATH, duthost.hostname, DUT_PCAP_FILEPATH.lstrip(os.path.sep)) - - if not _check_pcap(dummy_syslog_server_ip_a, dummy_syslog_server_ip_b, filepath): - default_route_v4 = duthost.shell("ip route show default table default")['stdout'] - logger.debug("DUT's IPv4 default route:\n%s" % default_route_v4) - default_route_v6 = duthost.shell("ip -6 route show default table default")['stdout'] - logger.debug("DUT's IPv6 default route:\n%s" % default_route_v6) - syslog_config = duthost.shell("grep 'remote syslog server' -A 7 /etc/rsyslog.conf")['stdout'] - logger.debug("DUT's syslog server IPs:\n%s" % syslog_config) - - pytest.fail("Dummy syslog server IP not seen in the pcap file") - @pytest.mark.parametrize("dummy_syslog_server_ip_a, dummy_syslog_server_ip_b", [("7.0.80.166", None), @@ -193,5 +15,7 @@ def run_syslog(rand_selected_dut, dummy_syslog_server_ip_a, dummy_syslog_server_ ("7.0.80.165", "7.0.80.166"), ("fd82:b34f:cc99::100", "7.0.80.166"), ("fd82:b34f:cc99::100", "fd82:b34f:cc99::200")]) -def test_syslog(rand_selected_dut, dummy_syslog_server_ip_a, dummy_syslog_server_ip_b, check_default_route): +def test_syslog(rand_selected_dut, dummy_syslog_server_ip_a, dummy_syslog_server_ip_b, + check_default_route # noqa: F811 + ): run_syslog(rand_selected_dut, dummy_syslog_server_ip_a, dummy_syslog_server_ip_b, check_default_route) diff --git a/tests/syslog/test_syslog_rate_limit.py b/tests/syslog/test_syslog_rate_limit.py index f12683ec57..e91f7f60f9 100644 --- a/tests/syslog/test_syslog_rate_limit.py +++ b/tests/syslog/test_syslog_rate_limit.py @@ -120,8 +120,15 @@ def verify_container_rate_limit(rand_selected_dut, ignore_containers=[]): service_name = item['feature'] if service_name in ignore_containers: continue - - logger.info('Start syslog rate limit test for container {}'.format(service_name)) + container_name = service_name + if rand_selected_dut.is_multi_asic: + config_facts = rand_selected_dut.get_running_config_facts() + if config_facts['FEATURE'][service_name]['has_per_asic_scope'] == "True": + asic_ids = rand_selected_dut.get_asic_ids() + asic_id = random.choice(asic_ids) + container_name = service_name + str(asic_id) + + logger.info('Start syslog rate limit test for container {}'.format(container_name)) if item['state'] in ['disabled', 'always_disabled']: logger.info('Container {} is {}'.format(service_name, item['state'])) continue @@ -129,14 +136,14 @@ def verify_container_rate_limit(rand_selected_dut, ignore_containers=[]): support_syslog_rate_limit = config_db.hget_key_value('FEATURE|{}'.format(service_name), 'support_syslog_rate_limit') if support_syslog_rate_limit.lower() != 'true': - logger.info('Container {} does not support syslog rate limit configuration'.format(service_name)) - verify_config_rate_limit_fail(rand_selected_dut, service_name) + logger.info('Container {} does not support syslog rate limit configuration'.format(container_name)) + verify_config_rate_limit_fail(rand_selected_dut, container_name) continue - rsyslog_pid = get_rsyslogd_pid(rand_selected_dut, service_name) + rsyslog_pid = get_rsyslogd_pid(rand_selected_dut, container_name) rand_selected_dut.command('config syslog rate-limit-container {} -b {} -i {}'.format( service_name, RATE_LIMIT_BURST, RATE_LIMIT_INTERVAL)) - assert wait_rsyslogd_restart(rand_selected_dut, service_name, rsyslog_pid) + assert wait_rsyslogd_restart(rand_selected_dut, container_name, rsyslog_pid) rate_limit_data = rand_selected_dut.show_and_parse('show syslog rate-limit-container {}'.format(service_name)) pytest_assert(rate_limit_data[0]['interval'] == str(RATE_LIMIT_INTERVAL), 'Expect rate limit interval {}, actual {}'.format(RATE_LIMIT_INTERVAL, @@ -145,19 +152,19 @@ def verify_container_rate_limit(rand_selected_dut, ignore_containers=[]): 'Expect rate limit burst {}, actual {}'.format(RATE_LIMIT_BURST, rate_limit_data[0]['burst'])) rand_selected_dut.command( - 'docker cp {} {}:{}'.format(REMOTE_LOG_GENERATOR_FILE, service_name, DOCKER_LOG_GENERATOR_FILE)) + 'docker cp {} {}:{}'.format(REMOTE_LOG_GENERATOR_FILE, container_name, DOCKER_LOG_GENERATOR_FILE)) verify_rate_limit_with_log_generator(rand_selected_dut, - service_name, + container_name, 'syslog_rate_limit_{}-interval_{}_burst_{}'.format(service_name, RATE_LIMIT_INTERVAL, RATE_LIMIT_BURST), [LOG_EXPECT_SYSLOG_RATE_LIMIT_REACHED, - LOG_EXPECT_LAST_MESSAGE.format(service_name + '#')], + LOG_EXPECT_LAST_MESSAGE.format(container_name + '#')], RATE_LIMIT_BURST + 1) - rsyslog_pid = get_rsyslogd_pid(rand_selected_dut, service_name) + rsyslog_pid = get_rsyslogd_pid(rand_selected_dut, container_name) rand_selected_dut.command('config syslog rate-limit-container {} -b {} -i {}'.format(service_name, 0, 0)) - assert wait_rsyslogd_restart(rand_selected_dut, service_name, rsyslog_pid) + assert wait_rsyslogd_restart(rand_selected_dut, container_name, rsyslog_pid) rate_limit_data = rand_selected_dut.show_and_parse('show syslog rate-limit-container {}'.format(service_name)) pytest_assert(rate_limit_data[0]['interval'] == '0', 'Expect rate limit interval {}, actual {}'.format(0, rate_limit_data[0]['interval'])) @@ -165,9 +172,9 @@ def verify_container_rate_limit(rand_selected_dut, ignore_containers=[]): 'Expect rate limit burst {}, actual {}'.format(0, rate_limit_data[0]['burst'])) verify_rate_limit_with_log_generator(rand_selected_dut, - service_name, + container_name, 'syslog_rate_limit_{}-interval_{}_burst_{}'.format(service_name, 0, 0), - [LOG_EXPECT_LAST_MESSAGE.format(service_name + '#')], + [LOG_EXPECT_LAST_MESSAGE.format(container_name + '#')], LOG_MESSAGE_GENERATE_COUNT) break # we only randomly test 1 container to reduce test time diff --git a/tests/syslog/test_syslog_source_ip.py b/tests/syslog/test_syslog_source_ip.py index 869a127f63..ce418b0371 100644 --- a/tests/syslog/test_syslog_source_ip.py +++ b/tests/syslog/test_syslog_source_ip.py @@ -7,8 +7,9 @@ from scapy.all import rdpcap from .syslog_utils import create_vrf, remove_vrf, add_syslog_server, del_syslog_server, capture_syslog_packets,\ - replace_ip_neigh, is_mgmt_vrf_enabled, bind_interface_to_vrf, check_vrf, TCPDUMP_CAPTURE_TIME, DUT_PCAP_FILEPATH + replace_ip_neigh, bind_interface_to_vrf, check_vrf, TCPDUMP_CAPTURE_TIME, DUT_PCAP_FILEPATH from tests.common.utilities import wait_until +from tests.common.helpers.syslog_helpers import is_mgmt_vrf_enabled from tests.common.helpers.assertions import pytest_assert from tests.common.reboot import reboot, SONIC_SSH_PORT, SONIC_SSH_REGEX from ipaddress import IPv4Address, IPv6Address, ip_address, ip_network, IPv6Network @@ -252,9 +253,9 @@ def configure_mgmt_vrf_test_data(self, localhost): logger.info("Create mgmt vrf") create_vrf(self.duthost, VRF_LIST[2]) # when create mgmt vrf, dut connection will be lost for a while - localhost.wait_for(host=self.duthost.hostname, port=SONIC_SSH_PORT, search_regex=SONIC_SSH_REGEX, + localhost.wait_for(host=self.duthost.mgmt_ip, port=SONIC_SSH_PORT, search_regex=SONIC_SSH_REGEX, state='absent', delay=1, timeout=30) - localhost.wait_for(host=self.duthost.hostname, port=SONIC_SSH_PORT, search_regex=SONIC_SSH_REGEX, + localhost.wait_for(host=self.duthost.mgmt_ip, port=SONIC_SSH_PORT, search_regex=SONIC_SSH_REGEX, state='started', delay=2, timeout=180) for k, v in list(MGMT_IP_ADDRESSES.items()): diff --git a/tests/system_health/mellanox/mellanox_device_mocker.py b/tests/system_health/mellanox/mellanox_device_mocker.py index 6ddbc95eca..0165242d01 100644 --- a/tests/system_health/mellanox/mellanox_device_mocker.py +++ b/tests/system_health/mellanox/mellanox_device_mocker.py @@ -1,7 +1,7 @@ from ..device_mocker import DeviceMocker from pkg_resources import parse_version from tests.common.mellanox_data import get_platform_data, get_hw_management_version -from tests.platform_tests.mellanox.mellanox_thermal_control_test_helper import MockerHelper, FanDrawerData, FanData, \ +from tests.common.helpers.mellanox_thermal_control_test_helper import MockerHelper, FanDrawerData, FanData, \ FAN_NAMING_RULE HW_MANAGE_VER = '7.0030.2003' diff --git a/tests/system_health/test_system_health.py b/tests/system_health/test_system_health.py index fb06e8930b..eca0bddc59 100644 --- a/tests/system_health/test_system_health.py +++ b/tests/system_health/test_system_health.py @@ -9,12 +9,14 @@ from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_require from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer -from tests.platform_tests.thermal_control_test_helper import disable_thermal_policy # noqa F401 +from tests.common.helpers.thermal_control_test_helper import disable_thermal_policy # noqa F401 from .device_mocker import device_mocker_factory # noqa F401 from tests.common.helpers.assertions import pytest_assert +from tests.common.fixtures.duthost_utils import is_support_mock_asic # noqa F401 pytestmark = [ - pytest.mark.topology('any') + pytest.mark.topology('any'), + pytest.mark.device_type('physical') ] logger = logging.getLogger(__name__) @@ -159,60 +161,51 @@ def test_service_checker_with_process_exit(duthosts, enum_rand_one_per_hwsku_hos @pytest.mark.disable_loganalyzer def test_device_checker(duthosts, enum_rand_one_per_hwsku_hostname, - device_mocker_factory, disable_thermal_policy): # noqa F811 + device_mocker_factory, disable_thermal_policy, is_support_mock_asic): # noqa F811 duthost = duthosts[enum_rand_one_per_hwsku_hostname] device_mocker = device_mocker_factory(duthost) wait_system_health_boot_up(duthost) with ConfigFileContext(duthost, os.path.join(FILES_DIR, DEVICE_CHECK_CONFIG_FILE)): time.sleep(DEFAULT_INTERVAL) - fan_mock_result, fan_name = device_mocker.mock_fan_speed(False) - fan_expect_value = EXPECT_FAN_INVALID_SPEED.format(fan_name) - asic_mock_result = device_mocker.mock_asic_temperature(False) + asic_mock_result = 'not support asic mock' + if is_support_mock_asic: + asic_mock_result = device_mocker.mock_asic_temperature(False) asic_expect_value = EXPECT_ASIC_HOT psu_mock_result, psu_name = device_mocker.mock_psu_presence(False) psu_expect_value = EXPECT_PSU_MISSING.format(psu_name) - if fan_mock_result and asic_mock_result and psu_mock_result: - logger.info('Mocked invalid fan speed for {}'.format(fan_name)) + if asic_mock_result and psu_mock_result: logger.info('Mocked ASIC overheated') logger.info('Mocked PSU absence for {}'.format(psu_name)) logger.info('Waiting {} seconds for it to take effect'.format( THERMAL_CHECK_INTERVAL)) time.sleep(THERMAL_CHECK_INTERVAL) - value = redis_get_field_value( - duthost, STATE_DB, HEALTH_TABLE_NAME, fan_name) - assert value and fan_expect_value in value,\ - 'Mock fan invalid speed, expect {}, but got {}'.format(fan_expect_value, value) - value = redis_get_field_value( - duthost, STATE_DB, HEALTH_TABLE_NAME, 'ASIC') - assert value and asic_expect_value in value,\ - 'Mock ASIC temperature overheated, expect {}, but got {}'.format(asic_expect_value, value) + if is_support_mock_asic: + value = redis_get_field_value(duthost, STATE_DB, HEALTH_TABLE_NAME, 'ASIC') + assert value and asic_expect_value in value,\ + 'Mock ASIC temperature overheated, expect {}, but got {}'.format(asic_expect_value, value) value = redis_get_field_value( duthost, STATE_DB, HEALTH_TABLE_NAME, psu_name) assert value and psu_expect_value == value,\ 'Mock PSU absence, expect {}, but got {}'.format(psu_expect_value, value) - fan_mock_result, fan_name = device_mocker.mock_fan_speed(True) - asic_mock_result = device_mocker.mock_asic_temperature(True) + + if is_support_mock_asic: + asic_mock_result = device_mocker.mock_asic_temperature(True) psu_mock_result, psu_name = device_mocker.mock_psu_presence(True) - if fan_mock_result and asic_mock_result and psu_mock_result: - logger.info('Mocked valid fan speed for {}'.format(fan_name)) + if asic_mock_result and psu_mock_result: logger.info('Mocked ASIC normal temperatue') logger.info('Mocked PSU presence for {}'.format(psu_name)) logger.info('Waiting {} seconds for it to take effect'.format( THERMAL_CHECK_INTERVAL)) time.sleep(THERMAL_CHECK_INTERVAL) - value = redis_get_field_value( - duthost, STATE_DB, HEALTH_TABLE_NAME, fan_name) - assert not value or fan_expect_value not in value,\ - 'Mock fan valid speed, expect {}, but it still report invalid speed'.format(fan_expect_value) - value = redis_get_field_value( - duthost, STATE_DB, HEALTH_TABLE_NAME, 'ASIC') - assert not value or asic_expect_value not in value,\ - 'Mock ASIC normal temperature, but it is still overheated' + if is_support_mock_asic: + value = redis_get_field_value(duthost, STATE_DB, HEALTH_TABLE_NAME, 'ASIC') + assert not value or asic_expect_value not in value,\ + 'Mock ASIC normal temperature, but it is still overheated' value = redis_get_field_value( duthost, STATE_DB, HEALTH_TABLE_NAME, psu_name) @@ -356,7 +349,7 @@ def test_external_checker(duthosts, enum_rand_one_per_hwsku_hostname): @pytest.mark.disable_loganalyzer @pytest.mark.parametrize('ignore_log_analyzer_by_vendor', [['mellanox']], indirect=True) def test_system_health_config(duthosts, enum_rand_one_per_hwsku_hostname, - device_mocker_factory, ignore_log_analyzer_by_vendor): # noqa F811 + device_mocker_factory, ignore_log_analyzer_by_vendor, is_support_mock_asic): # noqa F811 duthost = duthosts[enum_rand_one_per_hwsku_hostname] device_mocker = device_mocker_factory(duthost) wait_system_health_boot_up(duthost) @@ -375,16 +368,17 @@ def test_system_health_config(duthosts, enum_rand_one_per_hwsku_hostname, logger.info( 'Ignore ASIC check, verify there is no error information about ASIC') - with ConfigFileContext(duthost, os.path.join(FILES_DIR, IGNORE_ASIC_CHECK_CONFIG_FILE)): - time.sleep(FAST_INTERVAL) - mock_result = device_mocker.mock_asic_temperature(False) - expect_value = EXPECT_ASIC_HOT - if mock_result: - time.sleep(THERMAL_CHECK_INTERVAL) - value = redis_get_field_value( - duthost, STATE_DB, HEALTH_TABLE_NAME, 'ASIC') - assert not value or expect_value not in value, 'ASIC check is still performed after it ' \ - 'is configured to be ignored' + if is_support_mock_asic: + with ConfigFileContext(duthost, os.path.join(FILES_DIR, IGNORE_ASIC_CHECK_CONFIG_FILE)): + time.sleep(FAST_INTERVAL) + mock_result = device_mocker.mock_asic_temperature(False) + expect_value = EXPECT_ASIC_HOT + if mock_result: + time.sleep(THERMAL_CHECK_INTERVAL) + value = redis_get_field_value( + duthost, STATE_DB, HEALTH_TABLE_NAME, 'ASIC') + assert not value or expect_value not in value, 'ASIC check is still performed after it ' \ + 'is configured to be ignored' logger.info( 'Ignore PSU check, verify there is no error information about psu') @@ -400,6 +394,40 @@ def test_system_health_config(duthosts, enum_rand_one_per_hwsku_hostname, 'is configured to be ignored' +@pytest.mark.disable_loganalyzer +def test_device_fan_speed_checker(duthosts, enum_rand_one_per_hwsku_hostname, + device_mocker_factory, disable_thermal_policy, is_support_mock_asic): # noqa F811 + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + device_mocker = device_mocker_factory(duthost) + wait_system_health_boot_up(duthost) + with ConfigFileContext(duthost, os.path.join(FILES_DIR, DEVICE_CHECK_CONFIG_FILE)): + time.sleep(DEFAULT_INTERVAL) + + fan_mock_result, fan_name = device_mocker.mock_fan_speed(False) + fan_expect_value = EXPECT_FAN_INVALID_SPEED.format(fan_name) + + if fan_mock_result: + logger.info('Mocked invalid fan speed for {}'.format(fan_name)) + logger.info('Waiting {} seconds for it to take effect'.format( + THERMAL_CHECK_INTERVAL)) + time.sleep(THERMAL_CHECK_INTERVAL) + value = redis_get_field_value( + duthost, STATE_DB, HEALTH_TABLE_NAME, fan_name) + assert value and fan_expect_value in value, \ + 'Mock fan invalid speed, expect {}, but got {}'.format(fan_expect_value, value) + + fan_mock_result, fan_name = device_mocker.mock_fan_speed(True) + if fan_mock_result: + logger.info('Mocked valid fan speed for {}'.format(fan_name)) + logger.info('Waiting {} seconds for it to take effect'.format( + THERMAL_CHECK_INTERVAL)) + time.sleep(THERMAL_CHECK_INTERVAL) + value = redis_get_field_value( + duthost, STATE_DB, HEALTH_TABLE_NAME, fan_name) + assert not value or fan_expect_value not in value, \ + 'Mock fan valid speed, expect {}, but it still report invalid speed'.format(fan_expect_value) + + def wait_system_health_boot_up(duthost): boot_timeout = get_system_health_config( duthost, 'boot_timeout', DEFAULT_BOOT_TIMEOUT) diff --git a/tests/tacacs/conftest.py b/tests/tacacs/conftest.py index d60a5b7c9e..9f3c360679 100644 --- a/tests/tacacs/conftest.py +++ b/tests/tacacs/conftest.py @@ -1,51 +1,45 @@ import logging import pytest from tests.common.fixtures.tacacs import tacacs_creds # noqa F401 -from contextlib import contextmanager -from .utils import setup_tacacs_client, setup_tacacs_server,\ - cleanup_tacacs, restore_tacacs_servers +from tests.common.helpers.tacacs.tacacs_helper import setup_tacacs_client, setup_tacacs_server,\ + cleanup_tacacs, restore_tacacs_servers, _context_for_check_tacacs_v6 logger = logging.getLogger(__name__) +def check_nss_config(duthost): + nss_config_attribute = duthost.command("ls -la /etc/nsswitch.conf", module_ignore_errors=True) + if nss_config_attribute['failed']: + logger.error("NSS config file missing: %s", nss_config_attribute['stderr']) + else: + logger.debug("NSS config file attribute: %s", nss_config_attribute['stdout']) + + @pytest.fixture(scope="module") def check_tacacs(ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds): # noqa F811 duthost = duthosts[enum_rand_one_per_hwsku_hostname] tacacs_server_ip = ptfhost.mgmt_ip tacacs_server_passkey = tacacs_creds[duthost.hostname]['tacacs_passkey'] + + # Accounting test case randomly failed, need debug info to confirm NSS config file missing issue. + check_nss_config(duthost) + setup_tacacs_client(duthost, tacacs_creds, tacacs_server_ip, tacacs_server_passkey, ptfhost) setup_tacacs_server(ptfhost, tacacs_creds, duthost) + check_nss_config(duthost) + yield + check_nss_config(duthost) + cleanup_tacacs(ptfhost, tacacs_creds, duthost) restore_tacacs_servers(duthost) + check_nss_config(duthost) + @pytest.fixture(scope="module") def check_tacacs_v6(ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds): # noqa F811 with _context_for_check_tacacs_v6(ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds) as result: yield result - - -@pytest.fixture(scope="function") -def check_tacacs_v6_func(ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds): # noqa F811 - with _context_for_check_tacacs_v6(ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds) as result: - yield result - - -@contextmanager -def _context_for_check_tacacs_v6(ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds): # noqa F811 - duthost = duthosts[enum_rand_one_per_hwsku_hostname] - ptfhost_vars = ptfhost.host.options['inventory_manager'].get_host(ptfhost.hostname).vars - if 'ansible_hostv6' not in ptfhost_vars: - pytest.skip("Skip IPv6 test. ptf ansible_hostv6 not configured.") - tacacs_server_ip = ptfhost_vars['ansible_hostv6'] - tacacs_server_passkey = tacacs_creds[duthost.hostname]['tacacs_passkey'] - setup_tacacs_client(duthost, tacacs_creds, tacacs_server_ip, tacacs_server_passkey, ptfhost) - setup_tacacs_server(ptfhost, tacacs_creds, duthost) - - yield - - cleanup_tacacs(ptfhost, tacacs_creds, duthost) - restore_tacacs_servers(duthost) diff --git a/tests/tacacs/test_accounting.py b/tests/tacacs/test_accounting.py index 78c828f997..24ff82722d 100644 --- a/tests/tacacs/test_accounting.py +++ b/tests/tacacs/test_accounting.py @@ -1,17 +1,11 @@ import logging import time -from tests.common.devices.ptf import PTFHost - - import pytest - - -from .test_authorization import ssh_connect_remote_retry, ssh_run_command, \ - remove_all_tacacs_server -from .utils import stop_tacacs_server, start_tacacs_server, \ - check_server_received, per_command_accounting_skip_versions, \ - change_and_wait_aaa_config_update, get_auditd_config_reload_timestamp, \ - ensure_tacacs_server_running_after_ut # noqa: F401 +from tests.common.devices.ptf import PTFHost +from tests.common.helpers.tacacs.tacacs_helper import stop_tacacs_server, start_tacacs_server, \ + per_command_accounting_skip_versions, remove_all_tacacs_server +from .utils import check_server_received, change_and_wait_aaa_config_update, get_auditd_config_reload_timestamp, \ + ensure_tacacs_server_running_after_ut, ssh_connect_remote_retry, ssh_run_command # noqa: F401 from tests.common.errors import RunAnsibleModuleFail from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import skip_release @@ -23,7 +17,6 @@ pytest.mark.device_type('vs') ] - logger = logging.getLogger(__name__) @@ -117,10 +110,14 @@ def check_local_log_exist(duthost, tacacs_creds, command): logs = wait_for_log(duthost, "/var/log/syslog", log_pattern) if len(logs) == 0: - # print recent logs for debug - recent_logs = duthost.command("cat /var/log/syslog | tail -n 500") + # Print recent logs for debug + recent_logs = duthost.command("tail /var/log/syslog -n 1000") logger.debug("Found logs: %s", recent_logs) + # Missing log may caused by incorrect NSS config + tacacs_config = duthost.command("cat /etc/tacplus_nss.conf") + logger.debug("tacplus_nss.conf: %s", tacacs_config) + pytest_assert(len(logs) > 0) # exclude logs of the sed command produced by Ansible @@ -260,8 +257,8 @@ def test_accounting_tacacs_only_some_tacacs_server_down( duthost.shell("sudo config tacacs timeout 1") remove_all_tacacs_server(duthost) - duthost.shell("sudo config tacacs add %s" % invalid_tacacs_server_ip) - duthost.shell("sudo config tacacs add %s" % tacacs_server_ip) + duthost.shell("sudo config tacacs add %s --port 59" % invalid_tacacs_server_ip) + duthost.shell("sudo config tacacs add %s --port 59" % tacacs_server_ip) change_and_wait_aaa_config_update(duthost, "sudo config aaa accounting tacacs+", last_timestamp) diff --git a/tests/tacacs/test_authorization.py b/tests/tacacs/test_authorization.py index 1e58776cf5..5b7ff35c96 100644 --- a/tests/tacacs/test_authorization.py +++ b/tests/tacacs/test_authorization.py @@ -4,16 +4,17 @@ from _pytest.outcomes import Failed import time -from tests.tacacs.utils import stop_tacacs_server, start_tacacs_server -from tests.tacacs.utils import per_command_authorization_skip_versions, \ - remove_all_tacacs_server, get_ld_path, change_and_wait_aaa_config_update, \ - ensure_tacacs_server_running_after_ut # noqa: F401 +from tests.common.helpers.tacacs.tacacs_helper import stop_tacacs_server, start_tacacs_server, \ + per_command_authorization_skip_versions, remove_all_tacacs_server, get_ld_path +from tests.tacacs.utils import change_and_wait_aaa_config_update, ensure_tacacs_server_running_after_ut, \ + ssh_connect_remote_retry, ssh_run_command, TIMEOUT_LIMIT # noqa: F401 from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import skip_release, wait_until, paramiko_ssh from .utils import check_server_received -from tests.override_config_table.utilities import backup_config, restore_config, \ +from tests.common.utilities import backup_config, restore_config, \ reload_minigraph_with_golden_config from tests.common.helpers.dut_utils import is_container_running +from .utils import duthost_shell_with_unreachable_retry pytestmark = [ pytest.mark.disable_loganalyzer, @@ -23,24 +24,6 @@ logger = logging.getLogger(__name__) -TIMEOUT_LIMIT = 120 - - -def ssh_connect_remote_retry(remote_ip, remote_username, remote_password, duthost): - retry_count = 3 - while retry_count > 0: - try: - return paramiko_ssh(remote_ip, remote_username, remote_password) - except paramiko.ssh_exception.AuthenticationException as e: - logger.info("Paramiko SSH connect failed with authentication: " + repr(e)) - - # get syslog for debug - recent_syslog = duthost.shell('sudo tail -100 /var/log/syslog')['stdout'] - logger.debug("Target device syslog: {}".format(recent_syslog)) - - time.sleep(1) - retry_count -= 1 - def check_ssh_connect_remote_failed(remote_ip, remote_username, remote_password): login_failed = False @@ -53,12 +36,6 @@ def check_ssh_connect_remote_failed(remote_ip, remote_username, remote_password) pytest_assert(login_failed) -def ssh_run_command(ssh_client, command): - stdin, stdout, stderr = ssh_client.exec_command(command, timeout=TIMEOUT_LIMIT) - exit_code = stdout.channel.recv_exit_status() - return exit_code, stdout, stderr - - def check_ssh_output_any_of(res_stream, exp_vals, timeout=10): while timeout > 0: res_lines = res_stream.readlines() @@ -195,8 +172,6 @@ def test_authorization_tacacs_only( # check commands used by scripts commands = [ "show interfaces counters -a -p 3", - "show ip bgp neighbor", - "show ipv6 bgp neighbor", "touch testfile", "chmod +w testfile", "echo \"test\" > testfile", @@ -207,10 +182,7 @@ def test_authorization_tacacs_only( "rm -f testfi*", "mkdir -p test", "portstat -c", - "show ip bgp summary", - "show ipv6 bgp summary", "show interfaces portchannel", - "show muxcable firmware", "show platform summary", "show version", "show lldp table", @@ -219,7 +191,17 @@ def test_authorization_tacacs_only( "sonic-db-cli CONFIG_DB HGET \"FEATURE|macsec\" state" ] + frontend_commands = [ + "show ip bgp neighbor", + "show ipv6 bgp neighbor", + "show ip bgp summary", + "show ipv6 bgp summary", + "show muxcable firmware", + ] + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + if duthost.sonichost.is_frontend_node(): + commands.extend(frontend_commands) telemetry_is_running = is_container_running(duthost, 'telemetry') gnmi_is_running = is_container_running(duthost, 'gnmi') if not telemetry_is_running and gnmi_is_running: @@ -264,8 +246,8 @@ def test_authorization_tacacs_only_some_server_down( # cleanup all tacacs server, if UT break, tacacs server may still left in dut and will break next UT. remove_all_tacacs_server(duthost) - duthost.shell("sudo config tacacs add %s" % invalid_tacacs_server_ip) - duthost.shell("sudo config tacacs add %s" % tacacs_server_ip) + duthost.shell("sudo config tacacs add %s --port 59" % invalid_tacacs_server_ip) + duthost.shell("sudo config tacacs add %s --port 59" % tacacs_server_ip) """ Verify TACACS+ user run command in server side whitelist: @@ -604,7 +586,7 @@ def test_stop_request_next_server_after_reject( tacacs_server_ipv6 = ptfhost_vars['ansible_hostv6'] # Setup second tacacs server - duthost.shell("sudo config tacacs add {}".format(tacacs_server_ipv6)) + duthost_shell_with_unreachable_retry(duthost, "sudo config tacacs add {} --port 59".format(tacacs_server_ipv6)) duthost.shell("sudo config tacacs timeout 1") # Clean tacacs log @@ -665,7 +647,7 @@ def test_fallback_to_local_authorization_with_config_reload( "global": {"auth_type": "login", "passkey": tacacs_passkey} }, "TACPLUS_SERVER": { - tacacs_server_ip: {"priority": "60", "tcp_port": "49", "timeout": "2"} + tacacs_server_ip: {"priority": "60", "tcp_port": "59", "timeout": "2"} } } reload_minigraph_with_golden_config(duthost, override_config) diff --git a/tests/tacacs/test_jit_user.py b/tests/tacacs/test_jit_user.py index 21a4fbc8d9..9e8a805cbd 100644 --- a/tests/tacacs/test_jit_user.py +++ b/tests/tacacs/test_jit_user.py @@ -1,7 +1,8 @@ import logging import pytest -from .test_ro_user import ssh_remote_run -from .utils import check_output, setup_tacacs_server +from tests.common.helpers.tacacs.tacacs_helper import ssh_remote_run +from tests.common.helpers.tacacs.tacacs_helper import setup_tacacs_server +from tests.common.utilities import check_output pytestmark = [ pytest.mark.disable_loganalyzer, diff --git a/tests/tacacs/test_ro_disk.py b/tests/tacacs/test_ro_disk.py index 0ec34e8a86..9401053831 100644 --- a/tests/tacacs/test_ro_disk.py +++ b/tests/tacacs/test_ro_disk.py @@ -4,13 +4,16 @@ import time from ansible.errors import AnsibleConnectionFailure +from pytest_ansible.errors import AnsibleConnectionFailure as PytestAnsibleConnectionFailure from tests.common.devices.base import RunAnsibleModuleFail from tests.common.utilities import wait_until from tests.common.utilities import skip_release from tests.common.utilities import wait +from tests.common.utilities import pdu_reboot from tests.common.reboot import reboot -from .test_ro_user import ssh_remote_run -from .utils import setup_tacacs_client, change_and_wait_aaa_config_update +from tests.common.helpers.tacacs.tacacs_helper import ssh_remote_run +from tests.common.helpers.tacacs.tacacs_helper import setup_tacacs_client +from .utils import change_and_wait_aaa_config_update from tests.common.platform.interface_utils import check_interface_status_of_up_ports from tests.common.platform.processes_utils import wait_critical_processes @@ -55,6 +58,13 @@ def chk_ssh_remote_run(localhost, remote_ip, username, password, cmd): return rc == 0 +def do_pdu_reboot(duthost, localhost, duthosts, pdu_controller): + if not pdu_reboot(pdu_controller): + logger.error("Failed to do PDU reboot for {}".format(duthost.hostname)) + return + return post_reboot_healthcheck(duthost, localhost, duthosts, 20) + + def do_reboot(duthost, localhost, duthosts): # occasionally reboot command fails with some kernel error messages # Hence retry if needed. @@ -74,7 +84,7 @@ def do_reboot(duthost, localhost, duthosts): localhost.wait_for(host=duthost.mgmt_ip, port=22, state="stopped", delay=5, timeout=60) rebooted = True break - except AnsibleConnectionFailure as e: + except (AnsibleConnectionFailure, PytestAnsibleConnectionFailure) as e: logger.error("DUT not reachable, exception: {} attempt:{}/{}". format(repr(e), i, retries)) except RunAnsibleModuleFail as e: @@ -83,11 +93,25 @@ def do_reboot(duthost, localhost, duthosts): wait(wait_time, msg="Wait {} seconds before retry.".format(wait_time)) - assert rebooted, "Failed to reboot" - localhost.wait_for(host=duthost.mgmt_ip, port=22, state="started", delay=10, timeout=300) + if not rebooted: + logger.error("Failed to reboot DUT after {} retries".format(retries)) + return False + + return post_reboot_healthcheck(duthost, localhost, duthosts, wait_time) + + +def post_reboot_healthcheck(duthost, localhost, duthosts, wait_time): + timeout = 300 + if duthost.get_facts().get("modular_chassis"): + wait_time = max(wait_time, 900) + timeout = max(timeout, 600) + localhost.wait_for(host=duthost.mgmt_ip, port=22, state="started", delay=10, timeout=timeout) + else: + localhost.wait_for(host=duthost.mgmt_ip, port=22, state="started", delay=10, timeout=timeout) wait(wait_time, msg="Wait {} seconds for system to be stable.".format(wait_time)) - assert wait_until(300, 20, 0, duthost.critical_services_fully_started), \ - "All critical services should fully started!" + if not wait_until(300, 20, 0, duthost.critical_services_fully_started): + logger.error("Not all critical services fully started!") + return False # If supervisor node is rebooted in chassis, linecards also will reboot. # Check if all linecards are back up. if duthost.is_supervisor_node(): @@ -95,8 +119,10 @@ def do_reboot(duthost, localhost, duthosts): if host != duthost: logger.info("checking if {} critical services are up".format(host.hostname)) wait_critical_processes(host) - assert wait_until(300, 20, 0, check_interface_status_of_up_ports, host), \ - "Not all ports that are admin up on are operationally up" + if not wait_until(300, 20, 0, check_interface_status_of_up_ports, host): + logger.error("Not all ports that are admin up on are operationally up") + return False + return True def do_setup_tacacs(ptfhost, duthost, tacacs_creds): @@ -137,12 +163,18 @@ def log_rotate(duthost): if "logrotate does not support parallel execution on the same set of logfiles" in e.message: # command will failed when log already in rotating logger.warning("logrotate command failed: {}".format(e)) + elif "error: stat of /var/log/auth.log failed: Bad message" in e.message: + # command will failed because auth.log missing + logger.warning("logrotate command failed: {}".format(e)) + elif "du: cannot access '/var/log/auth.log': Bad message" in e.message: + # command will failed because auth.log missing + logger.warning("logrotate command failed: {}".format(e)) else: raise e def test_ro_disk(localhost, ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, - tacacs_creds, check_tacacs): + tacacs_creds, check_tacacs, pdu_controller): """test tacacs rw user """ duthost = duthosts[enum_rand_one_per_hwsku_hostname] @@ -162,7 +194,7 @@ def test_ro_disk(localhost, ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, # logger.info("PRETEST: reboot {} to restore system state". format(enum_rand_one_per_hwsku_hostname)) - do_reboot(duthost, localhost, duthosts) + assert do_reboot(duthost, localhost, duthosts), "Failed to reboot" assert do_check_clean_state(duthost), "state not good even after reboot" do_setup_tacacs(ptfhost, duthost, tacacs_creds) @@ -240,7 +272,15 @@ def test_ro_disk(localhost, ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, finally: logger.debug("START: reboot {} to restore disk RW state". format(enum_rand_one_per_hwsku_hostname)) - do_reboot(duthost, localhost, duthosts) + try: + if not do_reboot(duthost, localhost, duthosts): + logger.warning("Failed to reboot {}, try PDU reboot to restore disk RW state". + format(enum_rand_one_per_hwsku_hostname)) + do_pdu_reboot(duthost, localhost, duthosts, pdu_controller) + except Exception as e: + logger.warning("Failed to reboot {}, got exception {}, try PDU reboot to restore disk RW state". + format(enum_rand_one_per_hwsku_hostname, e)) + do_pdu_reboot(duthost, localhost, duthosts, pdu_controller) logger.debug(" END: reboot {} to restore disk RW state". format(enum_rand_one_per_hwsku_hostname)) diff --git a/tests/tacacs/test_ro_user.py b/tests/tacacs/test_ro_user.py index 63ee2c4856..3a7032a7e6 100644 --- a/tests/tacacs/test_ro_user.py +++ b/tests/tacacs/test_ro_user.py @@ -1,7 +1,8 @@ import pytest import time from tests.common.helpers.assertions import pytest_assert -from .utils import check_output, tacacs_running, start_tacacs_server +from tests.common.utilities import check_output +from tests.common.helpers.tacacs.tacacs_helper import ssh_remote_run, ssh_remote_run_retry import logging @@ -17,13 +18,6 @@ TIMEOUT_LIMIT = 120 -def ssh_remote_run(localhost, remote_ip, username, password, cmd): - res = localhost.shell("sshpass -p {} ssh " - "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null " - "{}@{} {}".format(password, username, remote_ip, cmd), module_ignore_errors=True) - return res - - def does_command_exist(localhost, remote_ip, username, password, command): usr_find_cmd = "find /usr -name {}".format(command) usr_result = ssh_remote_run(localhost, remote_ip, username, password, usr_find_cmd) @@ -44,7 +38,8 @@ def ssh_remote_allow_run(localhost, remote_ip, username, password, cmd): res = ssh_remote_run(localhost, remote_ip, username, password, cmd) # Verify that the command is allowed logger.info("check command \"{}\" rc={}".format(cmd, res['rc'])) - expected = "Make sure your account has RW permission to current device" not in res['stderr'] + expected = "Make sure your account has RW permission to current device" not in res['stderr'] \ + and "Permission denied" not in res['stderr'] if not expected: logger.error("error output=\"{}\"".format(res["stderr"])) return expected @@ -77,21 +72,6 @@ def wait_for_tacacs(localhost, remote_ip, username, password): current_attempt += 1 -def ssh_remote_run_retry(localhost, dutip, ptfhost, user, password, command, retry_count=3): - while retry_count > 0: - res = ssh_remote_run(localhost, dutip, user, - password, command) - - # TACACS server randomly crash after receive authorization request from IPV6 - if not tacacs_running(ptfhost): - start_tacacs_server(ptfhost) - retry_count -= 1 - else: - return res - - pytest_assert(False, "cat command failed because TACACS server not running") - - def test_ro_user(localhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds, check_tacacs): duthost = duthosts[enum_rand_one_per_hwsku_hostname] dutip = duthost.mgmt_ip diff --git a/tests/tacacs/test_rw_user.py b/tests/tacacs/test_rw_user.py index c934ea6594..68a7ce3f0e 100644 --- a/tests/tacacs/test_rw_user.py +++ b/tests/tacacs/test_rw_user.py @@ -1,7 +1,7 @@ import pytest -from .test_ro_user import ssh_remote_run, ssh_remote_run_retry -from .utils import check_output +from tests.common.helpers.tacacs.tacacs_helper import ssh_remote_run, ssh_remote_run_retry +from tests.common.utilities import check_output pytestmark = [ pytest.mark.disable_loganalyzer, diff --git a/tests/tacacs/utils.py b/tests/tacacs/utils.py index c2af5ef2a1..b3424cc04b 100644 --- a/tests/tacacs/utils.py +++ b/tests/tacacs/utils.py @@ -1,55 +1,18 @@ import binascii -import crypt import logging -import os import pytest -import re -import yaml - -from tests.common.errors import RunAnsibleModuleFail -from tests.common.utilities import wait_until, check_skip_release, delete_running_config +import paramiko +import time from tests.common.helpers.assertions import pytest_assert +from tests.common.helpers.tacacs.tacacs_helper import start_tacacs_server +from tests.common.utilities import wait_until, paramiko_ssh +from ansible.errors import AnsibleConnectionFailure logger = logging.getLogger(__name__) +TIMEOUT_LIMIT = 120 -# per-command authorization feature not available in following versions -per_command_authorization_skip_versions = ["201811", "201911", "202012", "202106"] - - -# per-command accounting feature not available in following versions -per_command_accounting_skip_versions = ["201811", "201911", "202106"] - - -def check_output(output, exp_val1, exp_val2): - pytest_assert(not output['failed'], output['stderr']) - for line in output['stdout_lines']: - fds = line.split(':') - if fds[0] == exp_val1: - pytest_assert(fds[4] == exp_val2) - - -def check_all_services_status(ptfhost): - res = ptfhost.command("service --status-all") - logger.info(res["stdout_lines"]) - - -def tacacs_running(ptfhost): - out = ptfhost.command("service tacacs_plus status", module_ignore_errors=True)["stdout"] - return "tacacs+ running" in out - - -def start_tacacs_server(ptfhost): - ptfhost.command("service tacacs_plus restart", module_ignore_errors=True) - return wait_until(5, 1, 0, tacacs_running, ptfhost) - - -def stop_tacacs_server(ptfhost): - def tacacs_not_running(ptfhost): - out = ptfhost.command("service tacacs_plus status", module_ignore_errors=True)["stdout"] - return "tacacs+ apparently not running" in out - ptfhost.shell("service tacacs_plus stop") - return wait_until(5, 1, 0, tacacs_not_running, ptfhost) +DEVICE_UNREACHABLE_MAX_RETRIES = 3 @pytest.fixture @@ -62,258 +25,6 @@ def ensure_tacacs_server_running_after_ut(duthosts, enum_rand_one_per_hwsku_host start_tacacs_server(duthost) -def setup_local_user(duthost, tacacs_creds): - try: - duthost.shell("sudo deluser {}".format(tacacs_creds['local_user'])) - except RunAnsibleModuleFail: - logger.info("local user not exist") - - duthost.shell("sudo useradd {}".format(tacacs_creds['local_user'])) - duthost.shell('sudo echo "{}:{}" | chpasswd'.format(tacacs_creds['local_user'], tacacs_creds['local_user_passwd'])) - - -def setup_tacacs_client(duthost, tacacs_creds, tacacs_server_ip, - tacacs_server_passkey, ptfhost, authorization="local"): - """setup tacacs client""" - - # UT should failed when set reachable TACACS server with this setup_tacacs_client - ping_result = duthost.shell("ping {} -c 1 -W 3".format(tacacs_server_ip))['stdout'] - logger.info("TACACS server ping result: {}".format(ping_result)) - if "100% packet loss" in ping_result: - # collect more information for debug testbed network issue - duthost_interface = duthost.shell("sudo ifconfig eth0")['stdout'] - ptfhost_interface = ptfhost.shell("ifconfig mgmt")['stdout'] - logger.debug("PTF IPV6 address not reachable, dut interfaces: {}, ptfhost interfaces:{}" - .format(duthost_interface, ptfhost_interface)) - pytest_assert(False, "TACACS server not reachable: {}".format(ping_result)) - - # configure tacacs client - default_tacacs_servers = [] - duthost.shell("sudo config tacacs passkey %s" % tacacs_server_passkey) - - # get default tacacs servers - config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] - for tacacs_server in config_facts.get('TACPLUS_SERVER', {}): - duthost.shell("sudo config tacacs delete %s" % tacacs_server) - default_tacacs_servers.append(tacacs_server) - duthost.shell("sudo config tacacs add %s" % tacacs_server_ip) - duthost.shell("sudo config tacacs authtype login") - - # enable tacacs+ - duthost.shell("sudo config aaa authentication login tacacs+") - - (skip, _) = check_skip_release(duthost, per_command_authorization_skip_versions) - if not skip: - duthost.shell("sudo config aaa authorization {}".format(authorization)) - - (skip, _) = check_skip_release(duthost, per_command_accounting_skip_versions) - if not skip: - duthost.shell("sudo config aaa accounting disable") - - # setup local user - setup_local_user(duthost, tacacs_creds) - return default_tacacs_servers - - -def restore_tacacs_servers(duthost): - # Restore the TACACS plus server in config_db.json - config_facts = duthost.config_facts(host=duthost.hostname, source="persistent")["ansible_facts"] - for tacacs_server in config_facts.get("TACPLUS_SERVER", {}): - duthost.shell("sudo config tacacs add %s" % tacacs_server) - - cmds = [] - aaa_config = config_facts.get("AAA", {}) - if aaa_config: - cfg = aaa_config.get("authentication", {}).get("login", "") - if cfg: - cmds.append("sonic-db-cli CONFIG_DB hset 'AAA|authentication' login %s" % cfg) - - cfg = aaa_config.get("authentication", {}).get("failthrough", "") - if cfg.lower() == "true": - cmds.append("config aaa authentication failthrough enable") - elif cfg.lower() == "false": - cmds.append("config aaa authentication failthrough disable") - - cfg = aaa_config.get("authorization", {}).get("login", "") - if cfg: - cmds.append("sonic-db-cli CONFIG_DB hset 'AAA|authorization' login %s" % cfg) - - cfg = aaa_config.get("accounting", {}).get("login", "") - if cfg: - cmds.append("sonic-db-cli CONFIG_DB hset 'AAA|accounting' login %s" % cfg) - - tacplus_config = config_facts.get("TACPLUS", {}) - if tacplus_config: - cfg = tacplus_config.get("global", {}).get("auth_type", "") - if cfg: - cmds.append("config tacacs authtype %s" % cfg) - - cfg = tacplus_config.get("global", {}).get("passkey", "") - if cfg: - cmds.append("config tacacs passkey %s" % cfg) - - cfg = tacplus_config.get("global", {}).get("timeout", "") - if cfg: - cmds.append("config tacacs timeout %s" % cfg) - - # Cleanup AAA and TACPLUS config - delete_tacacs_json = [{"AAA": {}}, {"TACPLUS": {}}] - delete_running_config(delete_tacacs_json, duthost) - - # Restore AAA and TACPLUS config - duthost.shell_cmds(cmds=cmds) - - -def fix_symbolic_link_in_config(duthost, ptfhost, symbolic_link_path, path_to_be_fix=None): - """ - Fix symbolic link in tacacs config - Because tac_plus server not support regex in command name, and SONiC will send full path to tacacs server side - for authorization, so the 'python' and 'ld' path in tac_plus config file need fix. - """ - read_link_command = "readlink -f {0}".format(symbolic_link_path) - target_path = duthost.shell(read_link_command)['stdout'] - # Escape path string, will use it as regex in sed command. - - link_path_regex = re.escape(symbolic_link_path) - if path_to_be_fix is not None: - link_path_regex = re.escape(path_to_be_fix) - - target_path_regex = re.escape(target_path) - ptfhost.shell("sed -i 's|{0}|{1}|g' /etc/tacacs+/tac_plus.conf".format(link_path_regex, target_path_regex)) - - -def get_ld_path(duthost): - """ - Fix symbolic link in tacacs config - Because tac_plus server not support regex in command name, and SONiC will send full path to tacacs server side - for authorization, so the 'python' and 'ld' path in tac_plus config file need fix. - """ - find_ld_command = "find /lib/ -type f,l -regex '\/lib\/.*-linux-.*/ld-linux-.*\.so\.[0-9]*'" # noqa W605 - return duthost.shell(find_ld_command)['stdout'] - - -def fix_ld_path_in_config(duthost, ptfhost): - """ - Fix ld path in tacacs config - """ - ld_symbolic_link_path = get_ld_path(duthost) - if not ld_symbolic_link_path: - fix_symbolic_link_in_config(duthost, ptfhost, ld_symbolic_link_path, "/lib/arch-linux-abi/ld-linux-arch.so") - - -def setup_tacacs_server(ptfhost, tacacs_creds, duthost): - """setup tacacs server""" - - # configure tacacs server - extra_vars = {'tacacs_passkey': tacacs_creds[duthost.hostname]['tacacs_passkey'], - 'tacacs_rw_user': tacacs_creds['tacacs_rw_user'], - 'tacacs_rw_user_passwd': crypt.crypt(tacacs_creds['tacacs_rw_user_passwd'], 'abc'), - 'tacacs_ro_user': tacacs_creds['tacacs_ro_user'], - 'tacacs_ro_user_passwd': crypt.crypt(tacacs_creds['tacacs_ro_user_passwd'], 'abc'), - 'tacacs_authorization_user': tacacs_creds['tacacs_authorization_user'], - 'tacacs_authorization_user_passwd': crypt.crypt( - tacacs_creds['tacacs_authorization_user_passwd'], - 'abc'), - 'tacacs_jit_user': tacacs_creds['tacacs_jit_user'], - 'tacacs_jit_user_passwd': crypt.crypt(tacacs_creds['tacacs_jit_user_passwd'], 'abc'), - 'tacacs_jit_user_membership': tacacs_creds['tacacs_jit_user_membership']} - - dut_options = duthost.host.options['inventory_manager'].get_host(duthost.hostname).vars - dut_creds = tacacs_creds[duthost.hostname] - logger.debug("setup_tacacs_server: dut_options:{}".format(dut_options)) - if 'ansible_user' in dut_options and 'ansible_password' in dut_options: - duthost_admin_user = dut_options['ansible_user'] - duthost_admin_passwd = dut_options['ansible_password'] - logger.debug("setup_tacacs_server: update extra_vars with ansible_user and ansible_password.") - extra_vars['duthost_admin_user'] = duthost_admin_user - extra_vars['duthost_admin_passwd'] = crypt.crypt(duthost_admin_passwd, 'abc') - elif 'sonicadmin_user' in dut_creds and 'sonicadmin_password' in dut_creds: - logger.debug("setup_tacacs_server: update extra_vars with sonicadmin_user and sonicadmin_password.") - extra_vars['duthost_admin_user'] = dut_creds['sonicadmin_user'] - extra_vars['duthost_admin_passwd'] = crypt.crypt(dut_creds['sonicadmin_password'], 'abc') - elif 'sonicadmin_user' in dut_creds and 'ansible_altpasswords' in dut_creds: - logger.debug("setup_tacacs_server: update extra_vars with sonicadmin_user and ansible_altpasswords.") - extra_vars['duthost_admin_user'] = dut_creds['sonicadmin_user'] - extra_vars['duthost_admin_passwd'] = crypt.crypt(dut_creds['ansible_altpasswords'][0], 'abc') - else: - logger.debug("setup_tacacs_server: update extra_vars with sonic_login and sonic_password.") - extra_vars['duthost_admin_user'] = dut_creds['sonic_login'] - extra_vars['duthost_admin_passwd'] = crypt.crypt(dut_creds['sonic_password'], 'abc') - - if 'ansible_ssh_user' in dut_options and 'ansible_ssh_pass' in dut_options: - duthost_ssh_user = dut_options['ansible_ssh_user'] - duthost_ssh_passwd = dut_options['ansible_ssh_pass'] - logger.debug("setup_tacacs_server: update extra_vars with ansible_ssh_user and ansible_ssh_pass.") - extra_vars['duthost_ssh_user'] = duthost_ssh_user - extra_vars['duthost_ssh_passwd'] = crypt.crypt(duthost_ssh_passwd, 'abc') - else: - logger.debug("setup_tacacs_server: duthost options does not contains config for ansible_ssh_user.") - - ptfhost.host.options['variable_manager'].extra_vars.update(extra_vars) - ptfhost.template(src="tacacs/tac_plus.conf.j2", dest="/etc/tacacs+/tac_plus.conf") - - # Find 'python' command symbolic link target, and fix the tac_plus config file - fix_symbolic_link_in_config(duthost, ptfhost, "/usr/bin/python") - - # Find ld lib symbolic link target, and fix the tac_plus config file - fix_ld_path_in_config(duthost, ptfhost) - - # config TACACS+ to use debug flag: '-d 2058', so received data will write to /var/log/tac_plus.log - ptfhost.lineinfile( - path="/etc/default/tacacs+", - line="DAEMON_OPTS=\"-d 2058 -l /var/log/tac_plus.log -C /etc/tacacs+/tac_plus.conf\"", - regexp='^DAEMON_OPTS=.*' - ) - check_all_services_status(ptfhost) - - # FIXME: This is a short term mitigation, we need to figure out why \nthe tacacs+ server does not start - # reliably all of a sudden. - wait_until(5, 1, 0, start_tacacs_server, ptfhost) - check_all_services_status(ptfhost) - - -def cleanup_tacacs(ptfhost, tacacs_creds, duthost): - # stop tacacs server - stop_tacacs_server(ptfhost) - - # reset tacacs client configuration - remove_all_tacacs_server(duthost) - cmds = [ - "config tacacs default passkey", - "config aaa authentication login default", - "config aaa authentication failthrough default" - ] - duthost.shell_cmds(cmds=cmds) - - (skip, _) = check_skip_release(duthost, per_command_authorization_skip_versions) - if not skip: - duthost.shell("sudo config aaa authorization local") - - (skip, _) = check_skip_release(duthost, per_command_accounting_skip_versions) - if not skip: - duthost.shell("sudo config aaa accounting disable") - - duthost.user( - name=tacacs_creds['tacacs_ro_user'], state='absent', remove='yes', force='yes', module_ignore_errors=True - ) - duthost.user( - name=tacacs_creds['tacacs_rw_user'], state='absent', remove='yes', force='yes', module_ignore_errors=True - ) - duthost.user( - name=tacacs_creds['tacacs_jit_user'], state='absent', remove='yes', force='yes', module_ignore_errors=True - ) - - -def remove_all_tacacs_server(duthost): - # use grep command to extract tacacs server address from tacacs config - find_server_command = 'show tacacs | grep -Po "TACPLUS_SERVER address \K.*"' # noqa W605 - server_list = duthost.shell(find_server_command, module_ignore_errors=True)['stdout_lines'] - for tacacs_server in server_list: - tacacs_server = tacacs_server.rstrip() - if tacacs_server: - duthost.shell("sudo config tacacs delete %s" % tacacs_server) - - def check_server_received(ptfhost, data, timeout=30): """ Check if tacacs server received the data. @@ -380,7 +91,36 @@ def log_exist(duthost): pytest_assert(exist, "Not found aaa config update log: {}".format(command)) -def load_tacacs_creds(): - TACACS_CREDS_FILE = 'tacacs_creds.yaml' - creds_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), TACACS_CREDS_FILE) - return yaml.safe_load(open(creds_file_path).read()) +def ssh_run_command(ssh_client, command): + stdin, stdout, stderr = ssh_client.exec_command(command, timeout=TIMEOUT_LIMIT) + exit_code = stdout.channel.recv_exit_status() + return exit_code, stdout, stderr + + +def ssh_connect_remote_retry(remote_ip, remote_username, remote_password, duthost): + retry_count = 3 + while retry_count > 0: + try: + return paramiko_ssh(remote_ip, remote_username, remote_password) + except paramiko.ssh_exception.AuthenticationException as e: + logger.info("Paramiko SSH connect failed with authentication: " + repr(e)) + + # get syslog for debug + recent_syslog = duthost.shell('sudo tail -100 /var/log/syslog')['stdout'] + logger.debug("Target device syslog: {}".format(recent_syslog)) + + time.sleep(1) + retry_count -= 1 + + +def duthost_shell_with_unreachable_retry(duthost, command): + retries = 0 + while True: + try: + return duthost.shell(command) + except AnsibleConnectionFailure as e: + retries += 1 + logger.warning("retry_when_dut_unreachable exception: {}, retry {}/{}" + .format(e, retries, DEVICE_UNREACHABLE_MAX_RETRIES)) + if retries > DEVICE_UNREACHABLE_MAX_RETRIES: + raise e diff --git a/tests/telemetry/conftest.py b/tests/telemetry/conftest.py index 3bb360d5c3..1c2369d794 100644 --- a/tests/telemetry/conftest.py +++ b/tests/telemetry/conftest.py @@ -4,11 +4,9 @@ import sys from tests.common.helpers.assertions import pytest_assert as py_assert -from tests.common.errors import RunAnsibleModuleFail -from tests.common.utilities import wait_until, wait_tcp_connection, get_mgmt_ipv6 -from tests.common.helpers.gnmi_utils import GNMIEnvironment -from tests.telemetry.telemetry_utils import get_list_stdout, setup_telemetry_forpyclient, restore_telemetry_forpyclient -from contextlib import contextmanager +from tests.common.utilities import wait_until +from tests.telemetry.telemetry_utils import get_list_stdout +from tests.common.helpers.telemetry_helper import _context_for_setup_streaming_telemetry EVENTS_TESTS_PATH = "./telemetry/events" sys.path.append(EVENTS_TESTS_PATH) @@ -19,25 +17,6 @@ logger = logging.getLogger(__name__) -@pytest.fixture(scope="module") -def gnxi_path(ptfhost): - """ - gnxi's location is updated from /gnxi to /root/gnxi - in RP https://github.com/sonic-net/sonic-buildimage/pull/10599. - But old docker-ptf images don't have this update, - test case will fail for these docker-ptf images, - because it should still call /gnxi files. - For avoiding this conflict, check gnxi path before test and set GNXI_PATH to correct value. - Add a new gnxi_path module fixture to make sure to set GNXI_PATH before test. - """ - path_exists = ptfhost.stat(path="/root/gnxi/") - if path_exists["stat"]["exists"] and path_exists["stat"]["isdir"]: - gnxipath = "/root/gnxi/" - else: - gnxipath = "/gnxi/" - return gnxipath - - @pytest.fixture(scope="module", autouse=True) def verify_telemetry_dockerimage(duthosts, enum_rand_one_per_hwsku_hostname): """If telemetry docker is available in image then return true @@ -51,41 +30,6 @@ def verify_telemetry_dockerimage(duthosts, enum_rand_one_per_hwsku_hostname): pytest.skip("docker-sonic-gnmi and docker-sonic-telemetry are not part of the image") -def check_gnmi_config(duthost): - cmd = 'sonic-db-cli CONFIG_DB HGET "GNMI|gnmi" port' - port = duthost.shell(cmd, module_ignore_errors=False)['stdout'] - return port != "" - - -def create_gnmi_config(duthost): - cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|gnmi' port 50052" - duthost.shell(cmd, module_ignore_errors=True) - cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|gnmi' client_auth true" - duthost.shell(cmd, module_ignore_errors=True) - cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|certs' "\ - "ca_crt /etc/sonic/telemetry/dsmsroot.cer" - duthost.shell(cmd, module_ignore_errors=True) - cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|certs' "\ - "server_crt /etc/sonic/telemetry/streamingtelemetryserver.cer" - duthost.shell(cmd, module_ignore_errors=True) - cmd = "sonic-db-cli CONFIG_DB hset 'GNMI|certs' "\ - "server_key /etc/sonic/telemetry/streamingtelemetryserver.key" - duthost.shell(cmd, module_ignore_errors=True) - - -def delete_gnmi_config(duthost): - cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|gnmi' port" - duthost.shell(cmd, module_ignore_errors=True) - cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|gnmi' client_auth" - duthost.shell(cmd, module_ignore_errors=True) - cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|certs' ca_crt" - duthost.shell(cmd, module_ignore_errors=True) - cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|certs' server_crt" - duthost.shell(cmd, module_ignore_errors=True) - cmd = "sonic-db-cli CONFIG_DB hdel 'GNMI|certs' server_key" - duthost.shell(cmd, module_ignore_errors=True) - - @pytest.fixture(scope="module") def setup_streaming_telemetry(request, duthosts, enum_rand_one_per_hwsku_hostname, localhost, ptfhost, gnxi_path): with _context_for_setup_streaming_telemetry(request, duthosts, enum_rand_one_per_hwsku_hostname, @@ -93,66 +37,6 @@ def setup_streaming_telemetry(request, duthosts, enum_rand_one_per_hwsku_hostnam yield result -@pytest.fixture(scope="function") -def setup_streaming_telemetry_func(request, duthosts, enum_rand_one_per_hwsku_hostname, localhost, ptfhost, gnxi_path): - with _context_for_setup_streaming_telemetry(request, duthosts, enum_rand_one_per_hwsku_hostname, - localhost, ptfhost, gnxi_path) as result: - yield result - - -@contextmanager -def _context_for_setup_streaming_telemetry(request, duthosts, enum_rand_one_per_hwsku_hostname, - localhost, ptfhost, gnxi_path): - """ - @summary: Post setting up the streaming telemetry before running the test. - """ - is_ipv6 = request.param - try: - duthost = duthosts[enum_rand_one_per_hwsku_hostname] - has_gnmi_config = check_gnmi_config(duthost) - if not has_gnmi_config: - create_gnmi_config(duthost) - env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) - default_client_auth = setup_telemetry_forpyclient(duthost) - - if default_client_auth == "true": - duthost.shell('sonic-db-cli CONFIG_DB HSET "%s|gnmi" "client_auth" "false"' % (env.gnmi_config_table), - module_ignore_errors=False) - duthost.shell("systemctl reset-failed %s" % (env.gnmi_container)) - duthost.service(name=env.gnmi_container, state="restarted") - else: - logger.info('client auth is false. No need to restart telemetry') - - # Wait until telemetry was restarted - py_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, env.gnmi_container), - "%s not started." % (env.gnmi_container)) - logger.info("telemetry process restarted. Now run pyclient on ptfdocker") - - # Wait until the TCP port was opened - dut_ip = duthost.mgmt_ip - if is_ipv6: - dut_ip = get_mgmt_ipv6(duthost) - wait_tcp_connection(localhost, dut_ip, env.gnmi_port, timeout_s=60) - - # pyclient should be available on ptfhost. If it was not available, then fail pytest. - if is_ipv6: - cmd = "docker cp %s:/usr/sbin/gnmi_get ~/" % (env.gnmi_container) - ret = duthost.shell(cmd)['rc'] - py_assert(ret == 0) - else: - file_exists = ptfhost.stat(path=gnxi_path + "gnmi_cli_py/py_gnmicli.py") - py_assert(file_exists["stat"]["exists"] is True) - except RunAnsibleModuleFail as e: - logger.info("Error happens in the setup period of setup_streaming_telemetry, recover the telemetry.") - restore_telemetry_forpyclient(duthost, default_client_auth) - raise e - - yield - restore_telemetry_forpyclient(duthost, default_client_auth) - if not has_gnmi_config: - delete_gnmi_config(duthost) - - def do_init(duthost): for i in [BASE_DIR, DATA_DIR]: try: @@ -164,17 +48,29 @@ def do_init(duthost): @pytest.fixture(scope="module") -def test_eventd_healthy(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, setup_streaming_telemetry, gnxi_path): +def test_eventd_healthy(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, ptfadapter, + setup_streaming_telemetry, gnxi_path): """ @summary: Test eventd heartbeat before testing all testcases """ duthost = duthosts[enum_rand_one_per_hwsku_hostname] + if duthost.is_multi_asic: + pytest.skip("Skip eventd testing on multi-asic") + + features_dict, succeeded = duthost.get_feature_status() + if succeeded and ('eventd' not in features_dict or features_dict['eventd'] == 'disabled'): + pytest.skip("eventd is disabled on the system") + do_init(duthost) module = __import__("eventd_events") - module.test_event(duthost, gnxi_path, ptfhost, DATA_DIR, None) + duthost.shell("systemctl restart eventd") + + py_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, "eventd"), "eventd not started.") + + module.test_event(duthost, gnxi_path, ptfhost, ptfadapter, DATA_DIR, None) logger.info("Completed test file: {}".format("eventd_events test completed.")) diff --git a/tests/telemetry/events/bgp_events.py b/tests/telemetry/events/bgp_events.py index c407d89183..e72c46a187 100644 --- a/tests/telemetry/events/bgp_events.py +++ b/tests/telemetry/events/bgp_events.py @@ -9,7 +9,7 @@ tag = "sonic-events-bgp" -def test_event(duthost, gnxi_path, ptfhost, data_dir, validate_yang): +def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, drop_tcp_packets, "bgp_notification.json", "sonic-events-bgp:notification", tag) run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, shutdown_bgp_neighbors, diff --git a/tests/telemetry/events/dhcp-relay_events.py b/tests/telemetry/events/dhcp-relay_events.py new file mode 100644 index 0000000000..331cdeaece --- /dev/null +++ b/tests/telemetry/events/dhcp-relay_events.py @@ -0,0 +1,110 @@ +#! /usr/bin/env python3 + +import pytest +import logging +import time +import ptf.testutils as testutils + +from tests.common.helpers.assertions import pytest_assert as py_assert +from tests.common.utilities import wait_until +from run_events_test import run_test +from event_utils import find_test_vlan, find_test_port_and_mac, create_dhcp_discover_packet + +logger = logging.getLogger(__name__) +tag = "sonic-events-dhcp-relay" + + +def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): + features_states, succeeded = duthost.get_feature_status() + if not succeeded or features_states["dhcp_relay"] != "enabled": + pytest.skip("dhcp_relay is not enabled, skipping dhcp_relay events") + logger.info("Beginning to test dhcp-relay events") + run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_dhcp_relay_discard, + "dhcp_relay_discard.json", "sonic-events-dhcp-relay:dhcp-relay-discard", tag, False, 30, ptfadapter) + run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_dhcp_relay_disparity, + "dhcp_relay_disparity.json", "sonic-events-dhcp-relay:dhcp-relay-disparity", tag, False, 30, ptfadapter) + run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_dhcp_relay_bind_failure, + "dhcp_relay_bind_failure.json", "sonic-events-dhcp-relay:dhcp-relay-bind-failure", tag, False, 30) + + +def trigger_dhcp_relay_discard(duthost, ptfadapter): + send_dhcp_discover_packets(duthost, ptfadapter) + + +def trigger_dhcp_relay_disparity(duthost, ptfadapter): + """11 packets because dhcpmon process will store up to 10 unhealthy status events + https://github.com/sonic-net/sonic-dhcpmon/blob/master/src/dhcp_mon.cpp#L94 + static int dhcp_unhealthy_max_count = 10; + Sending at interval of 18 seconds because dhcpmon process will check health at that interval + static int window_interval_sec = 18; + """ + send_dhcp_discover_packets(duthost, ptfadapter, 11, 18) + + +def trigger_dhcp_relay_bind_failure(duthost): + # Flush ipv6 vlan address and restart dhc6relay process + py_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, "dhcp_relay"), + "dhcp_relay container not started") + + # Get Vlan with IPv6 address configured + dhcp_test_info = find_test_vlan(duthost) + py_assert(len(dhcp_test_info) != 0, "Unable to find vlan for test") + + vlan = dhcp_test_info["vlan"] + dhcp6_relay_process = dhcp_test_info["dhcp6relay_process"] + ipv6_ip = dhcp_test_info["ipv6_address"] + + try: + # Flush ipv6 address from vlan + duthost.shell("ip -6 address flush dev {}".format(vlan)) + + # Restart dhcrelay process + duthost.shell("docker exec dhcp_relay supervisorctl restart {}".format(dhcp6_relay_process)) + + finally: + # Add back ipv6 address to vlan + duthost.shell("ip address add {} dev {}".format(ipv6_ip, vlan)) + + # Restart dhcrelay process + duthost.shell("docker exec dhcp_relay supervisorctl restart {}".format(dhcp6_relay_process)) + + +def send_dhcp_discover_packets(duthost, ptfadapter, packets_to_send=5, interval=1): + py_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, "dhcp_relay"), + "dhcp_relay container not started") + + # Get Vlan with IPv4 address configured + dhcp_test_info = find_test_vlan(duthost) + py_assert(len(dhcp_test_info) != 0, "Unable to find vlan for test") + + vlan = dhcp_test_info["vlan"] + dhcrelay_process = dhcp_test_info["dhcrelay_process"] + ipv4_ip = dhcp_test_info["ipv4_address"] + member_interfaces = dhcp_test_info["member_interface"] + + try: + # Flush ipv4 address from vlan + duthost.shell("ip -4 address flush dev {}".format(vlan)) + + # Restart dhcrelay process + duthost.shell("docker exec dhcp_relay supervisorctl restart {}".format(dhcrelay_process)) + + # Send packets + + # results contains up to 5 tuples of member interfaces from vlan (port, mac address) + results = find_test_port_and_mac(duthost, member_interfaces, 5) + + for i in range(packets_to_send): + result = results[i % len(results)] + port = result[0] + client_mac = result[1] + packet = create_dhcp_discover_packet(client_mac) + testutils.send_packet(ptfadapter, port, packet) + time.sleep(interval) + + finally: + # Add back ipv4 address to vlan + duthost.shell("ip address add {} dev {}".format(ipv4_ip, vlan)) + + # Restart dhcrelay process + duthost.shell("docker exec dhcp_relay supervisorctl restart {}".format(dhcrelay_process)) diff --git a/tests/telemetry/events/event_utils.py b/tests/telemetry/events/event_utils.py index 10a3316574..d71aaa5e54 100644 --- a/tests/telemetry/events/event_utils.py +++ b/tests/telemetry/events/event_utils.py @@ -1,7 +1,8 @@ import logging -import os -import json -import re +import pytest + +import ptf.packet as scapy +import ptf.testutils as testutils from tests.common.utilities import wait_until from tests.common.helpers.assertions import pytest_assert @@ -12,6 +13,21 @@ PUBLISHED = 1 +def add_test_watchdog_timeout_service(duthost): + logger.info("Adding mock watchdog.service to systemd") + duthost.copy(src="telemetry/events/events_data/test-watchdog-timeout.service", dest="/etc/systemd/system/") + duthost.shell("systemctl daemon-reload") + duthost.shell("systemctl start test-watchdog-timeout.service") + + +def delete_test_watchdog_timeout_service(duthost): + logger.info("Deleting mock test-watchdog-timeout.service") + duthost.shell("systemctl stop test-watchdog-timeout.service", module_ignore_errors=True) + duthost.shell("rm /etc/systemd/system/test-watchdog-timeout.service", module_ignore_errors=True) + duthost.shell("systemctl daemon-reload") + duthost.shell("systemctl reset-failed") + + def backup_monit_config(duthost): logger.info("Backing up monit config files") duthost.shell("cp -f /etc/monit/monitrc ~/") @@ -53,17 +69,6 @@ def check_monit_running(duthost): return monit_services_status -def create_ip_file(duthost, data_dir, json_file, start_idx, end_idx): - ip_file = os.path.join(data_dir, json_file) - with open(ip_file, "w") as f: - for i in range(start_idx, end_idx + 1): - json_string = f'{{"test-event-source:test": {{"test_key": "test_val_{i}"}}}}' - f.write(json_string + '\n') - dest = "~/" + json_file - duthost.copy(src=ip_file, dest=dest) - duthost.shell("docker cp {} eventd:/".format(dest)) - - def event_publish_tool(duthost, json_file='', count=1): cmd = "docker exec eventd python /usr/bin/events_publish_tool.py" if json_file == '': @@ -74,18 +79,13 @@ def event_publish_tool(duthost, json_file='', count=1): assert ret["rc"] == 0, "Unable to publish events via events_publish_tool.py" -def verify_received_output(received_file, N): - key = "test_key" - with open(received_file, 'r') as file: - json_array = json.load(file) - pytest_assert(len(json_array) == N, "Expected {} events, but found {}".format(N, len(json_array))) - for i in range(0, len(json_array)): - block = json_array[i]["test-event-source:test"] - pytest_assert(key in block and len(re.findall('test_val_{}'.format(i + 1), block[key])) > 0, - "Missing key or incorrect value") - - def restart_eventd(duthost): + if duthost.is_multi_asic: + pytest.skip("Skip eventd testing on multi-asic") + features_dict, succeeded = duthost.get_feature_status() + if succeeded and ('eventd' not in features_dict or features_dict['eventd'] == 'disabled'): + pytest.skip("eventd is disabled on the system") + duthost.shell("systemctl reset-failed eventd") duthost.service(name="eventd", state="restarted") pytest_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, "eventd"), "eventd not started") @@ -112,3 +112,58 @@ def verify_counter_increase(duthost, current_value, increase, stat): current_counters = read_event_counters(duthost) current_stat_counter = current_counters[stat] return current_stat_counter >= current_value + increase + + +def find_test_vlan(duthost): + """Returns vlan information for dhcp_relay tests + Returns dictionary of vlan port name, dhcrelay process name, ipv4 address, + dhc6relay process name, ipv6 address, and member interfaces + """ + vlan_brief = duthost.get_vlan_brief() + for vlan in vlan_brief: + # Find dhcrelay process + dhcrelay_process = duthost.shell("docker exec dhcp_relay supervisorctl status \ + | grep isc-dhcpv4-relay-%s | awk '{print $1}'" % vlan)['stdout'] + dhcp6relay_process = duthost.shell("docker exec dhcp_relay supervisorctl status \ + | grep dhcp6relay | awk '{print $1}'")['stdout'] + interface_ipv4 = vlan_brief[vlan]['interface_ipv4'] + interface_ipv6 = vlan_brief[vlan]['interface_ipv6'] + members = vlan_brief[vlan]['members'] + + # Check all returning fields are non empty + results = [dhcrelay_process, interface_ipv4, dhcp6relay_process, interface_ipv6, members] + if all(result for result in results): + return { + "vlan": vlan, + "dhcrelay_process": dhcrelay_process, + "ipv4_address": interface_ipv4[0], + "dhcp6relay_process": dhcp6relay_process, + "ipv6_address": interface_ipv6[0], + "member_interface": members + } + return {} + + +def find_test_port_and_mac(duthost, members, count): + # Will return up to count many up ports with their port index and mac address + results = [] + interf_status = duthost.show_interface(command="status")['ansible_facts']['int_status'] + for member_interface in members: + if len(results) == count: + return results + if interf_status[member_interface]['admin_state'] == "up": + mac = duthost.get_dut_iface_mac(member_interface) + minigraph_info = duthost.minigraph_facts(host=duthost.hostname)['ansible_facts'] + port_index = minigraph_info['minigraph_port_indices'][member_interface] + if mac != "" and port_index != "": + results.append([int(port_index), mac]) + return results + + +def create_dhcp_discover_packet(client_mac): + dst_mac = 'ff:ff:ff:ff:ff:ff' + dhcp_client_port = 68 + discover_packet = testutils.dhcp_discover_packet(eth_client=client_mac, set_broadcast_bit=True) + discover_packet[scapy.Ether].dst = dst_mac + discover_packet[scapy.IP].sport = dhcp_client_port + return discover_packet diff --git a/tests/telemetry/events/eventd_events.py b/tests/telemetry/events/eventd_events.py index b6288114fe..366230d14f 100644 --- a/tests/telemetry/events/eventd_events.py +++ b/tests/telemetry/events/eventd_events.py @@ -8,7 +8,7 @@ tag = "sonic-events-eventd" -def test_event(duthost, gnxi_path, ptfhost, data_dir, validate_yang): +def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): logger.info("Beginning to test eventd heartbeat") run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, None, "heartbeat.json", "sonic-events-eventd:heartbeat", tag, True) diff --git a/tests/telemetry/events/events_data/test-watchdog-timeout.service b/tests/telemetry/events/events_data/test-watchdog-timeout.service new file mode 100644 index 0000000000..1e9263cae0 --- /dev/null +++ b/tests/telemetry/events/events_data/test-watchdog-timeout.service @@ -0,0 +1,13 @@ +[Unit] +Description=Test Watchdog Timeout + +[Service] +Type=simple +# This service will sleep for 2 minutes therefore not sending the signal to watchdog within the specified 1 min requirement +# Since SIGABRT will be sent after watchdog times out due to no signal, it will crash the bash process and dump core +# Added logic to trap the SIGABRT so that no core dump is dropped. +ExecStart=/bin/bash -c 'trap "" SIGABRT; sleep 120' +WatchdogSec=60s + +[Install] +WantedBy=multi-user.target diff --git a/tests/telemetry/events/host_events.py b/tests/telemetry/events/host_events.py index 83703fbc8d..24cacc521d 100644 --- a/tests/telemetry/events/host_events.py +++ b/tests/telemetry/events/host_events.py @@ -4,17 +4,22 @@ import time from run_events_test import run_test from event_utils import backup_monit_config, customize_monit_config, restore_monit_config +from event_utils import add_test_watchdog_timeout_service, delete_test_watchdog_timeout_service from telemetry_utils import trigger_logger from tests.common.helpers.dut_utils import is_container_running +from tests.common.utilities import wait_until logger = logging.getLogger(__name__) tag = "sonic-events-host" -def test_event(duthost, gnxi_path, ptfhost, data_dir, validate_yang): +def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): logger.info("Beginning to test host events") run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger_kernel_event, "event_kernel.json", "sonic-events-host:event-kernel", tag, False) + run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, kill_critical_process, + "process_exited_unexpectedly.json", "sonic-events-host:process-exited-unexpectedly", + tag, False) backup_monit_config(duthost) customize_monit_config( duthost, @@ -34,10 +39,18 @@ def test_event(duthost, gnxi_path, ptfhost, data_dir, validate_yang): "mem_threshold_exceeded.json", "sonic-events-host:mem-threshold-exceeded", tag) run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, restart_container, "event_stopped_ctr.json", "sonic-events-host:event-stopped-ctr", tag, False) - run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, mask_container, + run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, stop_container, "event_down_ctr.json", "sonic-events-host:event-down-ctr", tag, False) finally: restore_monit_config(duthost) + add_test_watchdog_timeout_service(duthost) + try: + # We need to alot flat 60 seconds for watchdog timeout to fire since the timer is set to 60\ + # With a base limit of 30 seconds, we will use 90 seconds + run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, None, + "watchdog_timeout.json", "sonic-events-host:watchdog-timeout", tag, False, 90) + finally: + delete_test_watchdog_timeout_service(duthost) def trigger_mem_threshold_exceeded_alert(duthost): @@ -55,37 +68,81 @@ def trigger_kernel_event(duthost): trigger_logger(duthost, "zlib decompression failed, data probably corrupt", "kernel") +def is_container_down(duthost, container): + return not is_container_running(duthost, container) + + def get_running_container(duthost): logger.info("Check if acms or snmp container is running") - container = "acms" - container_running = is_container_running(duthost, container) - if not container_running: - container = "snmp" + if is_container_running(duthost, "acms"): + return "acms" + elif is_container_running(duthost, "snmp"): + return "snmp" else: - return container - container_running = is_container_running(duthost, container) - if not container_running: return "" - return container + + +def get_critical_process(duthost): + logger.info("Check if snmpd/bgpd process is running") + if is_container_running(duthost, "snmp"): + pid = duthost.shell("docker exec snmp pgrep -f sonic_ax_impl")["stdout"] + if pid != "": + return pid, "snmp" + if is_container_running(duthost, "bpg"): + pid = duthost.shell("docker exec bgp pgrep -f bpgd")["stdout"] + if pid != "": + return pid, "bgpd" + return "", "" def restart_container(duthost): logger.info("Stopping container for event stopped event") container = get_running_container(duthost) assert container != "", "No available container for testing" - + duthost.shell("systemctl reset-failed {}".format(container)) duthost.shell("systemctl restart {}".format(container)) + is_container_running = wait_until(100, 10, 0, duthost.is_service_fully_started, container) + assert is_container_running, "{} not running after restart".format(container) -def mask_container(duthost): - logger.info("Masking container for event down event") +def stop_container(duthost): + logger.info("Stop container for event down event") container = get_running_container(duthost) assert container != "", "No available container for testing" - duthost.shell("systemctl mask {}".format(container)) + duthost.shell("config feature autorestart {} disabled".format(container)) duthost.shell("docker stop {}".format(container)) + output = duthost.shell("sonic-db-cli STATE_DB hget \"FEATURE|{}\" \"container_id\"".format(container))['stdout'] + if output: + duthost.shell("sonic-db-cli STATE_DB hset \"FEATURE|{}\" \"container_id\" \"\"".format(container)) time.sleep(30) # Wait 30 seconds for container_checker to fire event - duthost.shell("systemctl unmask {}".format(container)) + if output: + duthost.shell("sonic-db-cli STATE_DB hset \"FEATURE|{}\" \"container_id\" {}".format(container, output)) + + duthost.shell("config feature autorestart {} enabled".format(container)) duthost.shell("systemctl restart {}".format(container)) + + +def kill_critical_process(duthost): + logger.info("Killing critical process for exited unexpectedly event") + pid, container = get_critical_process(duthost) + assert pid != "", "No available process for testing" + + change_autorestart = False + autorestart = duthost.shell("show feature autorestart {}".format(container))['stdout_lines'] + if "disabled" in str(autorestart): + change_autorestart = True + duthost.shell("config feature autorestart {} enabled".format(container)) + + duthost.shell("docker exec {} kill -9 {}".format(container, pid), module_ignore_errors=True) + + # Wait until specified container is not running because of critical process exit + wait_until(30, 5, 0, is_container_down, duthost, container) + + if change_autorestart: + duthost.shell("config feature autorestart {} disabled".format(container)) + + duthost.shell("systemctl reset-failed {}".format(container), module_ignore_errors=True) + wait_until(100, 10, 0, duthost.is_service_fully_started, container) diff --git a/tests/telemetry/events/run_events_test.py b/tests/telemetry/events/run_events_test.py index 64384ddb2e..ba9c6b4fa0 100644 --- a/tests/telemetry/events/run_events_test.py +++ b/tests/telemetry/events/run_events_test.py @@ -11,12 +11,15 @@ def run_test(duthost, gnxi_path, ptfhost, data_dir, validate_yang, trigger, json_file, - filter_event_regex, tag, heartbeat=False, thread_timeout=30): + filter_event_regex, tag, heartbeat=False, timeout=30, ptfadapter=None): op_file = os.path.join(data_dir, json_file) if trigger is not None: # no trigger for heartbeat - trigger(duthost) # add events to cache + if ptfadapter is None: + trigger(duthost) # add events to cache + else: + trigger(duthost, ptfadapter) listen_for_events(duthost, gnxi_path, ptfhost, filter_event_regex, op_file, - thread_timeout) # listen from cache + timeout) # listen from cache data = {} with open(op_file, "r") as f: data = json.load(f) diff --git a/tests/telemetry/events/swss_events.py b/tests/telemetry/events/swss_events.py index f7e8da1eeb..e9aa6a2eeb 100644 --- a/tests/telemetry/events/swss_events.py +++ b/tests/telemetry/events/swss_events.py @@ -28,7 +28,7 @@ WAIT_TIME = 3 -def test_event(duthost, gnxi_path, ptfhost, data_dir, validate_yang): +def test_event(duthost, gnxi_path, ptfhost, ptfadapter, data_dir, validate_yang): if duthost.topo_type.lower() in ["m0", "mx"]: logger.info("Skipping swss events test on MGFX topologies") return diff --git a/tests/telemetry/telemetry_utils.py b/tests/telemetry/telemetry_utils.py index 3f10cb5bed..ef4cac780e 100644 --- a/tests/telemetry/telemetry_utils.py +++ b/tests/telemetry/telemetry_utils.py @@ -5,7 +5,6 @@ from pkg_resources import parse_version from tests.common.helpers.assertions import pytest_assert -from tests.common.utilities import InterruptableThread from tests.common.helpers.gnmi_utils import GNMIEnvironment logger = logging.getLogger(__name__) @@ -53,30 +52,6 @@ def skip_201911_and_older(duthost): pytest.skip("Test not supported for 201911 images. Skipping the test") -def setup_telemetry_forpyclient(duthost): - """ Set client_auth=false. This is needed for pyclient to successfully set up channel with gnmi server. - Restart telemetry process - """ - env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) - client_auth_out = duthost.shell('sonic-db-cli CONFIG_DB HGET "%s|gnmi" "client_auth"' % (env.gnmi_config_table), - module_ignore_errors=False)['stdout_lines'] - client_auth = str(client_auth_out[0]) - return client_auth - - -def restore_telemetry_forpyclient(duthost, default_client_auth): - env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) - client_auth_out = duthost.shell('sonic-db-cli CONFIG_DB HGET "%s|gnmi" "client_auth"' % (env.gnmi_config_table), - module_ignore_errors=False)['stdout_lines'] - client_auth = str(client_auth_out[0]) - if client_auth != default_client_auth: - duthost.shell('sonic-db-cli CONFIG_DB HSET "%s|gnmi" "client_auth" %s' - % (env.gnmi_config_table, default_client_auth), - module_ignore_errors=False) - duthost.shell("systemctl reset-failed %s" % (env.gnmi_container)) - duthost.service(name=env.gnmi_container, state="restarted") - - def check_gnmi_cli_running(ptfhost): program_list = ptfhost.shell("pgrep -f 'python /root/gnxi/gnmi_cli_py/py_gnmicli.py'")["stdout"] return len(program_list) > 0 @@ -97,25 +72,17 @@ def fetch_json_ptf_output(regex, output, match_no): return match[:match_no+1] -def listen_for_event(ptfhost, cmd, results): - ret = ptfhost.shell(cmd) - assert ret["rc"] == 0, "PTF docker was not able to query EVENTS path" - results[0] = ret["stdout"] - - -def listen_for_events(duthost, gnxi_path, ptfhost, filter_event_regex, op_file, thread_timeout, update_count=1, +def listen_for_events(duthost, gnxi_path, ptfhost, filter_event_regex, op_file, timeout, update_count=1, match_number=0): cmd = generate_client_cli(duthost=duthost, gnxi_path=gnxi_path, method=METHOD_SUBSCRIBE, submode=SUBMODE_ONCHANGE, update_count=update_count, xpath="all[heartbeat=2]", - target="EVENTS", filter_event_regex=filter_event_regex) - results = [""] - event_thread = InterruptableThread(target=listen_for_event, args=(ptfhost, cmd, results,)) - event_thread.start() - event_thread.join(thread_timeout) # close thread after 30 sec, was not able to find event within reasonable time - assert results[0] != "", "No output from PTF docker, thread timed out after {} seconds".format(thread_timeout) + target="EVENTS", filter_event_regex=filter_event_regex, timeout=timeout) + result = ptfhost.shell(cmd) + assert result["rc"] == 0, "PTF command failed with non zero return code" + output = result["stdout"] + assert len(output) != 0, "No output from PTF docker, thread timed out after {} seconds".format(timeout) # regex logic and then to write to file - result = results[0] - event_strs = fetch_json_ptf_output(EVENT_REGEX, result, match_number) + event_strs = fetch_json_ptf_output(EVENT_REGEX, output, match_number) with open(op_file, "w") as f: f.write("[\n") for i in range(0, len(event_strs)): @@ -139,8 +106,22 @@ def trigger_logger(duthost, log, process, container="", priority="local0.notice" def generate_client_cli(duthost, gnxi_path, method=METHOD_GET, xpath="COUNTERS/Ethernet0", target="COUNTERS_DB", subscribe_mode=SUBSCRIBE_MODE_STREAM, submode=SUBMODE_SAMPLE, - intervalms=0, update_count=3, create_connections=1, filter_event_regex=""): + intervalms=0, update_count=3, create_connections=1, filter_event_regex="", + timeout=-1): """ Generate the py_gnmicli command line based on the given params. + t --target: gNMI target; required + p --port: port of target; required + m --mode: get/susbcribe; default get + x --xpath: gnmi path, table name; required + xt --xpath_target: gnmi path prefix, db name + o --host_override, targets hostname for certificate CN + subscribe_mode: 0=STREAM, 1=ONCE, 2=POLL; default 0 + submode: 0=TARGET_DEFINED, 1=ON_CHANGE, 2=SAMPLE; default 2 + interval: sample interval in milliseconds, default 10000ms + update_count: Max number of streaming updates to receive. 0 means no limit. default 0 + create_connections: Creates TCP connections with gNMI server; default 1; -1 for infinite connections + filter_event_regex: Regex to filter event when querying events path + timeout: Subscription duration in seconds; After X seconds, request terminates; default none """ env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) cmdFormat = 'python ' + gnxi_path + 'gnmi_cli_py/py_gnmicli.py -g -t {0} -p {1} -m {2} -x {3} -xt {4} -o {5}' @@ -153,4 +134,61 @@ def generate_client_cli(duthost, gnxi_path, method=METHOD_GET, xpath="COUNTERS/E update_count, create_connections) if filter_event_regex != "": cmd += " --filter_event_regex {}".format(filter_event_regex) + if timeout > 0: + cmd += " --timeout {}".format(timeout) return cmd + + +def unarchive_telemetry_certs(duthost): + # Move all files within old_certs directory to parent certs directory + path = "/etc/sonic/telemetry/" + archive_dir = path + "old_certs" + cmd = "ls {}".format(archive_dir) + filenames = duthost.shell(cmd)['stdout_lines'] + for filename in filenames: + cmd = "mv {}/{} {}".format(archive_dir, filename, path) + duthost.shell(cmd) + cmd = "rm -rf {}".format(archive_dir) + + +def archive_telemetry_certs(duthost): + # Move all files within certs directory to old_certs directory + path = "/etc/sonic/telemetry/" + archive_dir = path + "old_certs" + cmd = "mkdir -p {}".format(archive_dir) + duthost.shell(cmd) + cmd = "ls {}".format(path) + filenames = duthost.shell(cmd)['stdout_lines'] + for filename in filenames: + if filename.endswith(".cer") or filename.endswith(".key"): + cmd = "mv {} {}".format(path + filename, archive_dir) + duthost.shell(cmd) + + +def rotate_telemetry_certs(duthost, localhost): + path = "/etc/sonic/telemetry/" + # Create new certs to rotate + cmd = "openssl req \ + -x509 \ + -sha256 \ + -nodes \ + -newkey rsa:2048 \ + -keyout streamingtelemetryserver.key \ + -subj '/CN=ndastreamingservertest' \ + -out streamingtelemetryserver.cer" + localhost.shell(cmd) + cmd = "openssl req \ + -x509 \ + -sha256 \ + -nodes \ + -newkey rsa:2048 \ + -keyout dsmsroot.key \ + -subj '/CN=ndastreamingclienttest' \ + -out dsmsroot.cer" + localhost.shell(cmd) + + # Rotate certs + duthost.copy(src="streamingtelemetryserver.cer", dest=path) + duthost.copy(src="streamingtelemetryserver.key", dest=path) + duthost.copy(src="dsmsroot.cer", dest=path) + duthost.copy(src="dsmsroot.key", dest=path) diff --git a/tests/telemetry/test_events.py b/tests/telemetry/test_events.py index bf773228a5..7bb2a6fd17 100644 --- a/tests/telemetry/test_events.py +++ b/tests/telemetry/test_events.py @@ -3,11 +3,8 @@ import os import sys -from tests.common.utilities import InterruptableThread -from telemetry_utils import listen_for_events from telemetry_utils import skip_201911_and_older -from events.event_utils import create_ip_file -from events.event_utils import event_publish_tool, verify_received_output +from events.event_utils import event_publish_tool from events.event_utils import reset_event_counters, read_event_counters from events.event_utils import verify_counter_increase, restart_eventd @@ -37,7 +34,7 @@ def validate_yang(duthost, op_file="", yang_file=""): @pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) @pytest.mark.disable_loganalyzer -def test_events(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, setup_streaming_telemetry, gnxi_path, +def test_events(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, ptfadapter, setup_streaming_telemetry, gnxi_path, test_eventd_healthy): """ Run series of events inside duthost and validate that output is correct and conforms to YANG schema""" @@ -50,49 +47,14 @@ def test_events(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, setup_strea for file in os.listdir(EVENTS_TESTS_PATH): if file.endswith("_events.py") and not file.endswith("eventd_events.py"): module = __import__(file[:len(file)-3]) - module.test_event(duthost, gnxi_path, ptfhost, DATA_DIR, validate_yang) + try: + module.test_event(duthost, gnxi_path, ptfhost, ptfadapter, DATA_DIR, validate_yang) + except pytest.skip.Exception as e: + logger.info("Skipping test file: {} due to {}".format(file, e)) + continue logger.info("Completed test file: {}".format(os.path.join(EVENTS_TESTS_PATH, file))) -@pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) -@pytest.mark.disable_loganalyzer -def test_events_cache(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, setup_streaming_telemetry, gnxi_path): - """Create expected o/p file of events with N events. Call event-publisher tool to publish M events (M {}".format(ORIG_CFG_DB)) + data = json.loads(duthost.shell("cat {}".format(ORIG_CFG_DB), + verbose=False)['stdout']) + buffer_queues = list(data['BUFFER_QUEUE'].keys()) + iface_to_check = buffer_queues[0].split('|')[0] + iface_buffer_queues = [bq for bq in buffer_queues if any(val in iface_to_check for val in bq.split('|'))] + + # Add create_only_config_db_buffers entry to device metadata to enable + # counters optimization and get number of queue counters of Ethernet0 prior + # to removing buffer queues + data['DEVICE_METADATA']["localhost"]["create_only_config_db_buffers"] \ + = "true" + load_new_cfg(duthost, data) + pre_del_cnt = get_buffer_queues_cnt(ptfhost, gnxi_path, dut_ip, iface_to_check, env.gnmi_port) + + # Remove buffer queue and reload and get new number of queue counters + del data['BUFFER_QUEUE'][iface_buffer_queues[0]] + load_new_cfg(duthost, data) + post_del_cnt = get_buffer_queues_cnt(ptfhost, gnxi_path, dut_ip, iface_to_check, env.gnmi_port) + + pytest_assert(pre_del_cnt > post_del_cnt, + "Number of queue counters count differs from expected") + + +@pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) +def test_osbuild_version(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, + setup_streaming_telemetry, gnxi_path): """ Test osbuild/version query. """ duthost = duthosts[enum_rand_one_per_hwsku_hostname] @@ -119,7 +194,8 @@ def test_osbuild_version(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gn 0, "invalid build_version value at {0}".format(result)) -def test_sysuptime(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gnxi_path): +@pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) +def test_sysuptime(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gnxi_path, setup_streaming_telemetry): """ @summary: Run pyclient from ptfdocker and test the dataset 'system uptime' to check whether the value of 'system uptime' was float number and whether the value was @@ -168,7 +244,9 @@ def test_sysuptime(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gnxi_pat pytest.fail("The value of system uptime was not updated correctly.") -def test_virtualdb_table_streaming(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gnxi_path): +@pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) +def test_virtualdb_table_streaming(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gnxi_path, + setup_streaming_telemetry): """Run pyclient from ptfdocker to stream a virtual-db query multiple times. """ logger.info('start virtual db sample streaming testing') @@ -197,7 +275,9 @@ def invoke_py_cli_from_ptf(ptfhost, cmd, callback): callback(ret["stdout"]) -def test_on_change_updates(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gnxi_path): +@pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) +def test_on_change_updates(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gnxi_path, + setup_streaming_telemetry): logger.info("Testing on change update notifications") duthost = duthosts[enum_rand_one_per_hwsku_hostname] @@ -231,14 +311,15 @@ def callback(result): client_thread.join(60) # max timeout of 60s, expect update to come in <=30s +@pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) @pytest.mark.disable_loganalyzer -def test_mem_spike(duthosts, rand_one_dut_hostname, ptfhost, gnxi_path): +def test_mem_spike(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gnxi_path, setup_streaming_telemetry): """Test whether memory usage of telemetry container will exceed threshold if python gNMI client continuously creates channels with gNMI server. """ logger.info("Starting to test the memory spike issue of telemetry container") - duthost = duthosts[rand_one_dut_hostname] + duthost = duthosts[enum_rand_one_per_hwsku_hostname] env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) cmd = generate_client_cli(duthost=duthost, gnxi_path=gnxi_path, method=METHOD_SUBSCRIBE, diff --git a/tests/telemetry/test_telemetry_cert_rotation.py b/tests/telemetry/test_telemetry_cert_rotation.py new file mode 100644 index 0000000000..bab7adb3f1 --- /dev/null +++ b/tests/telemetry/test_telemetry_cert_rotation.py @@ -0,0 +1,159 @@ +import logging +import pytest + +from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import wait_until, wait_tcp_connection +from tests.common.helpers.gnmi_utils import GNMIEnvironment +from telemetry_utils import generate_client_cli +from telemetry_utils import archive_telemetry_certs, unarchive_telemetry_certs, rotate_telemetry_certs + +pytestmark = [ + pytest.mark.topology('any') +] + +logger = logging.getLogger(__name__) + +METHOD_GET = "get" +SUBMODE_POLL = 2 + +""" + +Testing cert rotation by telemetry + +1. Test that telemetry will stay up without certs +2. Test that when we serve one successful request, delete certs, second request will not work +3. Test that when we have no certs, first request will fail, rotate certs, second request will work +4. Test that when we have certs, request will succeed, rotate certs, second request will also succeed + +""" + + +@pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) +def test_telemetry_not_exit(duthosts, enum_rand_one_per_hwsku_hostname, setup_streaming_telemetry, localhost): + """ Test that telemetry server will not exit when certs are missing. We will shutdown telemetry, + remove certs and verify that telemetry is up and running. + """ + logger.info("Testing telemetry server will startup without certs") + + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) + + # Shutting down telemetry + duthost.service(name=env.gnmi_container, state="stopped") + + # Remove certs + archive_telemetry_certs(duthost) + + # Bring back telemetry + duthost.shell("systemctl reset-failed %s" % (env.gnmi_container), module_ignore_errors=True) + duthost.service(name=env.gnmi_container, state="restarted") + + # Wait until telemetry is active and running + pytest_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, env.gnmi_container), + "%s not started." % (env.gnmi_container)) + + # Restore certs + unarchive_telemetry_certs(duthost) + + # Wait for telemetry server to listen on port + dut_ip = duthost.mgmt_ip + wait_tcp_connection(localhost, dut_ip, env.gnmi_port, timeout_s=60) + + +@pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) +def test_telemetry_post_cert_del(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gnxi_path, localhost, + setup_streaming_telemetry): + """ Test that telemetry server with certificates will accept requests. + When certs are deleted, subsequent requests will not work. + """ + logger.info("Testing telemetry server post cert add") + + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) + + # Initial request should pass with certs + cmd = generate_client_cli(duthost=duthost, gnxi_path=gnxi_path, method=METHOD_GET, + target="OTHERS", xpath="proc/uptime") + ret = ptfhost.shell(cmd)['rc'] + assert ret == 0, "Telemetry server request should complete with certs" + + # Remove certs + archive_telemetry_certs(duthost) + + # Requests should fail without certs + cmd = generate_client_cli(duthost=duthost, gnxi_path=gnxi_path, method=METHOD_GET, + target="OTHERS", xpath="proc/uptime") + ret = ptfhost.shell(cmd, module_ignore_errors=True)['rc'] + assert ret != 0, "Telemetry server request should fail without certs" + + # Restore certs + unarchive_telemetry_certs(duthost) + + # Wait for telemetry server to listen on port + dut_ip = duthost.mgmt_ip + wait_tcp_connection(localhost, dut_ip, env.gnmi_port, timeout_s=60) + + +@pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) +def test_telemetry_post_cert_add(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gnxi_path, localhost, + setup_streaming_telemetry): + """ Test that telemetry server with no certificates will reject requests. + When certs are rotated, subsequent requests will work. + """ + logger.info("Testing telemetry server post cert add") + + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) + + # Remove certs + archive_telemetry_certs(duthost) + + # Initial request should fail without certs + cmd = generate_client_cli(duthost=duthost, gnxi_path=gnxi_path, method=METHOD_GET, + target="OTHERS", xpath="proc/uptime") + ret = ptfhost.shell(cmd, module_ignore_errors=True)['rc'] + assert ret != 0, "Telemetry server request should fail without certs" + + # Rotate certs + rotate_telemetry_certs(duthost, localhost) + + # Wait for telemetry server to listen on port + dut_ip = duthost.mgmt_ip + wait_tcp_connection(localhost, dut_ip, env.gnmi_port, timeout_s=60) + + # Requests should successfully complete with certs + cmd = generate_client_cli(duthost=duthost, gnxi_path=gnxi_path, method=METHOD_GET, + target="OTHERS", xpath="proc/uptime") + ret = ptfhost.shell(cmd)['rc'] + assert ret == 0, "Telemetry server request should complete with certs" + + +@pytest.mark.parametrize('setup_streaming_telemetry', [False], indirect=True) +def test_telemetry_cert_rotate(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost, gnxi_path, localhost, + setup_streaming_telemetry): + """ Test that telemetry server with certs will serve requests. + When certs are rotated, subsequent requests will work. + """ + logger.info("Testing telemetry server cert rotate") + + duthost = duthosts[enum_rand_one_per_hwsku_hostname] + env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE) + + # Initial request should complete with certs + cmd = generate_client_cli(duthost=duthost, gnxi_path=gnxi_path, method=METHOD_GET, + target="OTHERS", xpath="proc/uptime") + ret = ptfhost.shell(cmd)['rc'] + assert ret == 0, "Telemetry server request should fail without certs" + + # Rotate certs + rotate_telemetry_certs(duthost, localhost) + + # Wait for telemetry server to listen on port + dut_ip = duthost.mgmt_ip + wait_tcp_connection(localhost, dut_ip, env.gnmi_port, timeout_s=60) + + # Requests should successfully complete with certs + cmd = generate_client_cli(duthost=duthost, gnxi_path=gnxi_path, method=METHOD_GET, + target="OTHERS", xpath="proc/uptime") + ret = ptfhost.shell(cmd)['rc'] + assert ret == 0, "Telemetry server request should complete with certs" diff --git a/tests/templates/icmp_responder.conf.j2 b/tests/templates/icmp_responder.conf.j2 index dd8f653800..89881dd0ca 100644 --- a/tests/templates/icmp_responder.conf.j2 +++ b/tests/templates/icmp_responder.conf.j2 @@ -1,7 +1,6 @@ [program:icmp_responder] command=/root/env-python3/bin/python /opt/icmp_responder.py {{ icmp_responder_args }} process_name=icmp_responder -environment=PATH="/root/env-python3/bin" stdout_logfile=/tmp/icmp_responder.out.log stderr_logfile=/tmp/icmp_responder.err.log redirect_stderr=false diff --git a/tests/test_features.py b/tests/test_features.py index 8611e23566..de2458766c 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -1,25 +1,11 @@ # Helper Functions import pytest -from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import run_show_features pytestmark = [ pytest.mark.topology('any') ] -# Test Functions -def run_show_features(duthosts, enum_dut_hostname): - """Verify show features command output against CONFIG_DB - """ - duthost = duthosts[enum_dut_hostname] - features_dict, succeeded = duthost.get_feature_status() - pytest_assert(succeeded, "failed to obtain feature status") - for cmd_key, cmd_value in list(features_dict.items()): - redis_value = duthost.shell('/usr/bin/redis-cli -n 4 --raw hget "FEATURE|{}" "state"' - .format(cmd_key), module_ignore_errors=False)['stdout'] - pytest_assert(redis_value.lower() == cmd_value.lower(), - "'{}' is '{}' which does not match with config_db".format(cmd_key, cmd_value)) - - def test_show_features(duthosts, enum_dut_hostname): run_show_features(duthosts, enum_dut_hostname) diff --git a/tests/test_nbr_health.py b/tests/test_nbr_health.py index 20f03590d0..237720f6f9 100644 --- a/tests/test_nbr_health.py +++ b/tests/test_nbr_health.py @@ -11,7 +11,7 @@ pytestmark = [ pytest.mark.sanity_check(skip_sanity=True), pytest.mark.disable_loganalyzer, - pytest.mark.topology('util') # special marker + pytest.mark.topology('any') ] diff --git a/tests/test_pktgen.py b/tests/test_pktgen.py index b976d8be55..e42f1f970e 100644 --- a/tests/test_pktgen.py +++ b/tests/test_pktgen.py @@ -56,7 +56,7 @@ def clear_pktgen(duthosts, enum_dut_hostname): duthost.shell(cmd) -def test_pktgen(duthosts, enum_dut_hostname, enum_frontend_asic_index, tbinfo, loganalyzer, setup_thresholds): +def test_pktgen(duthosts, enum_dut_hostname, enum_frontend_asic_index, tbinfo, loganalyzer): ''' Testcase does the following steps: 1. Check max CPU utilized , number of core and dump files before starting the run diff --git a/tests/test_posttest.py b/tests/test_posttest.py index ac43eee643..d2c1d44c7b 100644 --- a/tests/test_posttest.py +++ b/tests/test_posttest.py @@ -1,3 +1,4 @@ +import os import pytest import logging import time @@ -88,3 +89,11 @@ def test_enable_startup_tsa_tsb_service(duthosts, localhost): else: logger.info("{} file does not exist in the specified path on dut {}". format(backup_tsa_tsb_file_path, duthost.hostname)) + + +def test_collect_ptf_logs(ptfhost): + log_files = ptfhost.shell('ls /tmp/*.log')['stdout'].split() + if not os.path.exists('logs/ptf'): + os.makedirs('logs/ptf') + for log_file in log_files: + ptfhost.fetch(src=log_file, dest='logs/ptf', fail_on_missing=False) diff --git a/tests/test_pretest.py b/tests/test_pretest.py index 06dd6ff7a7..061b3b7d95 100644 --- a/tests/test_pretest.py +++ b/tests/test_pretest.py @@ -317,7 +317,15 @@ def test_update_saithrift_ptf(request, ptfhost): pytest.skip("No URL specified for python saithrift package") pkg_name = py_saithrift_url.split("/")[-1] ptfhost.shell("rm -f {}".format(pkg_name)) - result = ptfhost.get_url(url=py_saithrift_url, dest="/root", module_ignore_errors=True, timeout=60) + # Retry download of saithrift library + retry_count = 5 + while retry_count > 0: + result = ptfhost.get_url(url=py_saithrift_url, dest="/root", module_ignore_errors=True, timeout=60) + if not result["failed"] or "OK" in result["msg"]: + break + time.sleep(60) + retry_count -= 1 + if result["failed"] or "OK" not in result["msg"]: pytest.skip("Download failed/error while installing python saithrift package") ptfhost.shell("dpkg -i {}".format(os.path.join("/root", pkg_name))) diff --git a/tests/testbed_setup/test_add_property_spytest_junit_xml.py b/tests/testbed_setup/test_add_property_spytest_junit_xml.py deleted file mode 100644 index 2e557e6978..0000000000 --- a/tests/testbed_setup/test_add_property_spytest_junit_xml.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Add test run and device properties to the Junit xml file generated by Spytest.""" -import datetime -import os.path -import pytest - -import xml.etree.ElementTree as ET - -pytestmark = [ - pytest.mark.topology('util') -] - - -def build_properties(properties_dict): - """Build global properties from passed key, value pairs.""" - properties = ET.Element("properties") - for name, value in list(properties_dict.items()): - properties.append(ET.Element("property", name=name, value=value)) - return properties - - -def test_add_property_spytest_junit_xml(duthost, request, tbinfo): - """Test to add device and testbed related properties to the junit xml file.""" - spytest_xmlfile = request.config.getoption("spytest_xmlpath") - spytest_updated_xmlfile = request.config.getoption("spytest_updated_xmlpath") - if spytest_xmlfile is None: - pytest.fail("Please specify the junit xml file generated by Spytest.") - if spytest_updated_xmlfile is None: - prefix, ext = os.path.splitext(spytest_xmlfile) - spytest_updated_xmlfile = prefix + "_update" + ext - - tree = ET.parse(spytest_xmlfile) - root = tree.getroot() - - properties = {} - properties["topology"] = tbinfo["topo"]["name"] - properties["testbed"] = tbinfo["conf-name"] - properties["timestamp"] = str(datetime.datetime.utcnow()) - properties["host"] = duthost.hostname - properties["asic"] = duthost.facts["asic_type"] - properties["platform"] = duthost.facts["platform"] - properties["hwsku"] = duthost.facts["hwsku"] - properties["os_version"] = duthost.os_version - - root.insert(0, build_properties(properties)) - - with open(spytest_updated_xmlfile, "w") as output: - output.write('') - output.write(ET.tostring(root)) diff --git a/tests/testbed_setup/test_gen_spy_testbed.py b/tests/testbed_setup/test_gen_spy_testbed.py deleted file mode 100644 index 7b41abc86f..0000000000 --- a/tests/testbed_setup/test_gen_spy_testbed.py +++ /dev/null @@ -1,103 +0,0 @@ -import logging -import json -import os -import pytest - -from collections import defaultdict -from jinja2 import Template -from tests.common.fixtures.conn_graph_facts import conn_graph_facts # noqa F401 - - -TESTBED_TEMPLATE = "templates/spytest_testbed.yaml.j2" -PTF_INTERFACE_TEMPLATE = "1/%d" - -pytestmark = [ - pytest.mark.topology("util"), - pytest.mark.sanity_check(skip_sanity=True) -] - - -@pytest.fixture(scope="function") -def hostvars(duthosts): - """Return host variables dicts for DUTs defined in testbed.""" - if not duthosts: - return {} - var_manager = duthosts[-1].host.options["variable_manager"] - hostvars_all = var_manager.get_vars()["hostvars"] - return {duthost.hostname: hostvars_all[duthost.hostname] - for duthost in duthosts} - - -def test_gen_spy_testbed(conn_graph_facts, hostvars, tbinfo, # noqa F811 - pytestconfig): - """Generate spytest testbed file.""" - - def _interface_key(interface): - """Get interface key to sort.""" - return list(map(int, interface.lstrip("Ethernet").split("/"))) - - hostnames = tbinfo["duts"] - connections = conn_graph_facts["device_conn"] - - # devices section - devices = [] - for hostname in hostnames: - hostvar = hostvars[hostname] - login_info = {} - login_info["login_access"] = \ - json.dumps(hostvar["login_access"]).replace('"', '') - login_info["login_credentials"] = \ - json.dumps(hostvar["login_credentials"]).replace('"', '') - devices.append((hostname, login_info)) - - # topology section - ptf_connections = [] - intf = 1 - for hostname in hostnames: - end_device = hostname - conns = connections[hostname] - end_ports = sorted( - (_ for _ in conns if conns[_]['peerdevice'] not in connections), - key=_interface_key) - for end_port in end_ports: - ptf_conn = { - "start_port": PTF_INTERFACE_TEMPLATE % intf, - "end_device": hostname, - "end_port": end_port - } - ptf_connections.append(ptf_conn) - conns.pop(end_port) - intf += 1 - - dev_connections = defaultdict(list) - for hostname in hostnames: - conns = connections[hostname] - for start_port in sorted(list(conns.keys()), key=_interface_key): - end_device = conns[start_port]["peerdevice"] - end_port = conns[start_port]["peerport"] - dev_connections[hostname].append( - { - "start_port": start_port, - "end_device": end_device, - "end_port": end_port - } - ) - connections[end_device].pop(end_port) - - # write to testbed dest file - with open(TESTBED_TEMPLATE) as tmpl_fd: - testbed_tmpl = Template( - tmpl_fd.read(), trim_blocks=True, lstrip_blocks=True) - testbed_file = os.path.join(str(pytestconfig.rootdir), - "../spytest/testbeds/spytest_testbed.yaml") - testbed_file = os.path.normpath(testbed_file) - logging.info("testbed save path: %s", testbed_file) - if os.path.exists(testbed_file): - logging.warn("testbed file(%s) exists, overwrite!", testbed_file) - testbed_stream = testbed_tmpl.stream( - devices=devices, - tbinfo=tbinfo, - ptf_connections=ptf_connections, - dev_connections=dev_connections - ) - testbed_stream.dump(testbed_file) diff --git a/tests/testsuites/platform_sanity b/tests/testsuites/platform_sanity new file mode 100644 index 0000000000..6be9148be7 --- /dev/null +++ b/tests/testsuites/platform_sanity @@ -0,0 +1,21 @@ +acl/test_acl.py::TestBasicAcl +decap/test_decap.py +fib/test_fib.py::test_basic_fib +pfcwd/test_pfc_config.py::TestPfcConfig +pfcwd/test_pfcwd_function.py::TestPfcwdFunc::test_pfcwd_actions +qos/test_qos_sai.py::TestQosSai::testQosSaiPfcXoffLimit +qos/test_qos_sai.py::TestQosSai::testQosSaiPfcXonLimit +qos/test_qos_sai.py::TestQosSai::testQosSaiLossyQueue +qos/test_qos_sai.py::TestQosSai::testQosSaiDscpQueueMapping +qos/test_qos_sai.py::TestQosSai::testQosSaiDot1pQueueMapping +qos/test_qos_sai.py::TestQosSai::testQosSaiDwrr +qos/test_qos_sai.py::TestQosSai::testQosSaiQSharedWatermark +voq/test_fabric_reach.py +voq/test_voq_fabric_status_all.py +voq/test_voq_init.py +voq/test_voq_ipfwd.py::TestTableValidation +voq/test_voq_ipfwd.py::TestVoqIPFwd +voq/test_voq_ipfwd.py::TestFPLinkFlap::test_front_panel_linkflap_port +voq/test_voq_nbr.py::test_neighbor_clear_one +voq/test_voq_nbr.py::test_neighbor_hw_mac_change +voq/test_voq_nbr.py::TestNeighborLinkFlap diff --git a/tests/upgrade_path/test_upgrade_path.py b/tests/upgrade_path/test_upgrade_path.py index 14f0ed1226..c0a41ad37c 100644 --- a/tests/upgrade_path/test_upgrade_path.py +++ b/tests/upgrade_path/test_upgrade_path.py @@ -2,21 +2,20 @@ import logging import re from tests.common import reboot -from tests.upgrade_path.upgrade_helpers import install_sonic, check_sonic_version,\ - upgrade_test_helper -from tests.upgrade_path.upgrade_helpers import restore_image # noqa F401 +from tests.common.helpers.upgrade_helpers import install_sonic, check_sonic_version,\ + upgrade_test_helper, check_asic_and_db_consistency +from tests.common.helpers.upgrade_helpers import restore_image # noqa F401 from tests.common.fixtures.advanced_reboot import get_advanced_reboot # noqa F401 -from tests.platform_tests.verify_dut_health import verify_dut_health # noqa F401 +from tests.common.fixtures.consistency_checker.consistency_checker import consistency_checker_provider # noqa F401 +from tests.common.platform.device_utils import verify_dut_health # noqa F401 from tests.common.fixtures.duthost_utils import backup_and_restore_config_db # noqa F401 - -from tests.platform_tests.conftest import advanceboot_loganalyzer, advanceboot_neighbor_restore # noqa F401 +from tests.common.platform.device_utils import advanceboot_loganalyzer, advanceboot_neighbor_restore # noqa F401 from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 from tests.common.fixtures.ptfhost_utils import remove_ip_addresses # noqa F401 from tests.common.fixtures.ptfhost_utils import copy_arp_responder_py # noqa F401 from tests.common.errors import RunAnsibleModuleFail - -from tests.platform_tests.warmboot_sad_cases import get_sad_case_list, SAD_CASE_LIST +from tests.common.platform.warmboot_sad_cases import get_sad_case_list, SAD_CASE_LIST pytestmark = [ @@ -94,9 +93,9 @@ def setup_upgrade_test(duthost, localhost, from_image, to_image, tbinfo, @pytest.mark.device_type('vs') def test_double_upgrade_path(localhost, duthosts, ptfhost, rand_one_dut_hostname, - nbrhosts, fanouthosts, tbinfo, restore_image, # noqa F811 + nbrhosts, fanouthosts, tbinfo, request, restore_image, # noqa F811 get_advanced_reboot, verify_dut_health, advanceboot_loganalyzer, # noqa F811 - upgrade_path_lists): # noqa F811 + consistency_checker_provider, upgrade_path_lists): # noqa F811 duthost = duthosts[rand_one_dut_hostname] upgrade_type, from_image, to_image, _, enable_cpa = upgrade_path_lists logger.info("Test upgrade path from {} to {}".format(from_image, to_image)) @@ -105,18 +104,23 @@ def upgrade_path_preboot_setup(): setup_upgrade_test(duthost, localhost, from_image, to_image, tbinfo, upgrade_type) + def upgrade_path_postboot_setup(): + check_asic_and_db_consistency(request.config, duthost, consistency_checker_provider) + upgrade_test_helper(duthost, localhost, ptfhost, from_image, to_image, tbinfo, upgrade_type, get_advanced_reboot, advanceboot_loganalyzer=advanceboot_loganalyzer, - preboot_setup=upgrade_path_preboot_setup, enable_cpa=enable_cpa, + preboot_setup=upgrade_path_preboot_setup, + postboot_setup=upgrade_path_postboot_setup, + enable_cpa=enable_cpa, reboot_count=2) @pytest.mark.device_type('vs') def test_upgrade_path(localhost, duthosts, ptfhost, rand_one_dut_hostname, - nbrhosts, fanouthosts, tbinfo, restore_image, # noqa F811 + nbrhosts, fanouthosts, tbinfo, request, restore_image, # noqa F811 get_advanced_reboot, verify_dut_health, advanceboot_loganalyzer, # noqa F811 - upgrade_path_lists): + consistency_checker_provider, upgrade_path_lists): # noqa F811 duthost = duthosts[rand_one_dut_hostname] upgrade_type, from_image, to_image, _, enable_cpa = upgrade_path_lists logger.info("Test upgrade path from {} to {}".format(from_image, to_image)) @@ -125,18 +129,24 @@ def upgrade_path_preboot_setup(): setup_upgrade_test(duthost, localhost, from_image, to_image, tbinfo, upgrade_type) + def upgrade_path_postboot_setup(): + check_asic_and_db_consistency(request.config, duthost, consistency_checker_provider) + upgrade_test_helper(duthost, localhost, ptfhost, from_image, to_image, tbinfo, upgrade_type, get_advanced_reboot, advanceboot_loganalyzer=advanceboot_loganalyzer, - preboot_setup=upgrade_path_preboot_setup, enable_cpa=enable_cpa) + preboot_setup=upgrade_path_preboot_setup, + postboot_setup=upgrade_path_postboot_setup, + enable_cpa=enable_cpa) @pytest.mark.device_type('vs') def test_warm_upgrade_sad_path(localhost, duthosts, ptfhost, rand_one_dut_hostname, - nbrhosts, fanouthosts, vmhost, tbinfo, restore_image, # noqa F811 + nbrhosts, fanouthosts, vmhost, tbinfo, request, restore_image, # noqa F811 get_advanced_reboot, verify_dut_health, advanceboot_loganalyzer, # noqa F811 upgrade_path_lists, backup_and_restore_config_db, # noqa F811 - advanceboot_neighbor_restore, sad_case_type): # noqa F811 + advanceboot_neighbor_restore, consistency_checker_provider, # noqa F811 + sad_case_type): # noqa F811 duthost = duthosts[rand_one_dut_hostname] upgrade_type, from_image, to_image, _, enable_cpa = upgrade_path_lists logger.info("Test upgrade path from {} to {}".format(from_image, to_image)) @@ -145,11 +155,15 @@ def upgrade_path_preboot_setup(): setup_upgrade_test(duthost, localhost, from_image, to_image, tbinfo, upgrade_type) + def upgrade_path_postboot_setup(): + check_asic_and_db_consistency(request.config, duthost, consistency_checker_provider) + sad_preboot_list, sad_inboot_list = get_sad_case_list( duthost, nbrhosts, fanouthosts, vmhost, tbinfo, sad_case_type) upgrade_test_helper(duthost, localhost, ptfhost, from_image, to_image, tbinfo, "warm", get_advanced_reboot, advanceboot_loganalyzer=advanceboot_loganalyzer, preboot_setup=upgrade_path_preboot_setup, + postboot_setup=upgrade_path_postboot_setup, sad_preboot_list=sad_preboot_list, sad_inboot_list=sad_inboot_list, enable_cpa=enable_cpa) diff --git a/tests/vlan/test_vlan.py b/tests/vlan/test_vlan.py index 1803b6e22a..0b0af119dd 100644 --- a/tests/vlan/test_vlan.py +++ b/tests/vlan/test_vlan.py @@ -14,6 +14,7 @@ from tests.common.helpers.portchannel_to_vlan import vlan_intfs_dict # noqa F401 from tests.common.helpers.portchannel_to_vlan import setup_po2vlan # noqa F401 from tests.common.helpers.portchannel_to_vlan import running_vlan_ports_list +from tests.common.helpers.portchannel_to_vlan import has_portchannels logger = logging.getLogger(__name__) @@ -112,8 +113,6 @@ def verify_icmp_packets(ptfadapter, send_pkt, vlan_ports_list, vlan_port, vlan_i masked_tagged_pkt = Mask(tagged_pkt) masked_tagged_pkt.set_do_not_care_scapy(scapy.Dot1Q, "prio") - logger.info("Verify untagged packets from ports " + - str(vlan_port["port_index"][0])) for port in vlan_ports_list: if vlan_port["port_index"] == port["port_index"]: # Skip src port @@ -132,10 +131,12 @@ def verify_icmp_packets(ptfadapter, send_pkt, vlan_ports_list, vlan_port, vlan_i ptfadapter.dataplane.flush() for src_port in vlan_port["port_index"]: testutils.send(ptfadapter, src_port, send_pkt) + logger.info("Verify untagged packets from ports " + str(vlan_port["port_index"][0])) verify_packets_with_portchannel(test=ptfadapter, pkt=untagged_pkt, ports=untagged_dst_ports, portchannel_ports=untagged_dst_pc_ports) + logger.info("Verify tagged packets from ports " + str(vlan_port["port_index"][0])) verify_packets_with_portchannel(test=ptfadapter, pkt=masked_tagged_pkt, ports=tagged_dst_ports, @@ -168,6 +169,12 @@ def test_vlan_tc1_send_untagged(ptfadapter, duthosts, rand_one_dut_hostname, ran if "dualtor" in tbinfo["topo"]["name"]: pytest.skip("Dual TOR device does not support broadcast packet") + # Skip the test if no portchannel interfaces are detected + # e.g., when sending packets to an egress port with PVID 0 on a portchannel interface + # the absence of portchannel interfaces means the expected destination doesn't exist + if not has_portchannels(duthosts, rand_one_dut_hostname): + pytest.skip("Test skipped: No portchannels detected when sending untagged packets") + untagged_pkt = build_icmp_packet(0) # Need a tagged packet for set_do_not_care_scapy tagged_pkt = build_icmp_packet(4095) @@ -175,7 +182,7 @@ def test_vlan_tc1_send_untagged(ptfadapter, duthosts, rand_one_dut_hostname, ran exp_pkt.set_do_not_care_scapy(scapy.Dot1Q, "vlan") vlan_ports_list = running_vlan_ports_list(duthosts, rand_one_dut_hostname, rand_selected_dut, tbinfo, ports_list) for vlan_port in vlan_ports_list: - logger.info("Send untagged packet from {} ...".format( + logger.info("Send untagged packet from the port {} ...".format( vlan_port["port_index"][0])) logger.info(untagged_pkt.sprintf( "%Ether.src% %IP.src% -> %Ether.dst% %IP.dst%")) @@ -207,11 +214,17 @@ def test_vlan_tc2_send_tagged(ptfadapter, duthosts, rand_one_dut_hostname, rand_ if "dualtor" in tbinfo["topo"]["name"]: pytest.skip("Dual TOR device does not support broadcast packet") + # Skip the test if no portchannel interfaces are detected + # e.g., when sending packets to an egress port with PVID 0 on a portchannel interface + # the absence of portchannel interfaces means the expected destination doesn't exist + if not has_portchannels(duthosts, rand_one_dut_hostname): + pytest.skip("Test skipped: No portchannels detected when sending tagged packets") + vlan_ports_list = running_vlan_ports_list(duthosts, rand_one_dut_hostname, rand_selected_dut, tbinfo, ports_list) for vlan_port in vlan_ports_list: for permit_vlanid in map(int, vlan_port["permit_vlanid"]): pkt = build_icmp_packet(permit_vlanid) - logger.info("Send tagged({}) packet from {} ...".format( + logger.info("Send tagged({}) packet from the port {} ...".format( permit_vlanid, vlan_port["port_index"][0])) logger.info(pkt.sprintf( "%Ether.src% %IP.src% -> %Ether.dst% %IP.dst%")) @@ -374,6 +387,12 @@ def test_vlan_tc6_tagged_untagged_unicast(ptfadapter, duthosts, rand_one_dut_hos Send packets w/ src and dst specified over tagged port and untagged port in vlan Verify that bidirectional communication between tagged port and untagged port work """ + # Skip the test if no portchannel interfaces are detected + # e.g., when sending packets to an egress port with PVID 0 on a portchannel interface + # the absence of portchannel interfaces means the expected destination doesn't exist + if not has_portchannels(duthosts, rand_one_dut_hostname): + pytest.skip("Test skipped: No portchannels detected when sending untagged packets") + vlan_ports_list = running_vlan_ports_list(duthosts, rand_one_dut_hostname, rand_selected_dut, tbinfo, ports_list) for test_vlan in vlan_intfs_dict: untagged_ports_for_test = [] diff --git a/tests/vlan/test_vlan_ping.py b/tests/vlan/test_vlan_ping.py index f2f5c5a93c..9fc4cc2e67 100644 --- a/tests/vlan/test_vlan_ping.py +++ b/tests/vlan/test_vlan_ping.py @@ -3,8 +3,14 @@ import ipaddress import logging import ptf.testutils as testutils +import ptf.packet as scapy +from ptf.mask import Mask import six +from ipaddress import ip_address, IPv4Address from tests.common.helpers.assertions import pytest_assert as py_assert +from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F401 +from tests.common.dualtor.dual_tor_utils import lower_tor_host # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 logger = logging.getLogger(__name__) @@ -47,7 +53,7 @@ def static_neighbor_entry(duthost, dic, oper, ip_version="both"): @pytest.fixture(scope='module') -def vlan_ping_setup(duthosts, rand_one_dut_hostname, ptfhost, nbrhosts, tbinfo): +def vlan_ping_setup(duthosts, rand_one_dut_hostname, ptfhost, nbrhosts, tbinfo, lower_tor_host): # noqa F811 """ Setup: Collecting vm_host_info, ptfhost_info Teardown: Removing all added ipv4 and ipv6 neighbors @@ -69,11 +75,28 @@ def vlan_ping_setup(duthosts, rand_one_dut_hostname, ptfhost, nbrhosts, tbinfo): else: vm_ip_with_prefix = six.ensure_text(vm_info['conf']['interfaces']['Port-Channel1']['ipv4']) output = vm_info['host'].command("ip addr show dev po1") + # in case of lower tor host we need to use the next portchannel + if "dualtor-aa" in tbinfo["topo"]["name"] and rand_one_dut_hostname == lower_tor_host.hostname: + vm_ip_with_prefix = six.ensure_text(vm_info['conf']['interfaces']['Port-Channel2']['ipv4']) + output = vm_info['host'].command("ip addr show dev po2") vm_host_info["mac"] = output['stdout_lines'][1].split()[1] vm_ip_intf = ipaddress.IPv4Interface(vm_ip_with_prefix).ip vm_host_info["ipv4"] = vm_ip_intf duthost = duthosts[rand_one_dut_hostname] mg_facts = duthost.get_extended_minigraph_facts(tbinfo) + if "dualtor-aa" in tbinfo["topo"]["name"]: + idx = duthosts.index(duthost) + unselected_duthost = duthosts[1 - idx] + unslctd_mg_facts = unselected_duthost.minigraph_facts(host=unselected_duthost.hostname)['ansible_facts'] + unslctd_mg_facts['mg_ptf_idx'] = unslctd_mg_facts['minigraph_port_indices'].copy() + try: + map = tbinfo['topo']['ptf_map'][str(1 - idx)] + if map: + for port, index in list(unslctd_mg_facts['minigraph_port_indices'].items()): + if str(index) in map: + unslctd_mg_facts['mg_ptf_idx'][port] = map[str(index)] + except (ValueError, KeyError): + pass my_cfg_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] ptfhost_info = {} ip4 = None @@ -88,14 +111,24 @@ def vlan_ping_setup(duthosts, rand_one_dut_hostname, ptfhost, nbrhosts, tbinfo): vm_host_info['port_index_list'] = [mg_facts['minigraph_ptf_indices'][intf['attachto']]] break else: - for intf in mg_facts['minigraph_portchannel_interfaces']: - if intf['peer_addr'] == str(vm_host_info['ipv4']): - portchannel = intf['attachto'] - ifaces_list = [] - for iface in mg_facts['minigraph_portchannels'][portchannel]['members']: - ifaces_list.append(mg_facts['minigraph_ptf_indices'][iface]) - vm_host_info['port_index_list'] = ifaces_list + ifaces_list = [] + # UL pkt may take any of the tor in case of dualtor-aa + if "dualtor-aa" in tbinfo["topo"]["name"]: + for intf in mg_facts['minigraph_portchannel_interfaces']: + if type(ip_address(intf['peer_addr'])) is IPv4Address: + portchannel = intf['attachto'] + for iface in mg_facts['minigraph_portchannels'][portchannel]['members']: + ifaces_list.append(mg_facts['minigraph_ptf_indices'][iface]) + ifaces_list.append(unslctd_mg_facts['mg_ptf_idx'][iface]) + ifaces_list = list(dict.fromkeys(ifaces_list)) + else: + for intf in mg_facts['minigraph_portchannel_interfaces']: + if intf['peer_addr'] == str(vm_host_info['ipv4']): + portchannel = intf['attachto'] + for iface in mg_facts['minigraph_portchannels'][portchannel]['members']: + ifaces_list.append(mg_facts['minigraph_ptf_indices'][iface]) break + vm_host_info['port_index_list'] = ifaces_list break # getting the ipv4, ipv6 and vlan id of a vlan in DUT with 2 or more vlan members @@ -143,21 +176,42 @@ def vlan_ping_setup(duthosts, rand_one_dut_hostname, ptfhost, nbrhosts, tbinfo): yield vm_host_info, ptfhost_info logger.info("Removing all added ipv4 and ipv6 neighbors") - neigh_list = duthost.shell("sudo ip neigh | grep PERMANENT")["stdout_lines"] - for neigh in neigh_list: - cmd = neigh.split(" PERMANENT")[0] - duthost.shell("sudo ip neigh del {}".format(cmd)) - - -def verify_icmp_packet(dut_mac, src_port, dst_port, ptfadapter): - pkt = testutils.simple_icmp_packet(eth_src=str(src_port['mac']), - eth_dst=str(dut_mac), - ip_src=str(src_port['ipv4']), - ip_dst=str(dst_port['ipv4']), ip_ttl=64) - exptd_pkt = testutils.simple_icmp_packet(eth_src=str(dut_mac), - eth_dst=str(dst_port['mac']), - ip_src=str(src_port['ipv4']), - ip_dst=str(dst_port['ipv4']), ip_ttl=63) + duthost.shell("sudo ip neigh flush nud permanent") + + +def verify_icmp_packet(dut_mac, src_port, dst_port, ptfadapter, tbinfo, + vlan_mac=None, dtor_ul=False, dtor_dl=False, skip_traffic_test=False): # noqa F811 + if skip_traffic_test is True: + logger.info("Skipping traffic test") + return + if dtor_ul is True: + # use vlan int mac in case of dualtor UL test pkt + pkt = testutils.simple_icmp_packet(eth_src=str(src_port['mac']), + eth_dst=str(vlan_mac), + ip_src=str(src_port['ipv4']), + ip_dst=str(dst_port['ipv4']), ip_ttl=64) + else: + # use dut mac addr for all other test pkts + pkt = testutils.simple_icmp_packet(eth_src=str(src_port['mac']), + eth_dst=str(dut_mac), + ip_src=str(src_port['ipv4']), + ip_dst=str(dst_port['ipv4']), ip_ttl=64) + if dtor_dl is True: + # expect vlan int mac as src mac in dualtor DL test pkt + exptd_pkt = testutils.simple_icmp_packet(eth_src=str(vlan_mac), + eth_dst=str(dst_port['mac']), + ip_src=str(src_port['ipv4']), + ip_dst=str(dst_port['ipv4']), ip_ttl=63) + else: + # expect dut mac as src mac for non dualtor DL test pkt + exptd_pkt = testutils.simple_icmp_packet(eth_src=str(dut_mac), + eth_dst=str(dst_port['mac']), + ip_src=str(src_port['ipv4']), + ip_dst=str(dst_port['ipv4']), ip_ttl=63) + # skip smac check for dualtor-aa UL test pkt + if "dualtor-aa" in tbinfo["topo"]["name"] and dtor_ul is True: + exptd_pkt = Mask(exptd_pkt) + exptd_pkt.set_do_not_care_scapy(scapy.Ether, "src") for i in range(5): testutils.send_packet(ptfadapter, src_port['port_index_list'][0], pkt) try: @@ -169,7 +223,8 @@ def verify_icmp_packet(dut_mac, src_port, dst_port, ptfadapter): raise e # If it fails on the last attempt, raise the exception -def test_vlan_ping(vlan_ping_setup, duthosts, rand_one_dut_hostname, ptfadapter): +def test_vlan_ping(vlan_ping_setup, duthosts, rand_one_dut_hostname, ptfadapter, tbinfo, + toggle_all_simulator_ports_to_rand_selected_tor_m, skip_traffic_test): # noqa F811 """ test for checking connectivity of statically added ipv4 and ipv6 arp entries """ @@ -177,14 +232,36 @@ def test_vlan_ping(vlan_ping_setup, duthosts, rand_one_dut_hostname, ptfadapter) vmhost_info, ptfhost_info = vlan_ping_setup device2 = dict(list(ptfhost_info.items())[1:]) device1 = dict(list(ptfhost_info.items())[:1]) + # use mac addr of vlan interface in case of dualtor + if 'dualtor' in tbinfo["topo"]["name"]: + vlan_table = duthost.get_running_config_facts()['VLAN'] + vlan_name = list(vlan_table.keys())[0] + vlan_mac = duthost.get_dut_iface_mac(vlan_name) + # dump neigh entries + logger.info("Dumping all ipv4 and ipv6 neighbors") + duthost.shell("sudo ip neigh show") + # flush entries of vlan interface in case of dualtor to avoid issue#12302 + logger.info("Flushing all ipv4 and ipv6 neighbors on {}".format(vlan_name)) + duthost.shell("sudo ip neigh flush dev {} all".format(vlan_name)) # initial setup and checking connectivity, try to break in more chunks logger.info("initializing setup for ipv4 and ipv6") static_neighbor_entry(duthost, ptfhost_info, "add") logger.info("Checking connectivity to ptf ports") + for member in ptfhost_info: - verify_icmp_packet(duthost.facts['router_mac'], ptfhost_info[member], vmhost_info, ptfadapter) - verify_icmp_packet(duthost.facts['router_mac'], vmhost_info, ptfhost_info[member], ptfadapter) + if 'dualtor' in tbinfo["topo"]["name"]: + verify_icmp_packet(duthost.facts['router_mac'], ptfhost_info[member], + vmhost_info, ptfadapter, tbinfo, vlan_mac, dtor_ul=True, + skip_traffic_test=skip_traffic_test) + verify_icmp_packet(duthost.facts['router_mac'], vmhost_info, ptfhost_info[member], + ptfadapter, tbinfo, vlan_mac, dtor_dl=True, + skip_traffic_test=skip_traffic_test) + else: + verify_icmp_packet(duthost.facts['router_mac'], ptfhost_info[member], vmhost_info, ptfadapter, tbinfo, + skip_traffic_test=skip_traffic_test) + verify_icmp_packet(duthost.facts['router_mac'], vmhost_info, ptfhost_info[member], ptfadapter, tbinfo, + skip_traffic_test=skip_traffic_test) # flushing and re-adding ipv6 static arp entry static_neighbor_entry(duthost, ptfhost_info, "del", "6") @@ -201,5 +278,15 @@ def test_vlan_ping(vlan_ping_setup, duthosts, rand_one_dut_hostname, ptfadapter) # Checking for connectivity logger.info("Check connectivity to both ptfhost") for member in ptfhost_info: - verify_icmp_packet(duthost.facts['router_mac'], ptfhost_info[member], vmhost_info, ptfadapter) - verify_icmp_packet(duthost.facts['router_mac'], vmhost_info, ptfhost_info[member], ptfadapter) + if 'dualtor' in tbinfo["topo"]["name"]: + verify_icmp_packet(duthost.facts['router_mac'], ptfhost_info[member], + vmhost_info, ptfadapter, tbinfo, vlan_mac, dtor_ul=True, + skip_traffic_test=skip_traffic_test) + verify_icmp_packet(duthost.facts['router_mac'], vmhost_info, ptfhost_info[member], + ptfadapter, tbinfo, vlan_mac, dtor_dl=True, + skip_traffic_test=skip_traffic_test) + else: + verify_icmp_packet(duthost.facts['router_mac'], ptfhost_info[member], vmhost_info, ptfadapter, tbinfo, + skip_traffic_test=skip_traffic_test) + verify_icmp_packet(duthost.facts['router_mac'], vmhost_info, ptfhost_info[member], ptfadapter, tbinfo, + skip_traffic_test=skip_traffic_test) diff --git a/tests/voq/conftest.py b/tests/voq/conftest.py index ed37dc2fe2..e4b8c314f1 100644 --- a/tests/voq/conftest.py +++ b/tests/voq/conftest.py @@ -1,7 +1,7 @@ import pytest import logging -from .voq_helpers import get_eos_mac +from tests.common.helpers.voq_helpers import get_eos_mac from tests.common.helpers.parallel import parallel_run, reset_ansible_local_tmp from tests.common.helpers.dut_utils import get_host_visible_vars diff --git a/tests/voq/test_fabric_reach.py b/tests/voq/test_fabric_reach.py index 6babf58445..215a91780d 100644 --- a/tests/voq/test_fabric_reach.py +++ b/tests/voq/test_fabric_reach.py @@ -37,7 +37,7 @@ def refData(duthosts): fileName = lc_sku + "_" + fabric_sku + "_" + "LC" + str(slot) + ".yaml" f = open("voq/fabric_data/{}".format(fileName)) pytest_assert(f, "Need to update expected data for {}".format(fileName)) - referenceData[slot] = yaml.load(f) + referenceData[slot] = yaml.safe_load(f) return referenceData @@ -54,7 +54,7 @@ def supData(duthosts): fileName = fabric_sku + ".yaml" f = open("voq/fabric_data/{}".format(fileName)) pytest_assert(f, "Need to update expected data for {}".format(fileName)) - supData = yaml.load(f) + supData = yaml.safe_load(f) f.close() return supData diff --git a/tests/voq/test_voq_disrupts.py b/tests/voq/test_voq_disrupts.py index 2a0c782d74..f4269e343d 100644 --- a/tests/voq/test_voq_disrupts.py +++ b/tests/voq/test_voq_disrupts.py @@ -2,14 +2,14 @@ import logging import time -from .voq_helpers import sonic_ping -from .voq_helpers import eos_ping +from tests.common.helpers.voq_helpers import sonic_ping +from tests.common.helpers.voq_helpers import eos_ping from .test_voq_ipfwd import pick_ports from .test_voq_ipfwd import check_packet from .test_voq_init import check_voq_interfaces -from .voq_helpers import dump_and_verify_neighbors_on_asic +from tests.common.helpers.voq_helpers import dump_and_verify_neighbors_on_asic from tests.common import reboot from tests.common import config_reload diff --git a/tests/voq/test_voq_fabric_status_all.py b/tests/voq/test_voq_fabric_status_all.py index 0787845d3a..7e65d742fb 100644 --- a/tests/voq/test_voq_fabric_status_all.py +++ b/tests/voq/test_voq_fabric_status_all.py @@ -38,7 +38,7 @@ def refData(duthosts): fileName = lc_sku + "_" + fabric_sku + "_" + "LC" + str(slot) + ".yaml" f = open("voq/fabric_data/{}".format(fileName)) pytest_assert(f, "Need to update expected data for {}".format(fileName)) - referenceData[slot] = yaml.load(f) + referenceData[slot] = yaml.safe_load(f) return referenceData diff --git a/tests/voq/test_voq_init.py b/tests/voq/test_voq_init.py index 772fc5df68..8ec943ca07 100644 --- a/tests/voq/test_voq_init.py +++ b/tests/voq/test_voq_init.py @@ -5,10 +5,10 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.helpers.sonic_db import AsicDbCli, VoqDbCli -from .voq_helpers import check_voq_remote_neighbor, get_sonic_mac -from .voq_helpers import check_local_neighbor_asicdb, get_device_system_ports, get_inband_info -from .voq_helpers import check_rif_on_sup, check_voq_neighbor_on_sup -from .voq_helpers import dump_and_verify_neighbors_on_asic +from tests.common.helpers.voq_helpers import check_voq_remote_neighbor, get_sonic_mac +from tests.common.helpers.voq_helpers import check_local_neighbor_asicdb, get_device_system_ports, get_inband_info +from tests.common.helpers.voq_helpers import check_rif_on_sup, check_voq_neighbor_on_sup +from tests.common.helpers.voq_helpers import dump_and_verify_neighbors_on_asic logger = logging.getLogger(__name__) @@ -124,8 +124,10 @@ def test_voq_local_port_create(duthosts, enum_frontend_dut_hostname, enum_asic_i show_intf = asic.show_interface(command="status", include_internal_intfs=True)['ansible_facts'] for portkey in keylist: - portkey = portkey.decode('unicode-escape') # need to handle the hyphen in the inband port name - port_name = hostif_table[portkey]['value']["SAI_HOSTIF_ATTR_NAME"].decode('unicode-escape') + # need to handle the hyphen in the inband port name + portkey = bytes(portkey, encoding='ascii').decode('unicode-escape') + port_name = bytes(hostif_table[portkey]['value']["SAI_HOSTIF_ATTR_NAME"], + encoding='ascii').decode('unicode-escape') port_state = hostif_table[portkey]['value']["SAI_HOSTIF_ATTR_OPER_STATUS"] port_type = hostif_table[portkey]['value']["SAI_HOSTIF_ATTR_TYPE"] @@ -206,8 +208,8 @@ def check_voq_interfaces(duthosts, per_host, asic, cfg_facts): if porttype == 'hostif': # find the hostif entry to get the physical port the router interface is on. hostifkey = asicdb.find_hostif_by_portid(portid) - hostif = asicdb.get_hostif_table(refresh=False)[hostifkey][ - 'value']['SAI_HOSTIF_ATTR_NAME'].decode('unicode-escape') + hostif = bytes(asicdb.get_hostif_table(refresh=False)[hostifkey][ + 'value']['SAI_HOSTIF_ATTR_NAME'], encoding='ascii').decode('unicode-escape') logger.info("RIF: %s is on local port: %s", rif, hostif) rif_ports_in_asicdb.append(hostif) if hostif not in dev_intfs and hostif not in voq_intfs: diff --git a/tests/voq/test_voq_ipfwd.py b/tests/voq/test_voq_ipfwd.py index 03a55e7313..dee41e3fd5 100644 --- a/tests/voq/test_voq_ipfwd.py +++ b/tests/voq/test_voq_ipfwd.py @@ -19,14 +19,14 @@ from .test_voq_nbr import LinkFlap -from .voq_helpers import sonic_ping -from .voq_helpers import eos_ping -from .voq_helpers import get_inband_info -from .voq_helpers import get_vm_with_ip -from .voq_helpers import asic_cmd -from .voq_helpers import get_port_by_ip -from .voq_helpers import get_sonic_mac -from .voq_helpers import get_ptf_port +from tests.common.helpers.voq_helpers import sonic_ping +from tests.common.helpers.voq_helpers import eos_ping +from tests.common.helpers.voq_helpers import get_inband_info +from tests.common.helpers.voq_helpers import get_vm_with_ip +from tests.common.helpers.voq_helpers import asic_cmd +from tests.common.helpers.voq_helpers import get_port_by_ip +from tests.common.helpers.voq_helpers import get_sonic_mac +from tests.common.helpers.voq_helpers import get_ptf_port import re logger = logging.getLogger(__name__) diff --git a/tests/voq/test_voq_nbr.py b/tests/voq/test_voq_nbr.py index 6475248502..595371f7ba 100644 --- a/tests/voq/test_voq_nbr.py +++ b/tests/voq/test_voq_nbr.py @@ -15,16 +15,16 @@ from tests.common.helpers.parallel import parallel_run from tests.common.helpers.parallel import reset_ansible_local_tmp -from .voq_helpers import get_neighbor_info -from .voq_helpers import get_port_by_ip -from .voq_helpers import check_all_neighbors_present, check_one_neighbor_present -from .voq_helpers import asic_cmd, sonic_ping -from .voq_helpers import check_neighbors_are_gone -from .voq_helpers import dump_and_verify_neighbors_on_asic -from .voq_helpers import poll_neighbor_table_delete -from .voq_helpers import get_inband_info -from .voq_helpers import get_ptf_port -from .voq_helpers import get_vm_with_ip +from tests.common.helpers.voq_helpers import get_neighbor_info +from tests.common.helpers.voq_helpers import get_port_by_ip +from tests.common.helpers.voq_helpers import check_all_neighbors_present, check_one_neighbor_present +from tests.common.helpers.voq_helpers import asic_cmd, sonic_ping +from tests.common.helpers.voq_helpers import check_neighbors_are_gone +from tests.common.helpers.voq_helpers import dump_and_verify_neighbors_on_asic +from tests.common.helpers.voq_helpers import poll_neighbor_table_delete +from tests.common.helpers.voq_helpers import get_inband_info +from tests.common.helpers.voq_helpers import get_ptf_port +from tests.common.helpers.voq_helpers import get_vm_with_ip from tests.common.devices.eos import EosHost from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 diff --git a/tests/vxlan/test_vnet_vxlan.py b/tests/vxlan/test_vnet_vxlan.py index 643f6797fd..41413003f9 100644 --- a/tests/vxlan/test_vnet_vxlan.py +++ b/tests/vxlan/test_vnet_vxlan.py @@ -13,11 +13,9 @@ from .vnet_utils import generate_dut_config_files, safe_open_template, \ apply_dut_config_files, cleanup_dut_vnets, cleanup_vxlan_tunnels, cleanup_vnet_routes -from tests.common.fixtures.ptfhost_utils import remove_ip_addresses, change_mac_addresses, \ - copy_arp_responder_py, copy_ptftests_directory # noqa F401 -from tests.flow_counter.flow_counter_utils import RouteFlowCounterTestContext,\ - is_route_flow_counter_supported # noqa F401 -import tests.arp.test_wr_arp as test_wr_arp +from tests.common.flow_counter.flow_counter_utils import RouteFlowCounterTestContext, is_route_flow_counter_supported # noqa F401 +from tests.common.arp_utils import set_up, tear_down, testWrArp +from tests.common.fixtures.ptfhost_utils import skip_traffic_test from tests.common.config_reload import config_reload @@ -159,13 +157,11 @@ def vxlan_status(setup, request, duthosts, rand_one_dut_hostname, cleanup_dut_vnets(duthost, vnet_config) cleanup_vxlan_tunnels(duthost, vnet_test_params) elif request.param == "WR_ARP": - testWrArp = test_wr_arp.TestWrArp() - testWrArp.Setup(duthost, ptfhost, tbinfo) + route, ptfIp, gwIp = set_up(duthost, ptfhost, tbinfo) try: - test_wr_arp.TestWrArp.testWrArp( - testWrArp, request, duthost, ptfhost, creds) + testWrArp(request, duthost, ptfhost, creds, skip_traffic_test) finally: - testWrArp.Teardown(duthost) + tear_down(duthost, route, ptfIp, gwIp) return vxlan_enabled, request.param @@ -194,7 +190,7 @@ def is_neigh_reachable(duthost, vnet_config): def test_vnet_vxlan(setup, vxlan_status, duthosts, rand_one_dut_hostname, ptfhost, - vnet_test_params, creds, is_route_flow_counter_supported): # noqa F811 + vnet_test_params, creds, is_route_flow_counter_supported, skip_traffic_test): # noqa F811 """ Test case for VNET VxLAN @@ -233,6 +229,9 @@ def test_vnet_vxlan(setup, vxlan_status, duthosts, rand_one_dut_hostname, ptfhos logger.info("Skipping cleanup") pytest.skip("Skip cleanup specified") + if skip_traffic_test is True: + logger.info("Skipping traffic test") + return logger.debug("Starting PTF runner") if scenario == 'Enabled' and vxlan_enabled: route_pattern = 'Vnet1|100.1.1.1/32' diff --git a/tests/vxlan/test_vxlan_bfd_tsa.py b/tests/vxlan/test_vxlan_bfd_tsa.py index 89dc07279b..1538efd891 100644 --- a/tests/vxlan/test_vxlan_bfd_tsa.py +++ b/tests/vxlan/test_vxlan_bfd_tsa.py @@ -13,10 +13,10 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until -from tests.common.fixtures.ptfhost_utils \ - import copy_ptftests_directory # noqa: F401 +from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.ptf_runner import ptf_runner -from tests.vxlan.vxlan_ecmp_utils import Ecmp_Utils +from tests.common.vxlan_ecmp_utils import Ecmp_Utils from tests.common.config_reload import config_system_checks_passed Logger = logging.getLogger(__name__) ecmp_utils = Ecmp_Utils() @@ -83,7 +83,7 @@ def fixture_setUp(duthosts, ''' data = {} asic_type = duthosts[rand_one_dut_hostname].facts["asic_type"] - if asic_type in ["cisco-8000", "mellanox"]: + if asic_type in ["cisco-8000", "mellanox", "vs"]: data['tolerance'] = 0.03 else: raise RuntimeError("Pls update this script for your platform.") @@ -238,7 +238,8 @@ def dump_self_info_and_run_ptf(self, random_sport=False, random_src_ip=False, tolerance=None, - payload=None): + payload=None, + skip_traffic_test=False): # noqa F811 ''' Just a wrapper for dump_info_to_ptf to avoid entering 30 lines everytime. @@ -290,6 +291,9 @@ def dump_self_info_and_run_ptf(self, Logger.info( "dest->nh mapping:%s", self.vxlan_test_setup[encap_type]['dest_to_nh_map']) + if skip_traffic_test is True: + Logger.info("Skipping traffic test.") + return ptf_runner(self.vxlan_test_setup['ptfhost'], "ptftests", "vxlan_traffic.VxLAN_in_VxLAN" if payload == 'vxlan' @@ -407,7 +411,7 @@ def verfiy_bfd_down(self, ep_list): return False return True - def test_tsa_case1(self, setUp, encap_type): + def test_tsa_case1(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc1: This test checks the basic TSA removal of BFD sessions. 1) Create Vnet route with 4 endpoints and BFD monitors. @@ -424,7 +428,7 @@ def test_tsa_case1(self, setUp, encap_type): dest, ep_list = self.create_vnet_route(encap_type) - self.dump_self_info_and_run_ptf("test1", encap_type, True, []) + self.dump_self_info_and_run_ptf("test1", encap_type, True, [], skip_traffic_test=skip_traffic_test) self.apply_tsa() pytest_assert(self.in_maintainence()) @@ -433,11 +437,11 @@ def test_tsa_case1(self, setUp, encap_type): self.apply_tsb() pytest_assert(not self.in_maintainence()) - self.dump_self_info_and_run_ptf("test1b", encap_type, True, []) + self.dump_self_info_and_run_ptf("test1b", encap_type, True, [], skip_traffic_test=skip_traffic_test) self.delete_vnet_route(encap_type, dest) - def test_tsa_case2(self, setUp, encap_type): + def test_tsa_case2(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc2: This test checks the basic route application while in TSA. 1) apply TSA. @@ -460,11 +464,11 @@ def test_tsa_case2(self, setUp, encap_type): self.apply_tsb() pytest_assert(not self.in_maintainence()) - self.dump_self_info_and_run_ptf("test2", encap_type, True, []) + self.dump_self_info_and_run_ptf("test2", encap_type, True, [], skip_traffic_test=skip_traffic_test) self.delete_vnet_route(encap_type, dest) - def test_tsa_case3(self, setUp, encap_type): + def test_tsa_case3(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc3: This test checks for lasting impact of TSA and TSB. 1) apply TSA. @@ -487,11 +491,11 @@ def test_tsa_case3(self, setUp, encap_type): dest, ep_list = self.create_vnet_route(encap_type) - self.dump_self_info_and_run_ptf("test3", encap_type, True, []) + self.dump_self_info_and_run_ptf("test3", encap_type, True, [], skip_traffic_test=skip_traffic_test) self.delete_vnet_route(encap_type, dest) - def test_tsa_case4(self, setUp, encap_type): + def test_tsa_case4(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc4: This test checks basic Vnet route state retention during config reload. 1) Create Vnet route with 4 endpoints and BFD monitors. @@ -510,7 +514,7 @@ def test_tsa_case4(self, setUp, encap_type): duthost.shell("sudo config save -y", executable="/bin/bash", module_ignore_errors=True) - self.dump_self_info_and_run_ptf("test4", encap_type, True, []) + self.dump_self_info_and_run_ptf("test4", encap_type, True, [], skip_traffic_test=skip_traffic_test) duthost.shell("sudo config reload -y", executable="/bin/bash", module_ignore_errors=True) @@ -520,11 +524,11 @@ def test_tsa_case4(self, setUp, encap_type): ecmp_utils.configure_vxlan_switch(duthost, vxlan_port=4789, dutmac=self.vxlan_test_setup['dut_mac']) dest, ep_list = self.create_vnet_route(encap_type) - self.dump_self_info_and_run_ptf("test4b", encap_type, True, []) + self.dump_self_info_and_run_ptf("test4b", encap_type, True, [], skip_traffic_test=skip_traffic_test) self.delete_vnet_route(encap_type, dest) - def test_tsa_case5(self, setUp, encap_type): + def test_tsa_case5(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc4: This test checks TSA state retention w.r.t BFD accross config reload. 1) Create Vnet route with 4 endpoints and BFD monitors. @@ -548,7 +552,7 @@ def test_tsa_case5(self, setUp, encap_type): duthost.shell("sudo config save -y", executable="/bin/bash", module_ignore_errors=True) - self.dump_self_info_and_run_ptf("test5", encap_type, True, []) + self.dump_self_info_and_run_ptf("test5", encap_type, True, [], skip_traffic_test=skip_traffic_test) self.apply_tsa() pytest_assert(self.in_maintainence()) @@ -565,11 +569,11 @@ def test_tsa_case5(self, setUp, encap_type): self.apply_tsb() pytest_assert(not self.in_maintainence()) - self.dump_self_info_and_run_ptf("test5b", encap_type, True, []) + self.dump_self_info_and_run_ptf("test5b", encap_type, True, [], skip_traffic_test=skip_traffic_test) self.delete_vnet_route(encap_type, dest) - def test_tsa_case6(self, setUp, encap_type): + def test_tsa_case6(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc6: This test checks that the BFD doesnt come up while device is in TSA and remains down accross config reload. @@ -611,6 +615,6 @@ def test_tsa_case6(self, setUp, encap_type): self.apply_tsb() pytest_assert(not self.in_maintainence()) - self.dump_self_info_and_run_ptf("test6", encap_type, True, []) + self.dump_self_info_and_run_ptf("test6", encap_type, True, [], skip_traffic_test=skip_traffic_test) self.delete_vnet_route(encap_type, dest) diff --git a/tests/vxlan/test_vxlan_crm.py b/tests/vxlan/test_vxlan_crm.py index 66e9bd03e8..83404e9e64 100644 --- a/tests/vxlan/test_vxlan_crm.py +++ b/tests/vxlan/test_vxlan_crm.py @@ -4,9 +4,8 @@ from functools import reduce from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.ptfhost_utils \ - import copy_ptftests_directory # noqa: F401 -from tests.vxlan.vxlan_ecmp_utils import Ecmp_Utils +from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa: F401 +from tests.common.vxlan_ecmp_utils import Ecmp_Utils from tests.vxlan.test_vxlan_ecmp import ( # noqa: F401 Test_VxLAN, fixture_setUp, @@ -21,6 +20,21 @@ ecmp_utils = Ecmp_Utils() +@pytest.fixture(autouse=True) +def _ignore_route_sync_errlogs(duthosts, rand_one_dut_hostname, loganalyzer): + """Ignore expected failures logs during test execution.""" + if loganalyzer: + # Ignore in KVM test + KVMIgnoreRegex = [ + ".*missed_in_asic_db_routes.*", + ".*Vnet Route Mismatch reported.*", + ] + duthost = duthosts[rand_one_dut_hostname] + if duthost.facts["asic_type"] == "vs": + loganalyzer[rand_one_dut_hostname].ignore_regex.extend(KVMIgnoreRegex) + return + + def uniq(lst): last = object() for item in sorted(lst): diff --git a/tests/vxlan/test_vxlan_decap.py b/tests/vxlan/test_vxlan_decap.py index 38df368c42..b6761ac25f 100644 --- a/tests/vxlan/test_vxlan_decap.py +++ b/tests/vxlan/test_vxlan_decap.py @@ -14,6 +14,7 @@ from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 from tests.common.fixtures.ptfhost_utils import copy_arp_responder_py # noqa F401 from tests.common.fixtures.ptfhost_utils import remove_ip_addresses # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.ptf_runner import ptf_runner from tests.common.dualtor.mux_simulator_control import mux_server_url,\ toggle_all_simulator_ports_to_rand_selected_tor_m # noqa F401 @@ -184,7 +185,7 @@ def vxlan_status(setup, request, duthosts, rand_one_dut_hostname): def test_vxlan_decap(setup, vxlan_status, duthosts, rand_one_dut_hostname, tbinfo, - ptfhost, creds, toggle_all_simulator_ports_to_rand_selected_tor_m): # noqa F811 + ptfhost, creds, toggle_all_simulator_ports_to_rand_selected_tor_m, skip_traffic_test): # noqa F811 duthost = duthosts[rand_one_dut_hostname] sonic_admin_alt_password = duthost.host.options['variable_manager'].\ @@ -197,6 +198,10 @@ def test_vxlan_decap(setup, vxlan_status, duthosts, rand_one_dut_hostname, tbinf logger.info("vxlan_enabled=%s, scenario=%s" % (vxlan_enabled, scenario)) log_file = "/tmp/vxlan-decap.Vxlan.{}.{}.log".format( scenario, datetime.now().strftime('%Y-%m-%d-%H:%M:%S')) + + if skip_traffic_test is True: + logger.info("Skip traffic test") + return ptf_runner(ptfhost, "ptftests", "vxlan-decap.Vxlan", diff --git a/tests/vxlan/test_vxlan_ecmp.py b/tests/vxlan/test_vxlan_ecmp.py index f021581d11..028e856fa2 100644 --- a/tests/vxlan/test_vxlan_ecmp.py +++ b/tests/vxlan/test_vxlan_ecmp.py @@ -59,11 +59,11 @@ import copy from tests.common.helpers.assertions import pytest_assert -from tests.common.fixtures.ptfhost_utils \ - import copy_ptftests_directory # noqa: F401 +from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.common.utilities import wait_until from tests.ptf_runner import ptf_runner -from tests.vxlan.vxlan_ecmp_utils import Ecmp_Utils +from tests.common.vxlan_ecmp_utils import Ecmp_Utils Logger = logging.getLogger(__name__) ecmp_utils = Ecmp_Utils() @@ -107,6 +107,15 @@ def _ignore_route_sync_errlogs(rand_one_dut_hostname, loganalyzer): ".*'vnetRouteCheck' status failed.*", ".*Vnet Route Mismatch reported.*", ".*_M_construct null not valid.*", + ".*Failed to bind BFD socket to local_addr.*-98.*", + ".*Failed to create TX socket for session.*-5.*", + ".*Parsing BFD command.*-5.*", + ".*[BFD.ERR] ioctl failed.*", + ".*[CORE_API.ERR] Failed in bfd_offload_set.*", + ".*mlnx_set_offload_bfd_tx_session.*", + ".*api SAI_COMMON_API_CREATE failed in syncd mode.*", + ".*ERR syncd.*SAI_BFD_SESSION_ATTR_.*", + ".*ERR swss.*SAI_STATUS_FAILURE.*", ]) return @@ -148,7 +157,7 @@ def fixture_setUp(duthosts, data = {} asic_type = duthosts[rand_one_dut_hostname].facts["asic_type"] - if asic_type in ["cisco-8000", "mellanox"]: + if asic_type in ["cisco-8000", "mellanox", "vs"]: data['tolerance'] = 0.03 else: raise RuntimeError("Pls update this script for your platform.") @@ -385,7 +394,8 @@ def dump_self_info_and_run_ptf(self, random_sport=False, random_src_ip=False, tolerance=None, - payload=None): + payload=None, + skip_traffic_test=False): # noqa F811 ''' Just a wrapper for dump_info_to_ptf to avoid entering 30 lines everytime. @@ -440,6 +450,9 @@ def dump_self_info_and_run_ptf(self, Logger.info( "dest->nh mapping:%s", self.vxlan_test_setup[encap_type]['dest_to_nh_map']) + if skip_traffic_test is True: + Logger.info("Skipping traffic test.") + return ptf_runner(self.vxlan_test_setup['ptfhost'], "ptftests", "vxlan_traffic.VxLAN_in_VxLAN" if payload == 'vxlan' @@ -497,18 +510,18 @@ class Test_VxLAN_route_tests(Test_VxLAN): Common class for the basic route test cases. ''' - def test_vxlan_single_endpoint(self, setUp, encap_type): + def test_vxlan_single_endpoint(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc1:Create a tunnel route to a single endpoint a. Send packets to the route prefix dst. ''' self.vxlan_test_setup = setUp - self.dump_self_info_and_run_ptf("tc1", encap_type, True) + self.dump_self_info_and_run_ptf("tc1", encap_type, True, skip_traffic_test=skip_traffic_test) self.dump_self_info_and_run_ptf("tc1", encap_type, True, - payload="vxlan") + payload="vxlan", skip_traffic_test=skip_traffic_test) def test_vxlan_modify_route_different_endpoint( - self, setUp, request, encap_type): + self, setUp, request, encap_type, skip_traffic_test): # noqa F811 ''' tc2: change the route to different endpoint. Packets are received only at endpoint b.") @@ -558,9 +571,9 @@ def test_vxlan_modify_route_different_endpoint( Logger.info( "Copy the new set of configs to the PTF and run the tests.") - self.dump_self_info_and_run_ptf("tc2", encap_type, True) + self.dump_self_info_and_run_ptf("tc2", encap_type, True, skip_traffic_test=skip_traffic_test) - def test_vxlan_remove_all_route(self, setUp, encap_type): + def test_vxlan_remove_all_route(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc3: remove the tunnel route. Send packets to the route prefix dst. packets should not @@ -575,7 +588,7 @@ def test_vxlan_remove_all_route(self, setUp, encap_type): ecmp_utils.get_payload_version(encap_type), "DEL") Logger.info("Verify that the traffic is not coming back.") - self.dump_self_info_and_run_ptf("tc3", encap_type, False) + self.dump_self_info_and_run_ptf("tc3", encap_type, False, skip_traffic_test=skip_traffic_test) finally: Logger.info("Restore the routes in the DUT.") ecmp_utils.set_routes_in_dut( @@ -592,7 +605,7 @@ class Test_VxLAN_ecmp_create(Test_VxLAN): create testcases. ''' - def test_vxlan_configure_route1_ecmp_group_a(self, setUp, encap_type): + def test_vxlan_configure_route1_ecmp_group_a(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc4:create tunnel route 1 with two endpoints a = {a1, a2...}. send packets to the route 1's prefix dst. packets are received at either @@ -633,12 +646,12 @@ def test_vxlan_configure_route1_ecmp_group_a(self, setUp, encap_type): Logger.info("Verify that the new config takes effect and run traffic.") - self.dump_self_info_and_run_ptf("tc4", encap_type, True) + self.dump_self_info_and_run_ptf("tc4", encap_type, True, skip_traffic_test=skip_traffic_test) # Add vxlan payload testing as well. self.dump_self_info_and_run_ptf("tc4", encap_type, True, - payload="vxlan") + payload="vxlan", skip_traffic_test=skip_traffic_test) - def test_vxlan_remove_ecmp_route1(self, setUp, encap_type): + def test_vxlan_remove_ecmp_route1(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' Remove tunnel route 1. Send multiple packets (varying tuple) to the route 1's prefix dst. @@ -682,7 +695,7 @@ def test_vxlan_remove_ecmp_route1(self, setUp, encap_type): ecmp_route1_end_point_list) Logger.info("Verify that the new config takes effect and run traffic.") - self.dump_self_info_and_run_ptf("tc5", encap_type, True) + self.dump_self_info_and_run_ptf("tc5", encap_type, True, skip_traffic_test=skip_traffic_test) # Deleting Tunnel route 1 ecmp_utils.create_and_apply_config( @@ -697,13 +710,13 @@ def test_vxlan_remove_ecmp_route1(self, setUp, encap_type): {ecmp_route1_new_dest: ecmp_route1_end_point_list} Logger.info("Verify that the new config takes effect and run traffic.") - self.dump_self_info_and_run_ptf("tc5", encap_type, False) + self.dump_self_info_and_run_ptf("tc5", encap_type, False, skip_traffic_test=skip_traffic_test) # Restoring dest_to_nh_map to old values self.vxlan_test_setup[encap_type]['dest_to_nh_map'][vnet] = copy.deepcopy(backup_dest) - self.dump_self_info_and_run_ptf("tc5", encap_type, True) + self.dump_self_info_and_run_ptf("tc5", encap_type, True, skip_traffic_test=skip_traffic_test) - def test_vxlan_configure_route1_ecmp_group_b(self, setUp, encap_type): + def test_vxlan_configure_route1_ecmp_group_b(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc5: set tunnel route 2 to endpoint group a = {a1, a2}. send packets to route 2"s prefix dst. packets are received at either a1 @@ -712,7 +725,7 @@ def test_vxlan_configure_route1_ecmp_group_b(self, setUp, encap_type): self.vxlan_test_setup = setUp self.setup_route2_ecmp_group_b(encap_type) Logger.info("Verify the configs work and traffic flows correctly.") - self.dump_self_info_and_run_ptf("tc5", encap_type, True) + self.dump_self_info_and_run_ptf("tc5", encap_type, True, skip_traffic_test=skip_traffic_test) def setup_route2_ecmp_group_b(self, encap_type): ''' @@ -754,7 +767,7 @@ def setup_route2_ecmp_group_b(self, encap_type): self.vxlan_test_setup[encap_type]['tc5_dest'] = tc5_new_dest - def test_vxlan_configure_route2_ecmp_group_b(self, setUp, encap_type): + def test_vxlan_configure_route2_ecmp_group_b(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc6: set tunnel route 2 to endpoint group b = {b1, b2}. send packets to route 2"s prefix dst. packets are received at either @@ -796,13 +809,13 @@ def test_vxlan_configure_route2_ecmp_group_b(self, setUp, encap_type): tc6_end_point_list) Logger.info("Verify that the traffic works.") - self.dump_self_info_and_run_ptf("tc6", encap_type, True) + self.dump_self_info_and_run_ptf("tc6", encap_type, True, skip_traffic_test=skip_traffic_test) @pytest.mark.skipif( "config.option.bfd is False", reason="This test will be run only if '--bfd=True' is provided.") def test_vxlan_bfd_health_state_change_a2down_a1up( - self, setUp, encap_type): + self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' Set BFD state for a1' to UP and a2' to Down. Send multiple packets (varying tuple) to the route 1's prefix dst. Packets are received @@ -850,12 +863,12 @@ def test_vxlan_bfd_health_state_change_a2down_a1up( end_point_list[1]) Logger.info("Verify that the new config takes effect and run traffic.") - self.dump_self_info_and_run_ptf("tc_a2down_a1up", encap_type, True) + self.dump_self_info_and_run_ptf("tc_a2down_a1up", encap_type, True, skip_traffic_test=skip_traffic_test) @pytest.mark.skipif( "config.option.bfd is False", reason="This test will be run only if '--bfd=True' is provided.") - def test_vxlan_bfd_health_state_change_a1a2_down(self, setUp, encap_type): + def test_vxlan_bfd_health_state_change_a1a2_down(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' Set BFD state for a1' to Down and a2' to Down. Send multiple packets (varying tuple) to the route 1's prefix dst. Packets @@ -902,13 +915,14 @@ def test_vxlan_bfd_health_state_change_a1a2_down(self, setUp, encap_type): "a1a2_down", encap_type, True, - packet_count=4) + packet_count=4, + skip_traffic_test=skip_traffic_test) @pytest.mark.skipif( "config.option.bfd is False", reason="This test will be run only if '--bfd=True' is provided.") def test_vxlan_bfd_health_state_change_a2up_a1down( - self, setUp, encap_type): + self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' Set BFD state for a2' to UP. Send packets to the route 1's prefix dst. Packets are received only at endpoint a2. Verify advertise @@ -956,9 +970,9 @@ def test_vxlan_bfd_health_state_change_a2up_a1down( end_point_list[0]) Logger.info("Verify that the new config takes effect and run traffic.") - self.dump_self_info_and_run_ptf("a2up_a1down", encap_type, True) + self.dump_self_info_and_run_ptf("a2up_a1down", encap_type, True, skip_traffic_test=skip_traffic_test) - def test_vxlan_bfd_health_state_change_a1a2_up(self, setUp, encap_type): + def test_vxlan_bfd_health_state_change_a1a2_up(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' Set BFD state for a1' & a2' to UP. Send multiple packets (varying tuple) to the route 1's prefix dst. Packets are received at both @@ -1001,7 +1015,7 @@ def test_vxlan_bfd_health_state_change_a1a2_up(self, setUp, encap_type): Logger.info("Verify that the new config takes effect and run traffic.") - self.dump_self_info_and_run_ptf("tc4", encap_type, True) + self.dump_self_info_and_run_ptf("tc4", encap_type, True, skip_traffic_test=skip_traffic_test) # perform cleanup by removing all the routes added by this test class. # reset to add only the routes added in the setup phase. @@ -1192,7 +1206,7 @@ def setup_route2_shared_different_endpoints(self, encap_type): encap_type, tc9_new_nhs) - def test_vxlan_remove_route2(self, setUp, encap_type): + def test_vxlan_remove_route2(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc7:send packets to route 1's prefix dst. by removing route 2 from group a, no change expected to route 1. @@ -1238,7 +1252,7 @@ def test_vxlan_remove_route2(self, setUp, encap_type): encap_type, tc7_end_point_list) Logger.info("Verify the setup works.") - self.dump_self_info_and_run_ptf("tc7", encap_type, True) + self.dump_self_info_and_run_ptf("tc7", encap_type, True, skip_traffic_test=skip_traffic_test) Logger.info("End of setup.") Logger.info("Remove one of the routes.") @@ -1258,7 +1272,7 @@ def test_vxlan_remove_route2(self, setUp, encap_type): "DEL") Logger.info("Verify the rest of the traffic still works.") - self.dump_self_info_and_run_ptf("tc7", encap_type, True) + self.dump_self_info_and_run_ptf("tc7", encap_type, True, skip_traffic_test=skip_traffic_test) # perform cleanup by removing all the routes added by this test class. # reset to add only the routes added in the setup phase. @@ -1275,18 +1289,18 @@ def test_vxlan_remove_route2(self, setUp, encap_type): ecmp_utils.get_payload_version(encap_type), "SET") - def test_vxlan_route2_single_nh(self, setUp, encap_type): + def test_vxlan_route2_single_nh(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc8: set tunnel route 2 to single endpoint b1. Send packets to route 2's prefix dst. ''' self.vxlan_test_setup = setUp self.setup_route2_single_endpoint(encap_type) - self.dump_self_info_and_run_ptf("tc8", encap_type, True) + self.dump_self_info_and_run_ptf("tc8", encap_type, True, skip_traffic_test=skip_traffic_test) self.dump_self_info_and_run_ptf("tc8", encap_type, True, - payload="vxlan") + payload="vxlan", skip_traffic_test=skip_traffic_test) - def test_vxlan_route2_shared_nh(self, setUp, encap_type): + def test_vxlan_route2_shared_nh(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc9: set tunnel route 2 to shared endpoints a1 and b1. Send packets to route 2's @@ -1294,9 +1308,9 @@ def test_vxlan_route2_shared_nh(self, setUp, encap_type): ''' self.vxlan_test_setup = setUp self.setup_route2_shared_endpoints(encap_type) - self.dump_self_info_and_run_ptf("tc9", encap_type, True) + self.dump_self_info_and_run_ptf("tc9", encap_type, True, skip_traffic_test=skip_traffic_test) - def test_vxlan_route2_shared_different_nh(self, setUp, encap_type): + def test_vxlan_route2_shared_different_nh(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc9.2: set tunnel route 2 to 2 completely different shared(no-reuse) endpoints a1 and b1. send packets @@ -1304,9 +1318,9 @@ def test_vxlan_route2_shared_different_nh(self, setUp, encap_type): ''' self.vxlan_test_setup = setUp self.setup_route2_shared_different_endpoints(encap_type) - self.dump_self_info_and_run_ptf("tc9.2", encap_type, True) + self.dump_self_info_and_run_ptf("tc9.2", encap_type, True, skip_traffic_test=skip_traffic_test) - def test_vxlan_remove_ecmp_route2(self, setUp, encap_type): + def test_vxlan_remove_ecmp_route2(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc10: remove tunnel route 2. send packets to route 2's prefix dst. ''' @@ -1355,7 +1369,7 @@ def test_vxlan_remove_ecmp_route2(self, setUp, encap_type): tc10_nhs Logger.info("The deleted route should fail to receive traffic.") - self.dump_self_info_and_run_ptf("tc10", encap_type, False) + self.dump_self_info_and_run_ptf("tc10", encap_type, False, skip_traffic_test=skip_traffic_test) # all others should be working. # Housekeeping: @@ -1366,7 +1380,7 @@ def test_vxlan_remove_ecmp_route2(self, setUp, encap_type): del_needed = False Logger.info("Check the traffic is working in the other routes.") - self.dump_self_info_and_run_ptf("tc10", encap_type, True) + self.dump_self_info_and_run_ptf("tc10", encap_type, True, skip_traffic_test=skip_traffic_test) except BaseException: self.vxlan_test_setup[encap_type]['dest_to_nh_map'][vnet] = full_map.copy() @@ -1385,7 +1399,7 @@ class Test_VxLAN_ecmp_random_hash(Test_VxLAN): Class for testing different tcp ports for payload. ''' - def test_vxlan_random_hash(self, setUp, encap_type): + def test_vxlan_random_hash(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc11: set tunnel route 3 to endpoint group c = {c1, c2, c3}. Ensure c1, c2, and c3 matches to underlay default route. @@ -1434,7 +1448,8 @@ def test_vxlan_random_hash(self, setUp, encap_type): "tc11", encap_type, True, - packet_count=1000) + packet_count=1000, + skip_traffic_test=skip_traffic_test) @pytest.mark.skipif( @@ -1447,7 +1462,7 @@ class Test_VxLAN_underlay_ecmp(Test_VxLAN): ''' @pytest.mark.parametrize("ecmp_path_count", [1, 2]) def test_vxlan_modify_underlay_default( - self, setUp, minigraph_facts, encap_type, ecmp_path_count): + self, setUp, minigraph_facts, encap_type, ecmp_path_count, skip_traffic_test): # noqa F811 ''' tc12: modify the underlay default route nexthop/s. send packets to route 3's prefix dst. @@ -1519,7 +1534,8 @@ def test_vxlan_modify_underlay_default( "tc12", encap_type, True, - packet_count=1000) + packet_count=1000, + skip_traffic_test=skip_traffic_test) Logger.info( "Reverse the action: bring up the selected_intfs" @@ -1566,7 +1582,8 @@ def test_vxlan_modify_underlay_default( "tc12", encap_type, True, - packet_count=1000) + packet_count=1000, + skip_traffic_test=skip_traffic_test) Logger.info("Recovery. Bring all up, and verify traffic works.") for intf in all_t2_intfs: @@ -1594,7 +1611,8 @@ def test_vxlan_modify_underlay_default( "tc12", encap_type, True, - packet_count=1000) + packet_count=1000, + skip_traffic_test=skip_traffic_test) except Exception: # If anything goes wrong in the try block, atleast bring the intf @@ -1622,7 +1640,8 @@ def test_vxlan_modify_underlay_default( def test_vxlan_remove_add_underlay_default(self, setUp, minigraph_facts, - encap_type): + encap_type, + skip_traffic_test): # noqa F811 ''' tc13: remove the underlay default route. tc14: add the underlay default route. @@ -1663,7 +1682,7 @@ def test_vxlan_remove_add_underlay_default(self, "BGP neighbors have not reached the required state after " "T2 intf are shutdown.") Logger.info("Verify that traffic is not flowing through.") - self.dump_self_info_and_run_ptf("tc13", encap_type, False) + self.dump_self_info_and_run_ptf("tc13", encap_type, False, skip_traffic_test=skip_traffic_test) # tc14: Re-add the underlay default route. Logger.info("Bring up the T2 interfaces.") @@ -1685,7 +1704,8 @@ def test_vxlan_remove_add_underlay_default(self, "tc14", encap_type, True, - packet_count=1000) + packet_count=1000, + skip_traffic_test=skip_traffic_test) except Exception: Logger.info( "If anything goes wrong in the try block," @@ -1704,7 +1724,7 @@ def test_vxlan_remove_add_underlay_default(self, " interfaces have been brought up.") raise - def test_underlay_specific_route(self, setUp, minigraph_facts, encap_type): + def test_underlay_specific_route(self, setUp, minigraph_facts, encap_type, skip_traffic_test): # noqa F811 ''' Create a more specific underlay route to c1. Verify c1 packets are received only on the c1's nexthop interface @@ -1775,7 +1795,8 @@ def test_underlay_specific_route(self, setUp, minigraph_facts, encap_type): self.dump_self_info_and_run_ptf( "underlay_specific_route", encap_type, - True) + True, + skip_traffic_test=skip_traffic_test) # Deletion of all static routes gateway = all_t2_neighbors[t2_neighbor][outer_layer_version].lower() for _, nexthops in list(endpoint_nhmap.items()): @@ -1823,12 +1844,14 @@ def test_underlay_specific_route(self, setUp, minigraph_facts, encap_type): self.dump_self_info_and_run_ptf( "underlay_specific_route", encap_type, - True) + True, + skip_traffic_test=skip_traffic_test) def test_underlay_portchannel_shutdown(self, setUp, minigraph_facts, - encap_type): + encap_type, + skip_traffic_test): # noqa F811 ''' Bring down one of the port-channels. Packets are equally recieved at c1, c2 or c3 @@ -1836,7 +1859,7 @@ def test_underlay_portchannel_shutdown(self, self.vxlan_test_setup = setUp # Verification of traffic before shutting down port channel - self.dump_self_info_and_run_ptf("tc12", encap_type, True) + self.dump_self_info_and_run_ptf("tc12", encap_type, True, skip_traffic_test=skip_traffic_test) # Gathering all portchannels all_t2_portchannel_intfs = \ @@ -1871,7 +1894,7 @@ def test_underlay_portchannel_shutdown(self, self.vxlan_test_setup[encap_type]['t2_ports'], list(self.vxlan_test_setup['list_of_bfd_monitors'])) time.sleep(10) - self.dump_self_info_and_run_ptf("tc12", encap_type, True) + self.dump_self_info_and_run_ptf("tc12", encap_type, True, skip_traffic_test=skip_traffic_test) for intf in all_t2_portchannel_members[selected_portchannel]: self.vxlan_test_setup['duthost'].shell( @@ -1883,7 +1906,7 @@ def test_underlay_portchannel_shutdown(self, self.vxlan_test_setup[encap_type]['t2_ports'], list(self.vxlan_test_setup['list_of_bfd_monitors'])) time.sleep(10) - self.dump_self_info_and_run_ptf("tc12", encap_type, True) + self.dump_self_info_and_run_ptf("tc12", encap_type, True, skip_traffic_test=skip_traffic_test) except BaseException: for intf in all_t2_portchannel_members[selected_portchannel]: self.vxlan_test_setup['duthost'].shell( @@ -1913,7 +1936,8 @@ def verify_entropy( random_sport=False, random_dport=True, random_src_ip=False, - tolerance=None): + tolerance=None, + skip_traffic_test=False): # noqa F811 ''' Function to be reused by the entropy testcases. Sets up a couple of endpoints on the top of the existing ones, and performs the traffic @@ -1957,9 +1981,10 @@ def verify_entropy( random_dport=random_dport, random_src_ip=random_src_ip, packet_count=1000, - tolerance=tolerance) + tolerance=tolerance, + skip_traffic_test=skip_traffic_test) - def test_verify_entropy(self, setUp, encap_type): + def test_verify_entropy(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' Verification of entropy - Create tunnel route 4 to endpoint group A. Send packets (fixed tuple) to route 4's prefix dst @@ -1970,9 +1995,10 @@ def test_verify_entropy(self, setUp, encap_type): random_dport=True, random_sport=True, random_src_ip=True, - tolerance=0.75) # More tolerance since this varies entropy a lot. + tolerance=0.75, # More tolerance since this varies entropy a lot. + skip_traffic_test=skip_traffic_test) - def test_vxlan_random_dst_port(self, setUp, encap_type): + def test_vxlan_random_dst_port(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' Verification of entropy - Change the udp dst port of original packet to route 4's prefix dst @@ -1980,7 +2006,7 @@ def test_vxlan_random_dst_port(self, setUp, encap_type): self.vxlan_test_setup = setUp self.verify_entropy(encap_type, tolerance=0.03) - def test_vxlan_random_src_port(self, setUp, encap_type): + def test_vxlan_random_src_port(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' Verification of entropy - Change the udp src port of original packet to route 4's prefix dst @@ -1990,9 +2016,10 @@ def test_vxlan_random_src_port(self, setUp, encap_type): encap_type, random_dport=False, random_sport=True, - tolerance=0.03) + tolerance=0.03, + skip_traffic_test=skip_traffic_test) - def test_vxlan_varying_src_ip(self, setUp, encap_type): + def test_vxlan_varying_src_ip(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' Verification of entropy - Change the udp src ip of original packet to route 4's prefix dst @@ -2002,4 +2029,5 @@ def test_vxlan_varying_src_ip(self, setUp, encap_type): encap_type, random_dport=False, random_src_ip=True, - tolerance=0.03) + tolerance=0.03, + skip_traffic_test=skip_traffic_test) diff --git a/tests/vxlan/test_vxlan_ecmp_switchover.py b/tests/vxlan/test_vxlan_ecmp_switchover.py index 796e9e4b04..dd8d2a5501 100644 --- a/tests/vxlan/test_vxlan_ecmp_switchover.py +++ b/tests/vxlan/test_vxlan_ecmp_switchover.py @@ -10,10 +10,10 @@ import json import pytest -from tests.common.fixtures.ptfhost_utils \ - import copy_ptftests_directory # noqa: F401 +from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.common.fixtures.ptfhost_utils import skip_traffic_test # noqa F401 from tests.ptf_runner import ptf_runner -from tests.vxlan.vxlan_ecmp_utils import Ecmp_Utils +from tests.common.vxlan_ecmp_utils import Ecmp_Utils Logger = logging.getLogger(__name__) ecmp_utils = Ecmp_Utils() @@ -41,19 +41,26 @@ def fixture_encap_type(request): @pytest.fixture(autouse=True) -def _ignore_route_sync_errlogs(rand_one_dut_hostname, loganalyzer): +def _ignore_route_sync_errlogs(duthosts, rand_one_dut_hostname, loganalyzer): """Ignore expected failures logs during test execution.""" if loganalyzer: - loganalyzer[rand_one_dut_hostname].ignore_regex.extend( - [ - ".*Unaccounted_ROUTE_ENTRY_TABLE_entries.*", - ".*missed_in_asic_db_routes.*", - ".*Look at reported mismatches above.*", - ".*Unaccounted_ROUTE_ENTRY_TABLE_entries.*", - ".*'vnetRouteCheck' status failed.*", - ".*Vnet Route Mismatch reported.*", - ".*_M_construct null not valid.*", - ]) + IgnoreRegex = [ + ".*Unaccounted_ROUTE_ENTRY_TABLE_entries.*", + ".*missed_in_asic_db_routes.*", + ".*Look at reported mismatches above.*", + ".*Unaccounted_ROUTE_ENTRY_TABLE_entries.*", + ".*'vnetRouteCheck' status failed.*", + ".*Vnet Route Mismatch reported.*", + ".*_M_construct null not valid.*", + ] + # Ignore in KVM test + KVMIgnoreRegex = [ + ".*doTask: Logic error: basic_string: construction from null is not valid.*", + ] + duthost = duthosts[rand_one_dut_hostname] + loganalyzer[rand_one_dut_hostname].ignore_regex.extend(IgnoreRegex) + if duthost.facts["asic_type"] == "vs": + loganalyzer[rand_one_dut_hostname].ignore_regex.extend(KVMIgnoreRegex) return @@ -77,7 +84,7 @@ def fixture_setUp(duthosts, ''' data = {} asic_type = duthosts[rand_one_dut_hostname].facts["asic_type"] - if asic_type in ["cisco-8000", "mellanox"]: + if asic_type in ["cisco-8000", "mellanox", "vs"]: data['tolerance'] = 0.03 else: raise RuntimeError("Pls update this script for your platform.") @@ -221,7 +228,8 @@ def dump_self_info_and_run_ptf(self, random_sport=False, random_src_ip=False, tolerance=None, - payload=None): + payload=None, + skip_traffic_test=False): # noqa F811 ''' Just a wrapper for dump_info_to_ptf to avoid entering 30 lines everytime. @@ -274,6 +282,9 @@ def dump_self_info_and_run_ptf(self, Logger.info( "dest->nh mapping:%s", self.vxlan_test_setup[encap_type]['dest_to_nh_map']) + if skip_traffic_test is True: + Logger.info("Skipping traffic test.") + return ptf_runner(self.vxlan_test_setup['ptfhost'], "ptftests", "vxlan_traffic.VxLAN_in_VxLAN" if payload == 'vxlan' @@ -287,7 +298,7 @@ def dump_self_info_and_run_ptf(self, datetime.now().strftime('%Y-%m-%d-%H:%M:%S')), is_python3=True) - def test_vxlan_priority_single_pri_sec_switchover(self, setUp, encap_type): + def test_vxlan_priority_single_pri_sec_switchover(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc1:create tunnel route 1 with two endpoints a = {a1, b1}. a1 is primary, b1 is secondary. 1) both a1,b1 are UP. @@ -364,7 +375,7 @@ def test_vxlan_priority_single_pri_sec_switchover(self, setUp, encap_type): ecmp_utils.HOST_MASK[ecmp_utils.get_payload_version(encap_type)])) assert str(result['stdout']) == ecmp_utils.OVERLAY_DMAC - self.dump_self_info_and_run_ptf("test1", encap_type, True) + self.dump_self_info_and_run_ptf("test1", encap_type, True, skip_traffic_test=skip_traffic_test) # Single primary-secondary switchover. # Endpoint list = [A, A`], Primary[A] | Active NH=[A] | @@ -382,7 +393,7 @@ def test_vxlan_priority_single_pri_sec_switchover(self, setUp, encap_type): ecmp_utils.HOST_MASK[ecmp_utils.get_payload_version(encap_type)], tc1_end_point_list[0], "down") time.sleep(10) - self.dump_self_info_and_run_ptf("test1", encap_type, True) + self.dump_self_info_and_run_ptf("test1", encap_type, True, skip_traffic_test=skip_traffic_test) # Single primary recovery. # Endpoint list = [A, A`], Primary[A] | Active NH=[A`] | @@ -400,7 +411,7 @@ def test_vxlan_priority_single_pri_sec_switchover(self, setUp, encap_type): ecmp_utils.HOST_MASK[ecmp_utils.get_payload_version(encap_type)], tc1_end_point_list[0], "up") time.sleep(10) - self.dump_self_info_and_run_ptf("test1", encap_type, True) + self.dump_self_info_and_run_ptf("test1", encap_type, True, skip_traffic_test=skip_traffic_test) # Single primary backup Failure. # Endpoint list = [A, A`]. Primary[A]| Active NH=[A`] A is DOWN | @@ -422,7 +433,7 @@ def test_vxlan_priority_single_pri_sec_switchover(self, setUp, encap_type): tc1_end_point_list[0], "down") time.sleep(10) - self.dump_self_info_and_run_ptf("test1", encap_type, True) + self.dump_self_info_and_run_ptf("test1", encap_type, True, skip_traffic_test=skip_traffic_test) ecmp_utils.create_and_apply_priority_config( self.vxlan_test_setup['duthost'], vnet, @@ -442,7 +453,7 @@ def test_vxlan_priority_single_pri_sec_switchover(self, setUp, encap_type): [tc1_end_point_list[0]], "DEL") - def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): + def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type, skip_traffic_test): # noqa F811 ''' tc2:create tunnel route 1 with 6 endpoints a = {A, B, A`, B`}. A,B are primary, A`,B` are secondary. @@ -540,7 +551,7 @@ def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): self.vxlan_test_setup['list_of_downed_endpoints'] = set(inactive_list) time.sleep(10) # ensure that the traffic is distributed to all 3 primary Endpoints. - self.dump_self_info_and_run_ptf("test2", encap_type, True) + self.dump_self_info_and_run_ptf("test2", encap_type, True, skip_traffic_test=skip_traffic_test) # Multiple primary backups. Single primary failure. # Endpoint list = [A, B, A`, B`], Primary = [A, B] | active NH = [A, B] | @@ -558,7 +569,7 @@ def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): ecmp_utils.HOST_MASK[ecmp_utils.get_payload_version(encap_type)], primary_nhg[0], "down") time.sleep(10) - self.dump_self_info_and_run_ptf("test2", encap_type, True) + self.dump_self_info_and_run_ptf("test2", encap_type, True, skip_traffic_test=skip_traffic_test) # Multiple primary backups. All primary failure. # Endpoint list = [A, B, A`, B`] Primary = [A, B] | A is Down. active NH = [B] | @@ -575,7 +586,7 @@ def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): ecmp_utils.HOST_MASK[ecmp_utils.get_payload_version(encap_type)], primary_nhg[1], "down") time.sleep(10) - self.dump_self_info_and_run_ptf("test2", encap_type, True) + self.dump_self_info_and_run_ptf("test2", encap_type, True, skip_traffic_test=skip_traffic_test) # Multiple primary backups. Backup Failure. # Endpoint list = [A, B, A`, B`] Primary = [A, B] | @@ -594,7 +605,7 @@ def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): ecmp_utils.HOST_MASK[ecmp_utils.get_payload_version(encap_type)], secondary_nhg[1], "down") time.sleep(10) - self.dump_self_info_and_run_ptf("test2", encap_type, True) + self.dump_self_info_and_run_ptf("test2", encap_type, True, skip_traffic_test=skip_traffic_test) # Multiple primary backups. Single primary recovery. # Endpoint list = [A, B, A`, B`] Primary = [A, B] | Active NH = [A`] | @@ -612,7 +623,7 @@ def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): ecmp_utils.HOST_MASK[ecmp_utils.get_payload_version(encap_type)], primary_nhg[0], "up") time.sleep(10) - self.dump_self_info_and_run_ptf("test2", encap_type, True) + self.dump_self_info_and_run_ptf("test2", encap_type, True, skip_traffic_test=skip_traffic_test) # Multiple primary backups. Multiple primary & backup recovery. # Edpoint list = [A, B, A`, B`] Primary = [A, B] | Active NH = [A] | @@ -634,7 +645,7 @@ def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): secondary_nhg[1], "up") time.sleep(10) - self.dump_self_info_and_run_ptf("test2", encap_type, True) + self.dump_self_info_and_run_ptf("test2", encap_type, True, skip_traffic_test=skip_traffic_test) # Multiple primary backups. Multiple primary & backup all failure. # Edpoint list = [A, B, A`, B`] Primary = [A, B] | Active NH = [A,B] | @@ -663,7 +674,7 @@ def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): ecmp_utils.HOST_MASK[ecmp_utils.get_payload_version(encap_type)], secondary_nhg[1], "down") time.sleep(10) - self.dump_self_info_and_run_ptf("test2", encap_type, True) + self.dump_self_info_and_run_ptf("test2", encap_type, True, skip_traffic_test=skip_traffic_test) # Multiple primary backups. Multiple primary & backup recovery. # Edpoint list = [A, B, A`, B`] Primary = [A, B] | Active NH = [] | @@ -693,7 +704,7 @@ def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): secondary_nhg[1], "up") time.sleep(10) - self.dump_self_info_and_run_ptf("test2", encap_type, True) + self.dump_self_info_and_run_ptf("test2", encap_type, True, skip_traffic_test=skip_traffic_test) # Multiple primary backups. Multiple primary & backup all failure 2. # Edpoint list = [A, B, A`, B`] Primary = [A, B] | Active NH = [A,B] | @@ -722,7 +733,7 @@ def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): ecmp_utils.HOST_MASK[ecmp_utils.get_payload_version(encap_type)], secondary_nhg[1], "down") time.sleep(10) - self.dump_self_info_and_run_ptf("test2", encap_type, True) + self.dump_self_info_and_run_ptf("test2", encap_type, True, skip_traffic_test=skip_traffic_test) # Multiple primary backups. Multiple primary & backup recovery of secondary. # Edpoint list = [A, B, A`, B`] Primary = [A, B] | Active NH = [] | @@ -744,7 +755,7 @@ def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): secondary_nhg[1], "up") time.sleep(10) - self.dump_self_info_and_run_ptf("test2", encap_type, True) + self.dump_self_info_and_run_ptf("test2", encap_type, True, skip_traffic_test=skip_traffic_test) # Multiple primary backups. Multiple primary & backup recovery of primary after secondary. # Edpoint list = [A, B, A`, B`] Primary = [A, B] | Active NH = [A`, B`] | @@ -766,7 +777,7 @@ def test_vxlan_priority_multi_pri_sec_switchover(self, setUp, encap_type): primary_nhg[1], "up") time.sleep(10) - self.dump_self_info_and_run_ptf("test2", encap_type, True) + self.dump_self_info_and_run_ptf("test2", encap_type, True, skip_traffic_test=skip_traffic_test) ecmp_utils.create_and_apply_priority_config( self.vxlan_test_setup['duthost'], vnet, diff --git a/tests/wol/conftest.py b/tests/wol/conftest.py new file mode 100644 index 0000000000..577920514b --- /dev/null +++ b/tests/wol/conftest.py @@ -0,0 +1,13 @@ +import pytest + + +@pytest.fixture(scope="module") +def get_connected_dut_intf_to_ptf_index(duthost, tbinfo): + disabled_host_interfaces = tbinfo['topo']['properties']['topology'].get('disabled_host_interfaces', []) + connected_ptf_ports_idx = [interface for interface in + tbinfo['topo']['properties']['topology'].get('host_interfaces', []) + if interface not in disabled_host_interfaces] + dut_intf_to_ptf_index = duthost.get_extended_minigraph_facts(tbinfo)['minigraph_ptf_indices'] + connected_dut_intf_to_ptf_index = [(k, v) for k, v in dut_intf_to_ptf_index.items() if v in connected_ptf_ports_idx] + + yield connected_dut_intf_to_ptf_index diff --git a/tests/wol/test_wol.py b/tests/wol/test_wol.py new file mode 100644 index 0000000000..5c06028032 --- /dev/null +++ b/tests/wol/test_wol.py @@ -0,0 +1,549 @@ +import binascii +import logging +import pytest +import random +import tempfile +import time +from socket import inet_aton +from scapy.all import sniff as scapy_sniff +from tests.common.utilities import capture_and_check_packet_on_dut +from tests.common.helpers.assertions import pytest_assert + +pytestmark = [ + pytest.mark.topology('mx'), +] + +WOL_SLL_PKT_FILTER = 'ether[14:2]==0x0842' +WOL_ETHER_PKT_FILTER = 'ether[12:2]==0x0842' +BROADCAST_MAC = 'ff:ff:ff:ff:ff:ff' +ETHER_TYPE_WOL_BIN = b'\x08\x42' +ETHER_TYPE_WOL_DEC = int('842', 16) +PACKET_TYPE_BROADCAST = 1 +PACKET_TYPE_UNICAST = 3 +LINK_LAYER_TYPE_ETHER = 1 +VLAN_MEMBER_CHANGE_ERR = r'.*Failed to get port by bridge port ID .*' + + +def generate_pcap_file_path(id: str) -> str: + return '/tmp/wol_test_%s.pcap' % id + + +def vlan_n2i(vlan_name): + """ + Convert vlan name to vlan id + """ + return vlan_name.replace("Vlan", "") + + +def p2b(password: str) -> bytes: + """ + convert password to bytes + """ + if not password: + return b'' + if ':' in password: + return binascii.unhexlify(password.replace(':', '')) + if '.' in password: + return inet_aton(password) + pytest.fail("invalid password %s" % password) + + +def m2b(mac: str) -> bytes: + """ + convert mac address to bytes + """ + return binascii.unhexlify(mac.replace(':', '')) + + +def build_magic_packet(src_mac: str, target_mac: str, broadcast: bool, password: str = "") -> bytes: + dst_mac = BROADCAST_MAC if broadcast else target_mac + return m2b(dst_mac) + m2b(src_mac) + ETHER_TYPE_WOL_BIN \ + + build_magic_packet_payload(target_mac, password) + + +def build_magic_packet_payload(target_mac: str, password: str = "") -> bytes: + return b'\xff' * 6 + m2b(target_mac) * 16 + p2b(password) + + +def test_send_to_single_specific_interface( + duthost, + ptfhost, + get_connected_dut_intf_to_ptf_index +): + dut_mac = duthost.facts['router_mac'] + target_mac = "1a:2b:3c:d1:e2:f0" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + + def validate_wol_packets(pkts): + pytest_assert(len(pkts) == 1, "Unexpected pkts count %s" % len(pkts)) + pkt = pkts[0] + pytest_assert(pkt.dst == target_mac, "Unexpected dst mac %s" % pkt.dst) + pytest_assert(pkt.src == dut_mac, "Unexpected src mac %s" % pkt.src) + pytest_assert(pkt.type == ETHER_TYPE_WOL_DEC) + pytest_assert(pkt.load == build_magic_packet_payload(target_mac)) + + with capture_and_check_packet_on_dut( + duthost=ptfhost, + interface='eth'+str(random_ptf_port), + pkts_filter=WOL_ETHER_PKT_FILTER, + pkts_validator=validate_wol_packets + ): + duthost.shell("wol %s %s" % (random_dut_port, target_mac)) + + +def test_send_to_vlan( + duthost, + ptfhost, + get_connected_dut_intf_to_ptf_index, + loganalyzer +): + loganalyzer[duthost.hostname].ignore_regex.append(VLAN_MEMBER_CHANGE_ERR) + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + dut_ptf_int_map = dict(get_connected_dut_intf_to_ptf_index) + connected_dut_intf = [dut_intf for dut_intf, _ in connected_dut_intf_to_ptf_index] + dut_mac = duthost.facts['router_mac'] + target_mac = "1a:2b:3c:d1:e2:f1" + vlan_brief = duthost.get_vlan_brief() + vlan_names = list(vlan_brief.keys()) + random_vlan = random.choice(vlan_names) + vlan_members = vlan_brief[random_vlan]['members'] + connected_vlan_members = [member for member in vlan_members if member in connected_dut_intf] + random_member_to_remove = random.choice(connected_vlan_members) + random_vlan_members = [member for member in connected_vlan_members if member != random_member_to_remove] + logging.info("Test with random vlan %s, members %s and member to remove %s" + % (random_vlan, random_vlan_members, random_member_to_remove)) + + duthost.del_member_from_vlan(vlan_n2i(random_vlan), random_member_to_remove) + + try: + tcpdump_cmd = 'nohup tcpdump -i %s -w %s %s >/dev/null 2>&1 & echo $!' % \ + (random_member_to_remove, generate_pcap_file_path(random_member_to_remove), WOL_ETHER_PKT_FILTER) + tcpdump_pid = ptfhost.shell(tcpdump_cmd)["stdout"] + for member in random_vlan_members + [random_member_to_remove]: + ptf_int = 'eth' + str(dut_ptf_int_map[member]) + tcpdump_cmd = 'nohup tcpdump -i %s -w %s %s >/dev/null 2>&1 & echo $!' % \ + (ptf_int, generate_pcap_file_path(ptf_int), WOL_ETHER_PKT_FILTER) + tcpdump_pid = ptfhost.shell(tcpdump_cmd)["stdout"] + cmd_check_if_process_running = "ps -p %s | grep %s |grep -v grep | wc -l" % (tcpdump_pid, tcpdump_pid) + success = ptfhost.shell(cmd_check_if_process_running)["stdout"] == "1" + if not success: + ptfhost.shell('killall tcpdump', module_ignore_errors=True) + pytest.fail("Failed to start tcpdump on %s" % member) + + def validate_wol_packets(pkts): + pytest_assert(len(pkts) == 1, "Unexpected pkts count %s" % len(pkts)) + pkt = pkts[0] + pytest_assert(pkt.dst == target_mac, "Unexpected dst mac %s" % pkt.dst) + pytest_assert(pkt.src == dut_mac, "Unexpected src mac %s" % pkt.src) + pytest_assert(pkt.type == ETHER_TYPE_WOL_DEC) + pytest_assert(pkt.load == build_magic_packet_payload(target_mac)) + + duthost.shell("wol %s %s" % (random_vlan, target_mac)) + + time.sleep(1) + ptfhost.shell('killall tcpdump') + time.sleep(1) + + ptf_int = 'eth' + str(dut_ptf_int_map[random_member_to_remove]) + with tempfile.NamedTemporaryFile() as temp_pcap: + ptfhost.fetch(src=generate_pcap_file_path(ptf_int), dest=temp_pcap.name, flat=True) + pytest_assert(len(scapy_sniff(offline=temp_pcap.name)) == 0) + + for member in random_vlan_members: + ptf_int = 'eth' + str(dut_ptf_int_map[member]) + with tempfile.NamedTemporaryFile() as temp_pcap: + ptfhost.fetch(src=generate_pcap_file_path(ptf_int), dest=temp_pcap.name, flat=True) + validate_wol_packets(scapy_sniff(offline=temp_pcap.name)) + + finally: + duthost.add_member_to_vlan(vlan_n2i(random_vlan), random_member_to_remove, False) + + +def test_send_broadcast_to_single_interface( + duthost, + ptfhost, + get_connected_dut_intf_to_ptf_index +): + dut_mac = duthost.facts['router_mac'] + target_mac = "1a:2b:3c:d1:e2:f2" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + + def validate_wol_packets(pkts): + pytest_assert(len(pkts) == 1, "Unexpected pkts count %s" % len(pkts)) + pkt = pkts[0] + pytest_assert(pkt.dst == BROADCAST_MAC, "Unexpected dst mac %s" % pkt.dst) + pytest_assert(pkt.src == dut_mac, "Unexpected src mac %s" % pkt.src) + pytest_assert(pkt.type == ETHER_TYPE_WOL_DEC) + pytest_assert(pkt.load == build_magic_packet_payload(target_mac)) + + with capture_and_check_packet_on_dut( + duthost=ptfhost, + interface='eth'+str(random_ptf_port), + pkts_filter=WOL_ETHER_PKT_FILTER, + pkts_validator=validate_wol_packets + ): + duthost.shell("wol %s %s -b" % (random_dut_port, target_mac)) + + +@pytest.mark.parametrize("password", ["11:22:33:44:55:66", "192.168.0.1"]) +def test_send_with_password( + duthost, + ptfhost, + get_connected_dut_intf_to_ptf_index, + password +): + dut_mac = duthost.facts['router_mac'] + target_mac = "1a:2b:3c:d1:e2:f3" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + + def validate_wol_packets(pkts): + pytest_assert(len(pkts) == 1, "Unexpected pkts count %s" % len(pkts)) + pkt = pkts[0] + pytest_assert(pkt.dst == target_mac, "Unexpected dst mac %s" % pkt.dst) + pytest_assert(pkt.src == dut_mac, "Unexpected src mac %s" % pkt.src) + pytest_assert(pkt.type == ETHER_TYPE_WOL_DEC) + pytest_assert(pkt.load == build_magic_packet_payload(target_mac, password)) + + with capture_and_check_packet_on_dut( + duthost=ptfhost, + interface='eth'+str(random_ptf_port), + pkts_filter=WOL_ETHER_PKT_FILTER, + pkts_validator=validate_wol_packets + ): + duthost.shell("wol %s %s -p %s" % (random_dut_port, target_mac, password)) + + +@pytest.mark.parametrize("interval", [0, 2000]) +@pytest.mark.parametrize("count", [2, 5]) +def test_single_interface_with_count_and_interval( + duthost, + ptfhost, + get_connected_dut_intf_to_ptf_index, + interval, + count +): + dut_mac = duthost.facts['router_mac'] + target_mac = "1a:2b:3c:d1:e2:f4" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + + def validate_wol_packets(pkts): + pytest_assert(len(pkts) == count, "Unexpected pkts count %s" % len(pkts)) + last_time = None + for pkt in pkts: + pytest_assert(pkt.dst == target_mac, "Unexpected dst mac %s" % pkt.dst) + pytest_assert(pkt.src == dut_mac, "Unexpected src mac %s" % pkt.src) + pytest_assert(pkt.type == ETHER_TYPE_WOL_DEC) + pytest_assert(pkt.load == build_magic_packet_payload(target_mac)) + if last_time: + millseconds_gap = (pkt.time - last_time) * 1000 + pytest_assert(millseconds_gap > interval - 5 and millseconds_gap < interval + 5, + "Unexpected interval %s" % (millseconds_gap)) + + with capture_and_check_packet_on_dut( + duthost=ptfhost, + interface='eth'+str(random_ptf_port), + pkts_filter=WOL_ETHER_PKT_FILTER, + pkts_validator=validate_wol_packets + ): + duthost.shell("wol %s %s -i %s -c %s" % (random_dut_port, target_mac, interval, count)) + + +@pytest.mark.parametrize("interval", [0, 2000]) +@pytest.mark.parametrize("count", [2, 5]) +def test_send_to_vlan_with_count_and_interval( + duthost, + ptfhost, + get_connected_dut_intf_to_ptf_index, + loganalyzer, + interval, + count +): + loganalyzer[duthost.hostname].ignore_regex.append(VLAN_MEMBER_CHANGE_ERR) + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + dut_ptf_int_map = dict(get_connected_dut_intf_to_ptf_index) + connected_dut_intf = [dut_intf for dut_intf, _ in connected_dut_intf_to_ptf_index] + dut_mac = duthost.facts['router_mac'] + target_mac = "1a:2b:3c:d1:e2:f5" + vlan_brief = duthost.get_vlan_brief() + vlan_names = list(vlan_brief.keys()) + random_vlan = random.choice(vlan_names) + vlan_members = vlan_brief[random_vlan]['members'] + connected_vlan_members = [member for member in vlan_members if member in connected_dut_intf] + random_member_to_remove = random.choice(connected_vlan_members) + random_vlan_members = [member for member in connected_vlan_members if member != random_member_to_remove] + logging.info("Test with random vlan %s, members %s and member to remove %s" + % (random_vlan, random_vlan_members, random_member_to_remove)) + + duthost.del_member_from_vlan(vlan_n2i(random_vlan), random_member_to_remove) + + try: + tcpdump_cmd = 'nohup tcpdump -i %s -w %s %s >/dev/null 2>&1 & echo $!' % \ + (random_member_to_remove, generate_pcap_file_path(random_member_to_remove), WOL_ETHER_PKT_FILTER) + tcpdump_pid = ptfhost.shell(tcpdump_cmd)["stdout"] + for member in random_vlan_members + [random_member_to_remove]: + ptf_int = 'eth' + str(dut_ptf_int_map[member]) + tcpdump_cmd = 'nohup tcpdump -i %s -w %s %s >/dev/null 2>&1 & echo $!' % \ + (ptf_int, generate_pcap_file_path(ptf_int), WOL_ETHER_PKT_FILTER) + tcpdump_pid = ptfhost.shell(tcpdump_cmd)["stdout"] + cmd_check_if_process_running = "ps -p %s | grep %s |grep -v grep | wc -l" % (tcpdump_pid, tcpdump_pid) + success = ptfhost.shell(cmd_check_if_process_running)["stdout"] == "1" + if not success: + ptfhost.shell('killall tcpdump', module_ignore_errors=True) + pytest.fail("Failed to start tcpdump on %s" % member) + + def validate_wol_packets(pkts): + pytest_assert(len(pkts) == count, "Unexpected pkts count %s" % len(pkts)) + last_time = None + for pkt in pkts: + pytest_assert(pkt.dst == target_mac, "Unexpected dst mac %s" % pkt.dst) + pytest_assert(pkt.src == dut_mac, "Unexpected src mac %s" % pkt.src) + pytest_assert(pkt.type == ETHER_TYPE_WOL_DEC) + pytest_assert(pkt.load == build_magic_packet_payload(target_mac)) + if last_time: + millseconds_gap = (pkt.time - last_time) * 1000 + pytest_assert(millseconds_gap > interval - 5 and millseconds_gap < interval + 5, + "Unexpected interval %s" % (millseconds_gap)) + + duthost.shell("wol %s %s -i %s -c %s" % (random_vlan, target_mac, interval, count)) + + time.sleep(1) + ptfhost.shell('killall tcpdump') + time.sleep(1) + + ptf_int = 'eth' + str(dut_ptf_int_map[random_member_to_remove]) + with tempfile.NamedTemporaryFile() as temp_pcap: + ptfhost.fetch(src=generate_pcap_file_path(ptf_int), dest=temp_pcap.name, flat=True) + pytest_assert(len(scapy_sniff(offline=temp_pcap.name)) == 0) + + for member in random_vlan_members: + ptf_int = 'eth' + str(dut_ptf_int_map[member]) + with tempfile.NamedTemporaryFile() as temp_pcap: + ptfhost.fetch(src=generate_pcap_file_path(ptf_int), dest=temp_pcap.name, flat=True) + validate_wol_packets(scapy_sniff(offline=temp_pcap.name)) + + finally: + duthost.add_member_to_vlan(vlan_n2i(random_vlan), random_member_to_remove, False) + + +def test_unicast_port( + duthost, + ptfhost, + get_connected_dut_intf_to_ptf_index +): + target_mac = "1a:2b:3c:d1:e2:f6" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + + def validate_wol_packets(pkts): + pytest_assert(len(pkts) == 1, "Unexpected pkts count %s" % len(pkts)) + pkt = pkts[0] + pytest_assert(pkt.lladdrtype == LINK_LAYER_TYPE_ETHER, "Unexpected link layer type %s" % pkt.lladdrtype) + pytest_assert(pkt.pkttype == PACKET_TYPE_UNICAST, "Unexpected packet type %s" % pkt.pkttype) + pytest_assert(pkt.proto == ETHER_TYPE_WOL_DEC) + pytest_assert(pkt.load == build_magic_packet_payload(target_mac)) + + with capture_and_check_packet_on_dut( + duthost=ptfhost, + interface='any', + pkts_filter=WOL_SLL_PKT_FILTER, + pkts_validator=validate_wol_packets + ): + duthost.shell("wol %s %s" % (random_dut_port, target_mac)) + + +def test_broadcast_port( + duthost, + ptfhost, + get_connected_dut_intf_to_ptf_index +): + target_mac = "1a:2b:3c:d1:e2:f7" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + + def validate_wol_packets(pkts): + pytest_assert(len(pkts) == 1, "Unexpected pkts count %s" % len(pkts)) + pkt = pkts[0] + pytest_assert(pkt.lladdrtype == LINK_LAYER_TYPE_ETHER, "Unexpected link layer type %s" % pkt.lladdrtype) + pytest_assert(pkt.pkttype == PACKET_TYPE_BROADCAST, "Unexpected packet type %s" % pkt.pkttype) + pytest_assert(pkt.proto == ETHER_TYPE_WOL_DEC) + pytest_assert(pkt.load == build_magic_packet_payload(target_mac)) + + with capture_and_check_packet_on_dut( + duthost=ptfhost, + interface='any', + pkts_filter=WOL_SLL_PKT_FILTER, + pkts_validator=validate_wol_packets + ): + duthost.shell("wol %s %s -b" % (random_dut_port, target_mac)) + + +@pytest.mark.parametrize("password", ["192.168.0.256", "q1:11:22:33:44:55"]) +def test_invalid_password( + duthost, + get_connected_dut_intf_to_ptf_index, + password +): + target_mac = "1a:2b:3c:d1:e2:f7" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + exception_catched = False + try: + duthost.shell("wol %s %s -b -p %s" % (random_dut_port, target_mac, password)) + except Exception as e: + exception_catched = True + pytest_assert("invalid password" in e.results['stderr'], "Unexpected exception %s" % str(e)) + pytest_assert(exception_catched, "No exception catched") + + +def test_invalid_mac( + duthost, + get_connected_dut_intf_to_ptf_index +): + invalid_mac = "1a:2b:3c:d1:e2:fq" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + exception_catched = False + try: + duthost.shell("wol %s %s -b" % (random_dut_port, invalid_mac)) + except Exception as e: + exception_catched = True + pytest_assert(r'Invalid value for "TARGET_MAC": invalid MAC address 1a:2b:3c:d1:e2:fq' in e.results['stderr'] + or r'Invalid MAC address' in e.results['stderr'], + "Unexpected exception %s" % str(e)) + pytest_assert(exception_catched, "No exception catched") + + +def test_invalid_interface( + duthost +): + target_mac = "1a:2b:3c:d1:e2:f8" + invalid_interface = "Ethernet999" + exception_catched = False + try: + duthost.shell("wol %s %s -b" % (invalid_interface, target_mac)) + except Exception as e: + exception_catched = True + pytest_assert(r'invalid SONiC interface name Ethernet999' in e.results['stderr'], + "Unexpected exception %s" % str(e)) + pytest_assert(exception_catched, "No exception catched") + + +def test_down_interface( + duthost, + get_connected_dut_intf_to_ptf_index +): + target_mac = "1a:2b:3c:d1:e2:f9" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + + duthost.shutdown(random_dut_port) + + exception_catched = False + try: + duthost.shell("wol %s %s -b" % (random_dut_port, target_mac)) + except Exception as e: + exception_catched = True + pytest_assert("interface %s is not up" % random_dut_port in e.results['stderr'], + "Unexpected exception %s" % str(e)) + pytest_assert(e.results['rc'] == 2, "Unexpected exception %s" % str(e)) + finally: + duthost.no_shutdown(random_dut_port) + pytest_assert(exception_catched, "No exception catched") + + +def test_invalid_interval( + duthost, + get_connected_dut_intf_to_ptf_index +): + target_mac = "1a:2b:3c:d1:e2:fa" + invalid_interval = "2001" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + exception_catched = False + try: + duthost.shell("wol %s %s -b -i %s" % (random_dut_port, target_mac, invalid_interval)) + except Exception as e: + exception_catched = True + pytest_assert(r'Invalid value for "-i": 2001 is not in the valid range of 0 to 2000.' in e.results['stderr'] + or r'Invalid value for "INTERVAL": interval must between 0 and 2000' in e.results['stderr'], + "Unexpected exception %s" % str(e)) + pytest_assert(exception_catched, "No exception catched") + + +def test_invalid_count( + duthost, + get_connected_dut_intf_to_ptf_index +): + target_mac = "1a:2b:3c:d1:e2:fb" + invalid_count = "10" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + exception_catched = False + try: + duthost.shell("wol %s %s -b -c %s" % (random_dut_port, target_mac, invalid_count)) + except Exception as e: + exception_catched = True + pytest_assert(r'Invalid value for "-c": 10 is not in the valid range of 1 to 5.' in e.results['stderr'] or + r'Invalid value for "COUNT": count must between 1 and 5' in e.results['stderr'], + "Unexpected exception %s" % str(e)) + pytest_assert(exception_catched, "No exception catched") + + +def test_parameter_constrain_of_count_and_interval( + duthost, + get_connected_dut_intf_to_ptf_index +): + target_mac = "1a:2b:3c:d1:e2:ee" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + exception_catched = False + try: + duthost.shell("wol %s %s -c 2" % (random_dut_port, target_mac)) + except Exception as e: + exception_catched = True + pytest_assert("count and interval must be used together" in e.results['stderr'] + or "required arguments were not provided", "Unexpected exception %s" % str(e)) + pytest_assert(exception_catched, "No exception catched") + + exception_catched = False + try: + duthost.shell("wol %s %s -i 1000" % (random_dut_port, target_mac)) + except Exception as e: + exception_catched = True + pytest_assert("count and interval must be used together" in e.results['stderr'] + or "required arguments were not provided", "Unexpected exception %s" % str(e)) + pytest_assert(exception_catched, "No exception catched") + + +def test_rc_2_invalid_parameter( + duthost, + get_connected_dut_intf_to_ptf_index +): + target_mac = "1a:2b:3c:d1:e2:fb" + invalid_count = "10" + connected_dut_intf_to_ptf_index = get_connected_dut_intf_to_ptf_index + random_dut_port, random_ptf_port = random.choice(connected_dut_intf_to_ptf_index) + logging.info("Test with random dut port %s and ptf port index %s" % (random_dut_port, random_ptf_port)) + exception_catched = False + try: + duthost.shell("wol %s %s -b -c %s" % (random_dut_port, target_mac, invalid_count)) + except Exception as e: + exception_catched = True + pytest_assert(e.results['rc'] == 2, "Unexpected exception %s" % str(e)) + pytest_assert(exception_catched, "No exception catched")