Additions to CCNP/CNP Pipeline

jobs/competitive-test.yml

@@ -97,10 +97,10 @@
       regions: ${{ parameters.regions }}
       engine_input: ${{ parameters.engine_input }}
       credential_type: ${{ parameters.credential_type }}
-  - template: /steps/cleanup-resources.yml
-    parameters:
-      cloud: ${{ parameters.cloud }}
-      regions: ${{ parameters.regions }}
-      terraform_arguments: ${{ parameters.terraform_arguments }}
-      retry_attempt_count: ${{ parameters.retry_attempt_count }}
-      credential_type: ${{ parameters.credential_type }}
+  # - template: /steps/cleanup-resources.yml
+  #   parameters:
+  #     cloud: ${{ parameters.cloud }}
+  #     regions: ${{ parameters.regions }}
+  #     terraform_arguments: ${{ parameters.terraform_arguments }}
+  #     retry_attempt_count: ${{ parameters.retry_attempt_count }}
+  #     credential_type: ${{ parameters.credential_type }}

modules/python/clusterloader2/slo/config/ccnp_template.yaml

@@ -6,14 +6,26 @@ spec:
   endpointSelector:
     matchLabels:
       group: cnp-ccnp
+  ingress:
+  - icmps:
+    - fields:
+      - type: 8
+        family: IPv4
+      - type: 128
+        family: IPv6
   ingressDeny:
-  - fromEndpoints:
-    - matchLabels:
-        io.kubernetes.pod.namespace: default
+    - fromEntities:
+      - world
   egress:
+  - icmps:
+    - fields:
+      - type: 8
+        family: IPv4
+      - type: 128
+        family: IPv6
   - toPorts:
     - ports:
-      - port: "53"
-        protocol: UDP
+      - port: "443"
+        protocol: ANY
     toEntities:
     - cluster

modules/python/clusterloader2/slo/config/cnp_template.yaml

@@ -1,20 +1,40 @@
+{{- $randNum := RandIntRange 1 2 -}}
+{{- $randomNamespace := printf "slo-%d" $randNum -}}
+
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
-  name: {{.basename}}
-  namespace: slo-1 # slo-1 was used because that is the ns pods are deployed in & tried passing in namespace from load-config but had object mismatch error, revise in future to possibly pass in ns
+  name: {{.Name}}
+  namespace: {{.Namespace}}
 spec:
   endpointSelector:
     matchLabels:
       group: cnp-ccnp
-  ingressDeny:
+      name: {{.Name}}
+  ingress:
   - fromEndpoints:
     - matchLabels:
-        io.kubernetes.pod.namespace: default
+        k8s:io.kubernetes.pod.namespace: {{$randomNamespace}}
+  - icmps:
+    - fields:
+      - type: 8
+        family: IPv4
+      - type: 128
+        family: IPv6
   egress:
+  - icmps:
+    - fields:
+      - type: 8
+        family: IPv4
+      - type: 128
+        family: IPv6
+    toEntities:
+    - cluster
   - toPorts:
     - ports:
       - port: "443"
         protocol: TCP
-    toCIDR:
-    - 0.0.0.0/0
+      - port: "53"
+        protocol: UDP
+    toEntities:
+    - cluster

modules/python/clusterloader2/slo/config/deployment_template.yaml

@@ -11,6 +11,7 @@ metadata:
   name: {{.Name}}
   labels:
     group: {{.Group}}
+    name: {{.Name}}
 {{if .SvcName}}
     svc: {{.SvcName}}-{{.Index}}
 {{end}}

modules/python/clusterloader2/slo/config/load-config.yaml

@@ -6,20 +6,21 @@ name: load-config
 {{$CCNP_TEST := DefaultParam .CL2_CCNP_TEST false}}
 
 # Config options for test parameters
-{{$nodesPerNamespace := DefaultParam .CL2_NODES_PER_NAMESPACE 100}}
+{{$nodesPerNamespace := DefaultParam .CL2_NODES_PER_NAMESPACE 1000}}
 {{$podsPerNode := DefaultParam .CL2_PODS_PER_NODE 50}}
 {{$loadTestThroughput := DefaultParam .CL2_LOAD_TEST_THROUGHPUT 100}}
 {{$deploymentSize := DefaultParam .CL2_DEPLOYMENT_SIZE 100}}
 {{$repeats := DefaultParam .CL2_REPEATS 1}}
 {{$groupName := DefaultParam .CL2_GROUP_NAME "service-discovery"}}
 
 # TODO(jshr-w): This should eventually use >1 namespace.
-{{$namespaces := 1}}
+{{$namespaces := DefaultParam .CL2_NO_OF_NAMESPACES 1}}
 {{$nodes := DefaultParam .CL2_NODES 1000}}
 
+#set nodesPerNamespace to 100 
 {{$deploymentQPS := DivideFloat $loadTestThroughput $deploymentSize}}
 {{$operationTimeout := DefaultParam .CL2_OPERATION_TIMEOUT "15m"}}
-{{$totalPods := MultiplyInt $namespaces $nodes $podsPerNode}}
+{{$totalPods := MultiplyInt $namespaces $nodesPerNamespace $podsPerNode}}
 {{$podsPerNamespace := DivideInt $totalPods $namespaces}}
 {{$deploymentsPerNamespace := DivideInt $podsPerNamespace $deploymentSize}}
 
@@ -40,12 +41,13 @@ name: load-config
 {{$CNPS_PER_NAMESPACE := DefaultParam .CL2_CNPS_PER_NAMESPACE 0}}
 {{$CCNPS := DefaultParam .CL2_CCNPS 0}}
 {{$DUALSTACK := DefaultParam .CL2_DUALSTACK false}}
+{{$smallDeploymentsPerNamespaceCNP := DivideInt $podsPerNamespace $SMALL_GROUP_SIZE}}
 
 namespace:
   number: {{$namespaces}}
   prefix: slo
-  deleteStaleNamespaces: true
-  deleteAutomanagedNamespaces: true
+  deleteStaleNamespaces: false
+  deleteAutomanagedNamespaces: false
   enableExistingNamespaces: false
 
 tuningSets:
@@ -121,88 +123,92 @@ steps:
     {{if or $CCNP_TEST $CNP_TEST}}
         bigDeploymentSize: 0
         bigDeploymentsPerNamespace: 0
-        cnp_test: {{$CNP_TEST}}
-        ccnp_test: {{$CCNP_TEST}}
-    {{else}}
-        bigDeploymentSize: {{$BIG_GROUP_SIZE}}
-        bigDeploymentsPerNamespace: {{$bigDeploymentsPerNamespace}}
-    {{end}}
         smallDeploymentSize: {{$SMALL_GROUP_SIZE}}
-        smallDeploymentsPerNamespace: {{$smallDeploymentsPerNamespace}}
-        CpuRequest: {{$latencyPodCpu}}m
-        MemoryRequest: {{$latencyPodMemory}}M
-        Group: {{$groupName}}
-        deploymentLabel: start
-
-  - module:
-      path: /modules/reconcile-objects.yaml
-      params:
-        actionName: "restart"
-        namespaces: {{$namespaces}}
-        tuningSet: Sequence
-        operationTimeout: {{$operationTimeout}}
-    {{if or $CCNP_TEST $CNP_TEST}}
-        bigDeploymentSize: 0
-        bigDeploymentsPerNamespace: 0
+        smallDeploymentsPerNamespace: {{$smallDeploymentsPerNamespaceCNP}}
         cnp_test: {{$CNP_TEST}}
         ccnp_test: {{$CCNP_TEST}}
     {{else}}
         bigDeploymentSize: {{$BIG_GROUP_SIZE}}
         bigDeploymentsPerNamespace: {{$bigDeploymentsPerNamespace}}
-    {{end}}
         smallDeploymentSize: {{$SMALL_GROUP_SIZE}}
         smallDeploymentsPerNamespace: {{$smallDeploymentsPerNamespace}}
+    {{end}}
         CpuRequest: {{$latencyPodCpu}}m
         MemoryRequest: {{$latencyPodMemory}}M
         Group: {{$groupName}}
-        deploymentLabel: restart
+        deploymentLabel: start 
+
 
-  - module:
-      path: /modules/reconcile-objects.yaml
-      params:
-        actionName: "delete"
-        namespaces: {{$namespaces}}
-        tuningSet: DeploymentDeleteQps
-        operationTimeout: {{$operationTimeout}}
-    {{if or $CCNP_TEST $CNP_TEST}}
-        bigDeploymentSize: 0
-        bigDeploymentsPerNamespace: 0
-        cnp_test: {{$CNP_TEST}}
-        ccnp_test: {{$CCNP_TEST}}
-    {{else}}
-        bigDeploymentSize: {{$BIG_GROUP_SIZE}}
-        bigDeploymentsPerNamespace: 0
-    {{end}}
-        smallDeploymentSize: {{$SMALL_GROUP_SIZE}}
-        smallDeploymentsPerNamespace: 0
-        deploymentLabel: restart
-        Group: {{$groupName}}
-{{if $SERVICE_TEST}}
-  - module:
-      path: /modules/services.yaml
-      params:
-        actionName: "Deleting"
-        namespaces: {{$namespaces}}
-        smallServicesPerNamespace: 0
-        bigServicesPerNamespace: 0
-{{end}}
-{{if $CNP_TEST}}
-  - module:
-      path: /modules/ciliumnetworkpolicy.yaml
-      params:
-        actionName: "Deleting"
-        namespaces: {{$namespaces}}
-        cnpsPerNamespace: 0
-{{end}}
-{{if $CCNP_TEST}}
-  - module:
-      path: /modules/ciliumclusternetworkpolicy.yaml
-      params:
-        actionName: "Deleting"
-        namespaces: {{$namespaces}}
-        ccnps: 0
-{{end}}
-{{end}}
+
+#   - module:
+#       path: /modules/reconcile-objects.yaml
+#       params:
+#         actionName: "restart"
+#         namespaces: {{$namespaces}}
+#         tuningSet: Sequence
+#         operationTimeout: {{$operationTimeout}}
+#     {{if or $CCNP_TEST $CNP_TEST}}
+#         bigDeploymentSize: 0
+#         bigDeploymentsPerNamespace: 0
+#         cnp_test: {{$CNP_TEST}}
+#         ccnp_test: {{$CCNP_TEST}}
+#     {{else}}
+#         bigDeploymentSize: {{$BIG_GROUP_SIZE}}
+#         bigDeploymentsPerNamespace: {{$bigDeploymentsPerNamespace}}
+#     {{end}}
+#         smallDeploymentSize: {{$SMALL_GROUP_SIZE}}
+#         smallDeploymentsPerNamespace: {{$smallDeploymentsPerNamespace}}
+#         CpuRequest: {{$latencyPodCpu}}m
+#         MemoryRequest: {{$latencyPodMemory}}M
+#         Group: {{$groupName}}
+#         deploymentLabel: restart
+
+#   - module:
+#       path: /modules/reconcile-objects.yaml
+#       params:
+#         actionName: "delete"
+#         namespaces: {{$namespaces}}
+#         tuningSet: DeploymentDeleteQps
+#         operationTimeout: {{$operationTimeout}}
+#     {{if or $CCNP_TEST $CNP_TEST}}
+#         bigDeploymentSize: 0
+#         bigDeploymentsPerNamespace: 0
+#         cnp_test: {{$CNP_TEST}}
+#         ccnp_test: {{$CCNP_TEST}}
+#     {{else}}
+#         bigDeploymentSize: {{$BIG_GROUP_SIZE}}
+#         bigDeploymentsPerNamespace: 0
+#     {{end}}
+#         smallDeploymentSize: {{$SMALL_GROUP_SIZE}}
+#         smallDeploymentsPerNamespace: 0
+#         deploymentLabel: restart
+#         Group: {{$groupName}}
+# {{if $SERVICE_TEST}}
+#   - module:
+#       path: /modules/services.yaml
+#       params:
+#         actionName: "Deleting"
+#         namespaces: {{$namespaces}}
+#         smallServicesPerNamespace: 0
+#         bigServicesPerNamespace: 0
+# {{end}}
+# {{if $CNP_TEST}}
+#   - module:
+#       path: /modules/ciliumnetworkpolicy.yaml
+#       params:
+#         actionName: "Deleting"
+#         namespaces: {{$namespaces}}
+#         cnpsPerNamespace: 0
+# {{end}}
+# {{if $CCNP_TEST}}
+#   - module:
+#       path: /modules/ciliumclusternetworkpolicy.yaml
+#       params:
+#         actionName: "Deleting"
+#         namespaces: {{$namespaces}}
+#         ccnps: 0
+# {{end}}
+# {{end}}
 
 {{if $CILIUM_METRICS_ENABLED}}
   - module: