Add API server NetworkPolicy to support a potential deny all egress

bundle/manifests/ibm-websphere-liberty.clusterserviceversion.yaml

@@ -869,6 +869,7 @@ spec:
                 app.kubernetes.io/managed-by: olm
                 app.kubernetes.io/name: websphere-liberty-operator
                 control-plane: controller-manager
+                webspherelibertyapps.liberty.websphere.ibm.com/allow-apiserver-access: "true"
             spec:
               affinity:
                 nodeAffinity:
@@ -1033,6 +1034,13 @@ spec:
           - list
           - update
           - watch
+        - apiGroups:
+          - ""
+          resources:
+          - endpoints
+          verbs:
+          - get
+          - list
         - apiGroups:
           - ""
           resources:

config/manager/manager.yaml

@@ -21,6 +21,7 @@ spec:
     metadata:
       labels:
         control-plane: controller-manager
+        webspherelibertyapps.liberty.websphere.ibm.com/allow-apiserver-access: "true"
       annotations:
         productID: "cb1747ecb831410f88006195f024183f"
         productName: "WebSphere Liberty Operator"

config/manifests/kustomization.yaml

@@ -7,6 +7,7 @@ resources:
 - ../scorecard
 patchesStrategicMerge:
   - patches/csvAnnotations.yaml
+  - patches/apiServerAccessPatch.yaml
 patches:
 - path: serviceBindingPatch.yaml
   target:

config/manifests/patches/apiServerAccessPatch.yaml

@@ -0,0 +1,14 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: ClusterServiceVersion
+metadata:
+  name: ibm-websphere-liberty.v0.0.0
+  namespace: placeholder
+spec:
+  install:
+    spec:
+      deployments:
+        - spec:
+            template:
+              metadata:
+                labels:
+                  webspherelibertyapps.liberty.websphere.ibm.com/allow-apiserver-access: "true"

config/rbac/role.yaml

@@ -73,6 +73,13 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

controllers/ltpa_keys_sharing.go

@@ -231,7 +231,7 @@ func (r *ReconcileWebSphereLiberty) generateLTPAKeys(instance *wlv1.WebSphereLib
 				err = r.GetClient().Get(context.TODO(), types.NamespacedName{Name: generateLTPAKeysJob.Name, Namespace: generateLTPAKeysJob.Namespace}, generateLTPAKeysJob)
 				if err != nil && kerrors.IsNotFound(err) {
 					err = r.CreateOrUpdate(generateLTPAKeysJob, instance, func() error {
-						lutils.CustomizeLTPAJob(generateLTPAKeysJob, instance, ltpaSecret.Name, ltpaServiceAccountName, ltpaKeysCreationScriptConfigMap.Name)
+						lutils.CustomizeLTPAJob(generateLTPAKeysJob, instance, ltpaSecret.Name, ltpaServiceAccountName, ltpaKeysCreationScriptConfigMap.Name, OperatorAllowAPIServerAccessLabel)
 						return nil
 					})
 					if err != nil {

controllers/webspherelibertyapplication_controller.go

@@ -43,6 +43,7 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	servingv1 "knative.dev/serving/pkg/apis/serving/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -54,8 +55,9 @@ import (
 )
 
 const (
-	OperatorName      = "websphere-liberty-operator"
-	OperatorShortName = "wlo"
+	OperatorName                      = "websphere-liberty-operator"
+	OperatorShortName                 = "wlo"
+	OperatorAllowAPIServerAccessLabel = "webspherelibertyapps.liberty.websphere.ibm.com/allow-apiserver-access"
 )
 
 // ReconcileWebSphereLiberty reconciles a WebSphereLibertyApplication object
@@ -74,6 +76,7 @@ const applicationFinalizer = "finalizer.webspherelibertyapps.liberty.websphere.i
 // +kubebuilder:rbac:groups=apps,resources=deployments;statefulsets,verbs=get;list;watch;create;update;delete,namespace=websphere-liberty-operator
 // +kubebuilder:rbac:groups=apps,resources=deployments/finalizers;statefulsets,verbs=update,namespace=websphere-liberty-operator
 // +kubebuilder:rbac:groups=core,resources=services;secrets;serviceaccounts;configmaps;persistentvolumeclaims,verbs=get;list;watch;create;update;delete,namespace=websphere-liberty-operator
+// +kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list,namespace=websphere-liberty-operator
 // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;delete,namespace=websphere-liberty-operator
 // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;delete,namespace=websphere-liberty-operator
 // +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;delete,namespace=websphere-liberty-operator
@@ -399,6 +402,72 @@ func (r *ReconcileWebSphereLiberty) Reconcile(ctx context.Context, request ctrl.
 			common.StatusConditionTypeReconciled, instance)
 	}
 
+	// Kube API Server NetworkPolicy (based upon impl. by Martin Smithson)
+	apiServerNetworkPolicy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{
+		Name:      instance.Name + "-egress-dns-and-apiserver-access",
+		Namespace: instance.Namespace,
+	}}
+	err = r.CreateOrUpdate(apiServerNetworkPolicy, instance, func() error {
+		apiServerNetworkPolicy.Spec.PodSelector = metav1.LabelSelector{
+			MatchLabels: map[string]string{
+				OperatorAllowAPIServerAccessLabel: "true",
+			},
+		}
+		apiServerNetworkPolicy.Spec.Egress = make([]networkingv1.NetworkPolicyEgressRule, 0)
+
+		var dnsRule networkingv1.NetworkPolicyEgressRule
+		var usingPermissiveRule bool
+		// If allowed, add an Egress rule to access the OpenShift DNS or K8s CoreDNS. Otherwise, use a permissive cluster-wide Egress rule.
+		if r.IsOpenShift() {
+			usingPermissiveRule, dnsRule = r.getDNSEgressRule(reqLogger, "dns-default", "openshift-dns")
+		} else {
+			usingPermissiveRule, dnsRule = r.getDNSEgressRule(reqLogger, "kube-dns", "kube-system")
+		}
+		apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, dnsRule)
+
+		// If the DNS rule is a specific Egress rule also check if another Egress rule can be created for the API server.
+		// Otherwise, fallback to a permissive cluster-wide Egress rule.
+		if !usingPermissiveRule {
+			if apiServerEndpoints, err := r.getEndpoints("kubernetes", "default"); err == nil {
+				rule := networkingv1.NetworkPolicyEgressRule{}
+				// Define the port
+				port := networkingv1.NetworkPolicyPort{}
+				port.Protocol = &apiServerEndpoints.Subsets[0].Ports[0].Protocol
+				var portNumber intstr.IntOrString = intstr.FromInt((int)(apiServerEndpoints.Subsets[0].Ports[0].Port))
+				port.Port = &portNumber
+				rule.Ports = append(rule.Ports, port)
+
+				// Add the endpoint address as ipBlock entries
+				for _, endpoint := range apiServerEndpoints.Subsets {
+					for _, address := range endpoint.Addresses {
+						peer := networkingv1.NetworkPolicyPeer{}
+						ipBlock := networkingv1.IPBlock{}
+						ipBlock.CIDR = address.IP + "/32"
+
+						peer.IPBlock = &ipBlock
+						rule.To = append(rule.To, peer)
+					}
+				}
+				apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, rule)
+				reqLogger.Info("Found endpoints for kubernetes service in the default namespace")
+			} else {
+				// The operator couldn't create a rule for the K8s API server so add a permissive Egress rule
+				rule := networkingv1.NetworkPolicyEgressRule{}
+				apiServerNetworkPolicy.Spec.Egress = append(apiServerNetworkPolicy.Spec.Egress, rule)
+				reqLogger.Info("Found endpoints for kubernetes service in the default namespace")
+			}
+		}
+
+		apiServerNetworkPolicy.Labels = ba.GetLabels()
+		apiServerNetworkPolicy.Annotations = oputils.MergeMaps(apiServerNetworkPolicy.Annotations, ba.GetAnnotations())
+		apiServerNetworkPolicy.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}
+		return nil
+	})
+	if err != nil {
+		reqLogger.Error(err, "Failed to reconcile API server network policy")
+		return r.ManageError(err, common.StatusConditionTypeReconciled, instance)
+	}
+
 	networkPolicy := &networkingv1.NetworkPolicy{ObjectMeta: defaultMeta}
 	if np := instance.Spec.NetworkPolicy; np == nil || np != nil && !np.IsDisabled() {
 		err = r.CreateOrUpdate(networkPolicy, instance, func() error {
@@ -874,3 +943,40 @@ func (r *ReconcileWebSphereLiberty) deletePVC(reqLogger logr.Logger, pvcName str
 		}
 	}
 }
+
+func (r *ReconcileWebSphereLiberty) getEndpoints(serviceName string, namespace string) (*corev1.Endpoints, error) {
+	endpoints := &corev1.Endpoints{}
+	if err := r.GetClient().Get(context.TODO(), types.NamespacedName{Name: serviceName, Namespace: namespace}, endpoints); err != nil {
+		return nil, err
+	} else {
+		return endpoints, nil
+	}
+}
+
+func (r *ReconcileWebSphereLiberty) getDNSEgressRule(reqLogger logr.Logger, endpointsName string, endpointsNamespace string) (bool, networkingv1.NetworkPolicyEgressRule) {
+	dnsRule := networkingv1.NetworkPolicyEgressRule{}
+	if dnsEndpoints, err := r.getEndpoints(endpointsName, endpointsNamespace); err == nil {
+		if len(dnsEndpoints.Subsets) > 0 {
+			if endpointPort := lutils.GetEndpointPortByName(&dnsEndpoints.Subsets[0].Ports, "dns"); endpointPort != nil {
+				dnsRule.Ports = append(dnsRule.Ports, lutils.CreateNetworkPolicyPortFromEndpointPort(endpointPort))
+			}
+			if endpointPort := lutils.GetEndpointPortByName(&dnsEndpoints.Subsets[0].Ports, "dns-tcp"); endpointPort != nil {
+				dnsRule.Ports = append(dnsRule.Ports, lutils.CreateNetworkPolicyPortFromEndpointPort(endpointPort))
+			}
+		}
+		peer := networkingv1.NetworkPolicyPeer{}
+		peer.NamespaceSelector = &metav1.LabelSelector{
+			MatchLabels: map[string]string{
+				"kubernetes.io/metadata.name": endpointsNamespace,
+			},
+		}
+		dnsRule.To = append(dnsRule.To, peer)
+		reqLogger.Info("Found endpoints for " + endpointsName + " service in the " + endpointsNamespace + " namespace")
+		return false, dnsRule
+	}
+	// use permissive rule
+	// egress:
+	//   - {}
+	reqLogger.Info("Failed to retrieve endpoints for " + endpointsName + " service in the " + endpointsNamespace + "  namespace. Using more permissive rule.")
+	return true, dnsRule
+}

go.mod

@@ -28,14 +28,14 @@ require (
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-logr/zapr v1.2.3 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.20.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gnostic v0.6.9 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-containerregistry v0.13.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
@@ -48,6 +48,8 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/onsi/ginkgo/v2 v2.9.4 // indirect
+	github.com/onsi/gomega v1.27.6 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect