VPA integration POC

cmd/autoscaler-vpa/OWNERS

@@ -0,0 +1,10 @@
+# The OWNERS file is used by prow to automatically merge approved PRs.
+
+approvers:
+- autoscaling-writers
+
+reviewers:
+- autoscaling-reviewers
+
+labels:
+- area/autoscale

cmd/autoscaler-vpa/kodata/HEAD

@@ -0,0 +1 @@
+../../../.git/HEAD

cmd/autoscaler-vpa/kodata/LICENSE

@@ -0,0 +1 @@
+../../../LICENSE

cmd/autoscaler-vpa/kodata/VENDOR-LICENSE

@@ -0,0 +1 @@
+../../../third_party/VENDOR-LICENSE

cmd/autoscaler-vpa/kodata/refs

@@ -0,0 +1 @@
+../../../.git/refs

cmd/autoscaler-vpa/main.go

@@ -0,0 +1,28 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"knative.dev/serving/pkg/reconciler/autoscaling/vpa"
+
+	// This defines the shared main for injected controllers.
+	"knative.dev/pkg/injection/sharedmain"
+)
+
+func main() {
+	sharedmain.Main("vpaautoscaler", vpa.NewController)
+}

config/core/200-serviceaccount.yaml

@@ -36,6 +36,20 @@ aggregationRule:
   - matchLabels:
       serving.knative.dev/controller: "true"
 ---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: knative-serving-vpa
+  labels:
+    app.kubernetes.io/component: autoscaler-vpa
+    app.kubernetes.io/name: knative-serving
+    app.kubernetes.io/version: devel
+    serving.knative.dev/release: devel
+rules:
+- apiGroups: ["autoscaling.k8s.io"] # "" indicates the core API group
+  resources: ["verticalpodautoscalers"]
+  verbs: ["*"]
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -71,3 +85,21 @@ roleRef:
   kind: ClusterRole
   name: knative-serving-aggregated-addressable-resolver
   apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: knative-serving-controller-vpa
+  labels:
+    app.kubernetes.io/component: autoscaler-vpa
+    app.kubernetes.io/name: knative-serving
+    app.kubernetes.io/version: devel
+    serving.knative.dev/release: devel
+subjects:
+  - kind: ServiceAccount
+    name: controller
+    namespace: knative-serving
+roleRef:
+  kind: ClusterRole
+  name: knative-serving-vpa
+  apiGroup: rbac.authorization.k8s.io

config/core/300-resources/podautoscaler.yaml

@@ -49,6 +49,9 @@ spec:
         - name: ActualScale
           type: integer
           jsonPath: ".status.actualScale"
+        - name: ActualMetricPercent
+          type: integer
+          jsonPath: ".status.actualMetricPercent"
         - name: Ready
           type: string
           jsonPath: ".status.conditions[?(@.type=='Ready')].status"
@@ -105,6 +108,10 @@ spec:
                 - metricsServiceName
                 - serviceName
               properties:
+                actualMetricPercent:
+                  description: ActualMetricPercent is a ratio of the current stable and panic requests to the total capacity that the deployment has, in terms of the KPA metric. It will be used by the VPA, if enabled, to make scaling decisions.
+                  type: integer
+                  format: int32
                 actualScale:
                   description: ActualScale shows the actual number of replicas for the revision.
                   type: integer
@@ -154,6 +161,29 @@ spec:
                   description: ObservedGeneration is the 'Generation' of the Service that was last processed by the controller.
                   type: integer
                   format: int64
+                resourceRecommendations:
+                  description: ResourceRecommendations are present if the VPA is enabled and has posted a recommendation
+                  type: array
+                  items:
+                    description: ResourceRecommendation is used to capture the CPU and memory recommendation from the VPA
+                    type: object
+                    properties:
+                      containerName:
+                        type: string
+                      desiredCPU:
+                        description: CPU shows the current recommended CPU request for each pod.
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        anyOf:
+                          - type: integer
+                          - type: string
+                        x-kubernetes-int-or-string: true
+                      desiredMemory:
+                        description: Memory shows the current recommended Memory request for each pod.
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        anyOf:
+                          - type: integer
+                          - type: string
+                        x-kubernetes-int-or-string: true
                 serviceName:
                   description: ServiceName is the K8s Service name that serves the revision, scaled by this PA. The service is created and owned by the ServerlessService object owned by this PA.
                   type: string

config/core/300-resources/revision.yaml

@@ -637,6 +637,10 @@ spec:
               description: RevisionStatus communicates the observed state of the Revision (from the controller).
               type: object
               properties:
+                actualMetricPercent:
+                  description: ActualMetricPercent is a ratio of the current stable and panic requests to the total capacity that the deployment has, in terms of the KPA metric. It will be used by the VPA, if enabled, to make scaling decisions.
+                  type: integer
+                  format: int32
                 actualReplicas:
                   description: ActualReplicas reflects the amount of ready pods running this revision.
                   type: integer
@@ -708,3 +712,26 @@ spec:
                   description: ObservedGeneration is the 'Generation' of the Service that was last processed by the controller.
                   type: integer
                   format: int64
+                resourceRecommendations:
+                  description: ResourceRecommendations are present if the VPA is enabled and has posted a recommendation
+                  type: array
+                  items:
+                    description: ResourceRecommendation is used to capture the CPU and memory recommendation from the VPA
+                    type: object
+                    properties:
+                      containerName:
+                        type: string
+                      desiredCPU:
+                        description: CPU shows the current recommended CPU request for each pod.
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        anyOf:
+                          - type: integer
+                          - type: string
+                        x-kubernetes-int-or-string: true
+                      desiredMemory:
+                        description: Memory shows the current recommended Memory request for each pod.
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        anyOf:
+                          - type: integer
+                          - type: string
+                        x-kubernetes-int-or-string: true

config/vpa-autoscaling/controller.yaml

@@ -0,0 +1,116 @@
+# Copyright 2019 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: autoscaler-vpa
+  namespace: knative-serving
+  labels:
+    app.kubernetes.io/component: autoscaler-vpa
+    app.kubernetes.io/name: knative-serving
+    app.kubernetes.io/version: devel
+    serving.knative.dev/release: devel
+spec:
+  selector:
+    matchLabels:
+      app: autoscaler-vpa
+  template:
+    metadata:
+      annotations:
+        cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
+      labels:
+        app: autoscaler-vpa
+        app.kubernetes.io/component: autoscaler-vpa
+        app.kubernetes.io/name: knative-serving
+        app.kubernetes.io/version: devel
+        serving.knative.dev/release: devel
+    spec:
+      # To avoid node becoming SPOF, spread our replicas to different nodes.
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  app: autoscaler-vpa
+              topologyKey: kubernetes.io/hostname
+            weight: 100
+
+      serviceAccountName: controller
+      containers:
+      - name: autoscaler-vpa
+        # This is the Go import path for the binary that is containerized
+        # and substituted here.
+        image: ko://knative.dev/serving/cmd/autoscaler-vpa
+
+        resources:
+          requests:
+            cpu: 30m
+            memory: 40Mi
+          limits:
+            cpu: 300m
+            memory: 400Mi
+
+        env:
+        - name: SYSTEM_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: CONFIG_LOGGING_NAME
+          value: config-logging
+        - name: CONFIG_OBSERVABILITY_NAME
+          value: config-observability
+
+        # TODO(https://github.com/knative/pkg/pull/953): Remove stackdriver specific config
+        - name: METRICS_DOMAIN
+          value: knative.dev/serving
+
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          capabilities:
+            drop:
+            - all
+
+        ports:
+        - name: metrics
+          containerPort: 9090
+        - name: profiling
+          containerPort: 8008
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: autoscaler-vpa
+    app.kubernetes.io/component: autoscaler-vpa
+    app.kubernetes.io/name: knative-serving
+    app.kubernetes.io/version: devel
+    serving.knative.dev/release: devel
+  name: autoscaler-vpa
+  namespace: knative-serving
+spec:
+  ports:
+  # Define metrics and profiling for them to be accessible within service meshes.
+  - name: http-metrics
+    port: 9090
+    targetPort: 9090
+  - name: http-profiling
+    port: 8008
+    targetPort: 8008
+  selector:
+    app: autoscaler-vpa

config/vpa-autoscaling/placeholder.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2020 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package autoscaling is a placeholder that allows us to pull in config files
+// via go mod vendor.
+package autoscaling