From 8320dc1d1f5cafd4754d57c9d76f3dc5493f0c3b Mon Sep 17 00:00:00 2001 From: Elliot West Date: Tue, 24 Dec 2024 09:00:59 +0000 Subject: [PATCH] Update based on integration Change-Id: I3fdacf9356b977d56161711661b453c329105c58 --- .../config/engine/engine-options.yml | 106 +++++++ .../engine/executor-statefulset-template.yml | 151 ++++++++++ .../crds/dremioengines.dremio.com-v1.yml | 263 ++++++++++-------- .../crds/dremioreplicas.dremio.com-v1.yml | 185 ++++++------ .../templates/dremio-engine-operator.yaml | 16 ++ charts/dremio_v2/values.yaml | 2 + 6 files changed, 518 insertions(+), 205 deletions(-) create mode 100644 charts/dremio_v2/config/engine/engine-options.yml create mode 100644 charts/dremio_v2/config/engine/executor-statefulset-template.yml diff --git a/charts/dremio_v2/config/engine/engine-options.yml b/charts/dremio_v2/config/engine/engine-options.yml new file mode 100644 index 0000000..7241907 --- /dev/null +++ b/charts/dremio_v2/config/engine/engine-options.yml @@ -0,0 +1,106 @@ +# +# Copyright (C) 2017-2019 Dremio Corporation. This file is confidential and private property. +# + +# This file uses kubernetes pod nomenclature: pods, cpu, memory. Values conform to the Kubernetes +# Quantity type: +# https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes +engineOptions: + # + # SIZES + # The number of pods to create for each engine replica, and the total memory to allocate to each + # executor pod in that replica. Note that the memory allocated to each executor container, may + # be lower depending on the podReservations setting. + sizes: + - name: 2XSmall + pods: 1 + memory: 64Gi + - name: XSmall + pods: 1 + memory: 128Gi + - name: Small + pods: 2 + memory: 128Gi + # + # TARGET CPU CAPACITIES + # The total number to CPUs to allocate to each executor pod. Note that the number of CPUs + # allocated to the executor container may be lower depending on the resourceAllocationOffsets setting. + targetCpuCapacities: + - name: 16C + cpu: 16 + - name: 32C + cpu: 32 + # + # RESOURCE ALLOCATION OFFSETS + # The amount of CPU/Memory to reserve on each pod for containers other than that of the executor. + # Use action "reserve" to reserve pod resources for other containers (substractive), or "over-commit" + # to over-commit pod resources to the executor container (additive). Keys are arbitrary, used only to + # identify a default offset value. + resourceAllocationOffsets: + offsets: + - name: reserve-0-0 + cpu: 0 + memory: 0Gi + action: reserve + - name: reserve-2-4 + cpu: 2 + memory: 4Gi + action: reserve + - name: reserve-2-8 + cpu: 2 + memory: 8Gi + action: reserve + - name: reserve-2-16 + cpu: 2 + memory: 16Gi + action: reserve + defaultOffset: reserve-2-4 + # + # EXECUTOR POD STORAGE OPTIONS + storage: + spillStorageSizes: + - name: 100GB + storage: 100Gi + - name: 250GB + storage: 250Gi + - name: 500GB + storage: 500Gi + defaultSpillStorageSize: 100GB + c3StorageSizes: + - name: 100GB + storage: 100Gi + - name: 250GB + storage: 250Gi + - name: 500GB + storage: 500Gi + defaultC3StorageSize: 100GB + # + # IDLE TIMEOUTS + # Periods are expressed in the ISO8601 duration format: https://en.wikipedia.org/wiki/ISO_8601#Durations. + idleTimeouts: + durations: + - PT2H + - PT1H30M + - PT1H + - PT30M + - PT15M + - PT10M + - PT5M + defaultDuration: PT2H + # + # JAVA OPTS + # Safe and curated Java options that can be applied to executor JVMs. + javaOpts: + - name: "UseGCLogFileRotation" + pattern: "-XX:+UseGCLogFileRotation" + applyByDefault: true + - name: "NumberOfGCLogFiles" + pattern: "-XX:NumberOfGCLogFiles=%s" + defaultValue: "5" + valueMatcher: "^[1-9][0-9]*$" + applyByDefault: true + - name: "GCLogFileSize" + pattern: "-XX:GCLogFileSize=%s" + defaultValue: "4000k" + valueMatcher: "^[1-9][0-9]*[kKmMgG]$" + applyByDefault: true diff --git a/charts/dremio_v2/config/engine/executor-statefulset-template.yml b/charts/dremio_v2/config/engine/executor-statefulset-template.yml new file mode 100644 index 0000000..e5e1973 --- /dev/null +++ b/charts/dremio_v2/config/engine/executor-statefulset-template.yml @@ -0,0 +1,151 @@ +# +# Copyright (C) 2017-2019 Dremio Corporation. This file is confidential and private property. +# +# +# NOTE: This is NOT a concrete StatefulSet. It is a ConfigMap that contains a base StatefulSet that is used as a +# template for replica executors. +# +# Based on: https://github.com/dremio/dremio-cloud-tools/blob/master/charts/dremio_v2/templates/dremio-executor.yaml +# +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: template-dremio-executor +spec: + serviceName: "dremio-cluster-pod" + replicas: 0 + podManagementPolicy: "Parallel" + revisionHistoryLimit: 1 + selector: + matchLabels: + app: dremio-executor + template: + metadata: + labels: + role: dremio-cluster-pod + diagnostics-collector-role: dremio-executor + annotations: + dremio-configmap/checksum: 0 + spec: + terminationGracePeriodSeconds: 720 + securityContext: + fsGroup: 999 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: dremio-executor + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: false + runAsGroup: 999 + runAsNonRoot: true + runAsUser: 999 + seccompProfile: + type: RuntimeDefault + image: dremio/dremio-oss:latqest + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 0 + memory: "0Gi" + volumeMounts: + - name: dremio-config + mountPath: /opt/dremio/conf + - name: dremio-hive2-config + mountPath: /opt/dremio/plugins/connectors/hive2.d + - name: dremio-hive2-config + mountPath: /opt/dremio/plugins/connectors/hive2-ee.d + - name: dremio-hive3-config + mountPath: /opt/dremio/plugins/connectors/hive3.d + - name: dremio-hive3-config + mountPath: /opt/dremio/plugins/connectors/hive3-ee.d + - name: dremio-default-executor-volume + mountPath: /opt/dremio/data + - name: dremio-default-executor-c3-0 + mountPath: /opt/dremio/cloudcache/c0 + env: + - name: DREMIO_MAX_HEAP_MEMORY_SIZE_MB + value: "0" + - name: DREMIO_MAX_DIRECT_MEMORY_SIZE_MB + value: "0" + - name: DREMIO_JAVA_SERVER_EXTRA_OPTS + value: >- + -XX:+UseG1GC + -XX:+AlwaysPreTouch + -Xms8g + -Xmx8g + -XX:HeapDumpPath=/opt/dremio/data + -XX:ErrorFile=/opt/dremio/data/hs_err_pid%p.log + -XX:MaxGCPauseMillis=500 + -XX:InitiatingHeapOccupancyPercent=25 + -XX:G1HeapRegionSize=32M + -XX:+PrintGCDetails + -XX:+PrintGCTimeStamps + -XX:+PrintGCDateStamps + -XX:+PrintAdaptiveSizePolicy + -XX:+PrintClassHistogramBeforeFullGC + -XX:+PrintClassHistogramAfterFullGC + -XX:+PrintReferenceGC + -Dzookeeper=zk-hs:2181 + -Dservices.coordinator.enabled=false + -Dservices.coordinator.master.enabled=false + -Dservices.coordinator.master.embedded-zookeeper.enabled=false + -Dservices.executor.enabled=true + -Dservices.conduit.port=45679 + - name: AWS_CREDENTIAL_PROFILES_FILE + value: "/opt/dremio/aws/credentials" + - name: AWS_SHARED_CREDENTIALS_FILE + value: "/opt/dremio/aws/credentials" + - name: DREMIO_LOG_TO_CONSOLE + value: "1" + command: [ "/opt/dremio/bin/dremio" ] + args: [ "start-fg" ] + ports: + - containerPort: 45678 + name: server-fabric + - containerPort: 45679 + name: server-conduit + initContainers: + - name: wait-for-zookeeper + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: false + runAsGroup: 999 + runAsNonRoot: true + runAsUser: 999 + seccompProfile: + type: RuntimeDefault + image: busybox + command: [ "sh", "-c", "until nc zk-hs 2181 -w1 > /dev/null; do echo Waiting for Zookeeper to be ready.; sleep 2; done;" ] + volumes: + - name: dremio-config + configMap: + name: dremio-config + - name: dremio-hive2-config + configMap: + name: dremio-hive2-config + - name: dremio-hive3-config + configMap: + name: dremio-hive3-config + volumeClaimTemplates: + - metadata: + name: dremio-default-executor-volume + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 128Gi + - metadata: + name: dremio-default-executor-c3-0 + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 100Gi \ No newline at end of file diff --git a/charts/dremio_v2/crds/dremioengines.dremio.com-v1.yml b/charts/dremio_v2/crds/dremioengines.dremio.com-v1.yml index a2746ae..73a322a 100644 --- a/charts/dremio_v2/crds/dremioengines.dremio.com-v1.yml +++ b/charts/dremio_v2/crds/dremioengines.dremio.com-v1.yml @@ -10,119 +10,150 @@ spec: singular: dremioengine scope: Namespaced versions: - - additionalPrinterColumns: - - jsonPath: .spec.imageName - name: IMAGE_NAME - priority: 3 - type: string - - jsonPath: .spec.maxReplicas - name: MAX_REPLICAS - priority: 2 - type: integer - - jsonPath: .spec.minReplicas - name: MIN_REPLICAS - priority: 2 - type: integer - - jsonPath: .spec.replicas - name: REPLICAS - priority: 1 - type: integer - - jsonPath: .spec.size - name: SIZE - priority: 1 - type: string - - jsonPath: .status.runningExecutors - name: RUNNING_EXECUTORS - priority: 2 - type: integer - - jsonPath: .status.runningReplicas - name: RUNNING_REPLICAS - priority: 2 - type: integer - - jsonPath: .status.state - name: STATE - priority: 1 - type: string - name: v1 - schema: - openAPIV3Schema: - properties: - spec: - properties: - imageName: - description: Executor Image name - type: string - maxReplicas: - description: TODO - maximum: 128.0 - minimum: 0.0 - type: integer - minReplicas: - description: TODO - maximum: 128.0 - minimum: 0.0 - type: integer - replicas: - description: Count of replicas comprising this engine - maximum: 128.0 - minimum: 0.0 - type: integer - size: - description: "Then engine size, typically a T-shirt style name" - enum: - - LARGE_V1 - - MEDIUM_V1 - - SMALL_V1 - - XXX_LARGE_V1 - - XX_LARGE_V1 - - XX_SMALL_V1 - - X_LARGE_V1 - - X_SMALL_V1 - type: string - required: - - imageName - - maxReplicas - - minReplicas - - replicas - - size - type: object - status: - properties: - labelSelector: - description: Label selector for the Replica scaling - type: string - message: - description: TODO - type: string - observedGeneration: - type: integer - reason: - description: TODO - type: string - runningExecutors: - description: The status of executors that comprise the Engine - type: integer - runningReplicas: - description: Count of replicas comprising this engine - type: integer - state: - description: dTODO - enum: - - FAILED - - RUNNING - - STARTING - type: string - required: - - message - - runningReplicas - - state - type: object - type: object - served: true - storage: true - subresources: - scale: - labelSelectorPath: .status.labelSelector - specReplicasPath: .spec.replicas - statusReplicasPath: .status.runningReplicas - status: {} + - additionalPrinterColumns: + - jsonPath: .spec.c3StorageSize + name: C3_STORAGE_SIZE + priority: 2 + type: string + - jsonPath: .spec.imageName + name: IMAGE_NAME + priority: 3 + type: string + - jsonPath: .spec.maxReplicas + name: MAX_REPLICAS + priority: 2 + type: integer + - jsonPath: .spec.minReplicas + name: MIN_REPLICAS + priority: 2 + type: integer + - jsonPath: .spec.replicas + name: REPLICAS + priority: 1 + type: integer + - jsonPath: .spec.resourceAllocationOffset + name: RESOURCE_ALLOCATION_OFFSET + priority: 1 + type: string + - jsonPath: .spec.size + name: SIZE + priority: 1 + type: string + - jsonPath: .spec.spillStorageSize + name: SPILL_STORAGE_SIZE + priority: 2 + type: string + - jsonPath: .spec.targetCpuCapacity + name: TARGET_CPU_CAPACITY + priority: 1 + type: string + - jsonPath: .status.runningExecutors + name: RUNNING_EXECUTORS + priority: 2 + type: integer + - jsonPath: .status.runningReplicas + name: RUNNING_REPLICAS + priority: 2 + type: integer + - jsonPath: .status.state + name: STATE + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + c3StorageSize: + description: C3 storage size identifier. Applies a per-executor storage + request. + type: string + imageName: + description: Executor Image name. + type: string + javaOpts: + description: Optional JVM arguments. + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + maxReplicas: + description: Maximum number of replicas that can comprise this engine. + maximum: 1.0 + minimum: 0.0 + type: integer + minReplicas: + description: Minimum number of replicas that can comprise this engine. + maximum: 1.0 + minimum: 0.0 + type: integer + replicas: + description: Count of replicas comprising this engine. + maximum: 1.0 + minimum: 0.0 + type: integer + resourceAllocationOffset: + description: Resource allocation offset identifier. Applies an executor + resource request adjustment with respect to pod resource request. + type: string + size: + description: Engine size identifier. Applies a replica executor count + and executor memory request. + type: string + spillStorageSize: + description: Spill storage size identifier. Applies a per-executor + storage request. + type: string + targetCpuCapacity: + description: Executor cpu request identifier. Applies a per-pod cpu + request. + type: string + required: + - c3StorageSize + - imageName + - maxReplicas + - minReplicas + - replicas + - resourceAllocationOffset + - size + - spillStorageSize + - targetCpuCapacity + type: object + status: + properties: + message: + description: Description of the current state of the engine + type: string + observedGeneration: + type: integer + reason: + description: Reason for the current state of the engine + type: string + runningExecutors: + description: The status of executors that comprise the Engine. + type: integer + runningReplicas: + description: Count of replicas comprising this engine. + type: integer + state: + description: Current state of the engine + enum: + - FAILED + - RUNNING + - STARTING + type: string + required: + - message + - runningReplicas + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/dremio_v2/crds/dremioreplicas.dremio.com-v1.yml b/charts/dremio_v2/crds/dremioreplicas.dremio.com-v1.yml index d993a77..367d8c2 100644 --- a/charts/dremio_v2/crds/dremioreplicas.dremio.com-v1.yml +++ b/charts/dremio_v2/crds/dremioreplicas.dremio.com-v1.yml @@ -10,92 +10,99 @@ spec: singular: dremioreplica scope: Namespaced versions: - - additionalPrinterColumns: - - jsonPath: .spec.c3Storage - name: C3_STORAGE - priority: 3 - type: string - - jsonPath: .spec.cpu - name: CPU - priority: 2 - type: string - - jsonPath: .spec.executors - name: EXECUTORS - priority: 1 - type: integer - - jsonPath: .spec.executorStorage - name: EXECUTOR_STORAGE - priority: 3 - type: string - - jsonPath: .spec.imageName - name: IMAGE_NAME - priority: 2 - type: string - - jsonPath: .spec.memory - name: MEMORY - priority: 2 - type: string - - jsonPath: .status.state - name: STATE - priority: 1 - type: string - name: v1 - schema: - openAPIV3Schema: - properties: - spec: - properties: - c3Storage: - description: TODO - type: string - cpu: - description: TODO - type: string - executorStorage: - description: TODO - type: string - executors: - description: TODO - type: integer - imageName: - description: Executor Image name - type: string - memory: - description: TODO - type: string - required: - - c3Storage - - cpu - - executorStorage - - executors - - imageName - - memory - type: object - status: - properties: - message: - description: TODO - type: string - observedGeneration: - type: integer - reason: - description: TODO - type: string - runningExecutors: - type: integer - state: - description: dTODO - enum: - - FAILED - - RUNNING - - STARTING - type: string - required: - - message - - state - type: object - type: object - served: true - storage: true - subresources: - status: {} \ No newline at end of file + - additionalPrinterColumns: + - jsonPath: .spec.c3Storage + name: C3_STORAGE + priority: 3 + type: string + - jsonPath: .spec.executorCpu + name: EXECUTOR_CPU + priority: 2 + type: string + - jsonPath: .spec.executorMemory + name: EXECUTOR_MEMORY + priority: 2 + type: string + - jsonPath: .spec.executors + name: EXECUTORS + priority: 1 + type: integer + - jsonPath: .spec.imageName + name: IMAGE_NAME + priority: 2 + type: string + - jsonPath: .spec.spillStorage + name: SPILL_STORAGE + priority: 3 + type: string + - jsonPath: .status.state + name: STATE + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + c3Storage: + description: C3 storage request for each executor. A resource request + Quantity. + type: string + executorCpu: + description: Cpu request for each executor. A resource request Quantity. + type: string + executorMemory: + description: Memory request for each executor. A resource request + Quantity. + type: string + executors: + description: Number of executors per replica. + type: integer + imageName: + description: Executor image name. + type: string + javaOpts: + description: Optional JVM arguments string. + type: string + spillStorage: + description: Spill storage request for each executor. A resource request + Quantity. + type: string + required: + - c3Storage + - executorCpu + - executorMemory + - executors + - imageName + - spillStorage + type: object + status: + properties: + message: + description: Description of the current state of the replica + type: string + observedGeneration: + type: integer + reason: + description: Reason for the current state of the replica + type: string + runningExecutors: + description: Number of executors running in the running state + type: integer + state: + description: Current state of the replica + enum: + - FAILED + - RUNNING + - STARTING + type: string + required: + - message + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/dremio_v2/templates/dremio-engine-operator.yaml b/charts/dremio_v2/templates/dremio-engine-operator.yaml index c89653b..3f67092 100644 --- a/charts/dremio_v2/templates/dremio-engine-operator.yaml +++ b/charts/dremio_v2/templates/dremio-engine-operator.yaml @@ -178,4 +178,20 @@ roleRef: subjects: - kind: "ServiceAccount" name: {{ default "default" (default .Values.serviceAccount .Values.coordinator.serviceAccount)}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: executor-statefulset-template +data: + executor-statefulset-template.yml: | + {{ .Files.Get "config/engine/executor-statefulset-template.yml" | nindent 4 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: engine-options +data: + engine-options.yml: | + {{ .Files.Get "config/engine/engine-options.yml" | nindent 4 }} {{- end -}} diff --git a/charts/dremio_v2/values.yaml b/charts/dremio_v2/values.yaml index 7f83f4d..ff3e0d7 100644 --- a/charts/dremio_v2/values.yaml +++ b/charts/dremio_v2/values.yaml @@ -335,6 +335,8 @@ engine: memory: 1024 count: 1 + # NOTE: The following options relate to the engine operator pod and not the executors created by the operator. + # Kubernetes Service Account # Uncomment below to use a custom Kubernetes service account for the operator. #serviceAccount: ""