ggml baichuan2 7b 4bit

apecloud · Oct 25, 2023 · 990685d · 990685d
1 parent aec1be5
commit 990685d
Showing 1 changed file with 28 additions and 0 deletions.
diff --git a/deploy/llm/templates/clusterversion.yaml b/deploy/llm/templates/clusterversion.yaml
@@ -271,3 +271,31 @@ spec:
             volumeMounts:
               - name: models
                 mountPath: /models
+---
+apiVersion: apps.kubeblocks.io/v1alpha1
+kind: ClusterVersion
+metadata:
+  name: ggml-baichuan2-7b-4q
+  labels:
+    {{- include "llm.labels" . | nindent 4 }}
+spec:
+  clusterDefinitionRef: ggml
+  componentVersions:
+    - componentDefRef: ggml
+      versionsContext:
+        initContainers:
+          - name: download
+            image: infracreate-registry.cn-zhangjiakou.cr.aliyuncs.com/apecloud/baichuan2-7b-gguf:ggml-model-q4
+            command: ["sh", "-c", "cp /models/ggml-model-q4.gguf /models-target/"]
+            volumeMounts:
+              - name: models
+                mountPath: /models-target
+        containers:
+          - name: ggml
+            image: {{ .Values.imageDev.registry | default "docker.io" }}/{{ .Values.imageDev.repository}}:{{ default .Chart.AppVersion .Values.imageDev.tagNew }}
+            env:
+                - name: MODEL
+                  value: /models/ggml-model-q4.gguf
+            volumeMounts:
+              - name: models
+                mountPath: /models