diff --git a/charts/flyte-core/values-eks.yaml b/charts/flyte-core/values-eks.yaml index 3b7cbac830..0ab0dfc150 100644 --- a/charts/flyte-core/values-eks.yaml +++ b/charts/flyte-core/values-eks.yaml @@ -4,7 +4,10 @@ userSettings: certificateArn: dbPassword: rdsHost: - bucketName: +# These two storage buckets could be the same or you could specify different buckets if required. Both keys are required. +# Learn more https://docs.flyte.org/en/latest/concepts/data_management.html#understand-how-flyte-handles-data + bucketName: + rawDataBucketName: logGroup: redisHostUrl: redisHostKey: @@ -239,7 +242,7 @@ configmap: core: propeller: - rawoutput-prefix: "s3://{{ .Values.userSettings.bucketName }}/" + rawoutput-prefix: "s3://{{ .Values.userSettings.rawDataBucketName }}/" workers: 40 gc-interval: 12h max-workflow-retries: 50 diff --git a/charts/flyte-core/values-gcp.yaml b/charts/flyte-core/values-gcp.yaml index 4c91115c8d..b402924699 100644 --- a/charts/flyte-core/values-gcp.yaml +++ b/charts/flyte-core/values-gcp.yaml @@ -5,7 +5,10 @@ userSettings: googleProjectId: dbHost: dbPassword: - bucketName: +# These two storage buckets could be the same or you could specify different buckets if required. Both keys are required. +# Learn more https://docs.flyte.org/en/latest/concepts/data_management.html#understand-how-flyte-handles-data + bucketName: + rawDataBucketName: hostName: # @@ -268,7 +271,7 @@ configmap: core: propeller: - rawoutput-prefix: "gs://{{ .Values.userSettings.bucketName }}/" + rawoutput-prefix: "gs://{{ .Values.userSettings.rawDataBucketName }}/" workers: 40 gc-interval: 12h max-workflow-retries: 50 diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml index 4420424c78..200fe365f2 100644 --- a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml +++ b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml @@ -182,7 +182,7 @@ data: storage.yaml: | storage: type: s3 - container: "" + container: "" connection: auth-type: iam region: @@ -391,7 +391,7 @@ data: storage.yaml: | storage: type: s3 - container: "" + container: "" connection: auth-type: iam region: @@ -473,7 +473,7 @@ data: rate: 100 type: bucket type: batch - rawoutput-prefix: s3:/// + rawoutput-prefix: s3:/// workers: 40 workflow-reeval-duration: 30s webhook: @@ -504,7 +504,7 @@ data: storage.yaml: | storage: type: s3 - container: "" + container: "" connection: auth-type: iam region: @@ -1167,7 +1167,7 @@ spec: template: metadata: annotations: - configChecksum: "59ef5b555bd41c3e854a315f21031c76dfa876455ff8069b989cb6c28ec1f17" + configChecksum: "c2a15ce5dc2fa465986d6006f93450723da58166b3ad5ee35a91cb37d5c39da" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte diff --git a/deployment/eks/flyte_helm_controlplane_generated.yaml b/deployment/eks/flyte_helm_controlplane_generated.yaml index 3c558f9485..36667d78f8 100644 --- a/deployment/eks/flyte_helm_controlplane_generated.yaml +++ b/deployment/eks/flyte_helm_controlplane_generated.yaml @@ -163,7 +163,7 @@ data: storage.yaml: | storage: type: s3 - container: "" + container: "" connection: auth-type: iam region: @@ -357,7 +357,7 @@ data: storage.yaml: | storage: type: s3 - container: "" + container: "" connection: auth-type: iam region: @@ -872,7 +872,7 @@ spec: template: metadata: annotations: - configChecksum: "59ef5b555bd41c3e854a315f21031c76dfa876455ff8069b989cb6c28ec1f17" + configChecksum: "c2a15ce5dc2fa465986d6006f93450723da58166b3ad5ee35a91cb37d5c39da" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte diff --git a/deployment/eks/flyte_helm_dataplane_generated.yaml b/deployment/eks/flyte_helm_dataplane_generated.yaml index 7ca99eee27..8627896512 100644 --- a/deployment/eks/flyte_helm_dataplane_generated.yaml +++ b/deployment/eks/flyte_helm_dataplane_generated.yaml @@ -137,7 +137,7 @@ data: rate: 100 type: bucket type: batch - rawoutput-prefix: s3:/// + rawoutput-prefix: s3:/// workers: 40 workflow-reeval-duration: 30s webhook: @@ -168,7 +168,7 @@ data: storage.yaml: | storage: type: s3 - container: "" + container: "" connection: auth-type: iam region: diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml index bbfb0f32bc..4bec7d622f 100644 --- a/deployment/eks/flyte_helm_generated.yaml +++ b/deployment/eks/flyte_helm_generated.yaml @@ -194,7 +194,7 @@ data: storage.yaml: | storage: type: s3 - container: "" + container: "" connection: auth-type: iam region: @@ -388,7 +388,7 @@ data: storage.yaml: | storage: type: s3 - container: "" + container: "" connection: auth-type: iam region: @@ -504,7 +504,7 @@ data: rate: 100 type: bucket type: batch - rawoutput-prefix: s3:/// + rawoutput-prefix: s3:/// workers: 40 workflow-reeval-duration: 30s webhook: @@ -535,7 +535,7 @@ data: storage.yaml: | storage: type: s3 - container: "" + container: "" connection: auth-type: iam region: @@ -1198,7 +1198,7 @@ spec: template: metadata: annotations: - configChecksum: "59ef5b555bd41c3e854a315f21031c76dfa876455ff8069b989cb6c28ec1f17" + configChecksum: "c2a15ce5dc2fa465986d6006f93450723da58166b3ad5ee35a91cb37d5c39da" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte diff --git a/deployment/gcp/flyte_helm_controlplane_generated.yaml b/deployment/gcp/flyte_helm_controlplane_generated.yaml index cda8eb8df6..68fb7acb6b 100644 --- a/deployment/gcp/flyte_helm_controlplane_generated.yaml +++ b/deployment/gcp/flyte_helm_controlplane_generated.yaml @@ -171,7 +171,7 @@ data: json: "" project_id: scopes: https://www.googleapis.com/auth/cloud-platform - container: "" + container: "" enable-multicontainer: false limits: maxDownloadMBs: 10 @@ -373,7 +373,7 @@ data: json: "" project_id: scopes: https://www.googleapis.com/auth/cloud-platform - container: "" + container: "" enable-multicontainer: false limits: maxDownloadMBs: 10 @@ -887,7 +887,7 @@ spec: template: metadata: annotations: - configChecksum: "bc69ed841506b28a42ac19bd0884d483472b3d11fe85fe7e546b879aeb30a85" + configChecksum: "905a2a911a85dbf8d4f1dc3be24b9c4fd7bb46481db0e174274d6aea6129b4c" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte diff --git a/deployment/gcp/flyte_helm_dataplane_generated.yaml b/deployment/gcp/flyte_helm_dataplane_generated.yaml index ed40cc352e..4048f53c17 100644 --- a/deployment/gcp/flyte_helm_dataplane_generated.yaml +++ b/deployment/gcp/flyte_helm_dataplane_generated.yaml @@ -137,7 +137,7 @@ data: rate: 100 type: bucket type: batch - rawoutput-prefix: gs:/// + rawoutput-prefix: gs:/// workers: 40 workflow-reeval-duration: 30s webhook: @@ -174,7 +174,7 @@ data: json: "" project_id: scopes: https://www.googleapis.com/auth/cloud-platform - container: "" + container: "" enable-multicontainer: false limits: maxDownloadMBs: 10 diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml index fd314eb457..a5a028f15c 100644 --- a/deployment/gcp/flyte_helm_generated.yaml +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -202,7 +202,7 @@ data: json: "" project_id: scopes: https://www.googleapis.com/auth/cloud-platform - container: "" + container: "" enable-multicontainer: false limits: maxDownloadMBs: 10 @@ -404,7 +404,7 @@ data: json: "" project_id: scopes: https://www.googleapis.com/auth/cloud-platform - container: "" + container: "" enable-multicontainer: false limits: maxDownloadMBs: 10 @@ -517,7 +517,7 @@ data: rate: 100 type: bucket type: batch - rawoutput-prefix: gs:/// + rawoutput-prefix: gs:/// workers: 40 workflow-reeval-duration: 30s webhook: @@ -554,7 +554,7 @@ data: json: "" project_id: scopes: https://www.googleapis.com/auth/cloud-platform - container: "" + container: "" enable-multicontainer: false limits: maxDownloadMBs: 10 @@ -1221,7 +1221,7 @@ spec: template: metadata: annotations: - configChecksum: "bc69ed841506b28a42ac19bd0884d483472b3d11fe85fe7e546b879aeb30a85" + configChecksum: "905a2a911a85dbf8d4f1dc3be24b9c4fd7bb46481db0e174274d6aea6129b4c" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte