Merge pull request opendatahub-io-contrib#69 from eformat/add-iceberg…

…-trino-connector 🕹️ add trino iceberg connector 🕹️
eformat · Jun 26, 2023 · 9a68a88 · 9a68a88
2 parents 33bdadb + c2b74b7
commit 9a68a88
Show file tree

Hide file tree

Showing 6 changed files with 1,231 additions and 1,178 deletions.
diff --git a/docs/3-supply-chain/3-s3-buckets.md b/docs/3-supply-chain/3-s3-buckets.md
@@ -30,6 +30,7 @@ Ideally we could automate this S3 configuration. But we keep this separate for n
    ```bash
    mc mb dev/airflow-${PROJECT_NAME}
    mc mb dev/hive-${PROJECT_NAME}
+   mc mb dev/hive-iceberg-${PROJECT_NAME}
    mc mb dev/pachyderm
    mc mb dev/iceberg
    mc mb dev/data

diff --git a/gitops/secrets/vault-rainforest b/gitops/secrets/vault-rainforest
diff --git a/gitops/trino/chart/trino/templates/hive-s3-secret.yaml b/gitops/trino/chart/trino/templates/hive-s3-secret.yaml
@@ -14,7 +14,7 @@ stringData:
   AWS_ACCESS_KEY_ID: <{{ $cat.name | upper }}_HIVE_S3_AWS_ACCESS_KEY_ID>
   AWS_SECRET_ACCESS_KEY: <{{ $cat.name | upper }}_HIVE_S3_AWS_ACCESS_KEY>
   BUCKET: <{{ $cat.name | upper }}_HIVE_S3_BUCKET>
-  S3_ENDPOINT: <{{ $cat.name | upper }}_HIVE_S3_ENDPOINT>
+  S3_ENDPOINT: <{{ $cat.name | upper }}_HIVE_S3_ENDPOINT_URL_PREFIX><{{ $cat.name | upper }}_HIVE_S3_ENDPOINT>
   S3_ENDPOINT_URL_PREFIX: <{{ $cat.name | upper }}_HIVE_S3_ENDPOINT_URL_PREFIX>
   S3_DATA_DIR: <{{ $cat.name | upper }}_HIVE_S3_DATA_DIR>
 {{- end }}

diff --git a/gitops/trino/chart/trino/templates/trino-catalog-secret.yaml b/gitops/trino/chart/trino/templates/trino-catalog-secret.yaml
@@ -11,12 +11,13 @@ stringData:
 {{- if .Values.catalogs }}
 {{- range $cat := .Values.catalogs }}
 {{- if $cat.enabled }}
+{{- if eq $cat.connector "hive" }}
   {{ $cat.name }}.properties: |
     connector.name=hive
     hive.hdfs.authentication.type=NONE
     hive.metastore.authentication.type=NONE
     hive.metastore.uri=thrift://{{ $cat.name }}-hive-metastore:9083
-    hive.s3.endpoint=<{{ $cat.name | upper }}_HIVE_S3_ENDPOINT>
+    hive.s3.endpoint=<{{ $cat.name | upper }}_HIVE_S3_ENDPOINT_URL_PREFIX><{{ $cat.name | upper }}_HIVE_S3_ENDPOINT>
     hive.s3.aws-access-key=<{{ $cat.name | upper }}_HIVE_S3_AWS_ACCESS_KEY_ID>
     hive.s3.aws-secret-key=<{{ $cat.name | upper }}_HIVE_S3_AWS_ACCESS_KEY>
     hive.s3.signer-type=S3SignerType
@@ -28,5 +29,25 @@ stringData:
     hive.parquet.use-column-names=true
     hive.recursive-directories=true
 {{- end }}
+{{- if eq $cat.connector "iceberg" }}
+  {{ $cat.name }}.properties: |
+    connector.name=iceberg
+    iceberg.file-format=PARQUET
+    iceberg.compression-codec=GZIP
+    iceberg.max-partitions-per-writer=300
+    iceberg.unique-table-location=true
+    hive.hdfs.authentication.type=NONE
+    hive.metastore.authentication.type=NONE
+    hive.metastore.uri=thrift://{{ $cat.name }}-hive-metastore:9083
+    hive.s3.endpoint=<{{ $cat.name | upper }}_HIVE_S3_ENDPOINT_URL_PREFIX><{{ $cat.name | upper }}_HIVE_S3_ENDPOINT>
+    hive.s3.aws-access-key=<{{ $cat.name | upper }}_HIVE_S3_AWS_ACCESS_KEY_ID>
+    hive.s3.aws-secret-key=<{{ $cat.name | upper }}_HIVE_S3_AWS_ACCESS_KEY>
+    hive.s3.signer-type=S3SignerType
+    hive.s3.path-style-access=true
+    hive.s3.staging-directory=/tmp
+    hive.s3.ssl.enabled=false
+    hive.s3.sse.enabled=false
+{{- end }}
+{{- end }}
 {{- end }}
 {{- end }}
diff --git a/gitops/trino/chart/trino/values.yaml b/gitops/trino/chart/trino/values.yaml
@@ -81,5 +81,12 @@ catalogs:
   # Hive Demo Catalog
   - name: demo
     enabled: true
+    connector: hive
+    replicaCountHive: 1
+    replicaCountDb: 1
+  # Hive Iceberg Catalog
+  - name: iceberg
+    enabled: true
+    connector: iceberg
     replicaCountHive: 1
     replicaCountDb: 1
diff --git a/supply-chain/trino/trino/Dockerfile b/supply-chain/trino/trino/Dockerfile
@@ -47,7 +47,7 @@ COPY --chown=trino:root trino/default/etc $TRINO_HOME
 RUN chmod 755 /usr/lib/trino/jmx_exporter.jar /usr/bin/trino
 
 # Remove all unused plugins
-RUN find /usr/lib/trino/plugin -maxdepth 1 -type d ! \( -name plugin -o -name hive -o -name blackhole -o -name jmx -o -name memory -o -name postgresql -o -name tpcds -o -name tpch -o -name password-authenticators -o -name thrift -o -name sqlserver -o -name oracle \) -exec rm -rf {} \;
+RUN find /usr/lib/trino/plugin -maxdepth 1 -type d ! \( -name plugin -o -name hive -o -name blackhole -o -name jmx -o -name memory -o -name postgresql -o -name tpcds -o -name tpch -o -name password-authenticators -o -name thrift -o -name sqlserver -o -name oracle -o -name iceberg \) -exec rm -rf {} \;
 
 # certs
 COPY cacerts.pem /etc/pki/ca-trust/source/anchors/cacerts.pem