WIP: Test Upcoming Iceberg and Nessie version

- Testing with Iceberg 1.5.0 RC0 and Nessie 0.77.1
- Remove deprecated usage

plugin/trino-iceberg/pom.xml

@@ -16,7 +16,7 @@
     <properties>
         <air.main.basedir>${project.parent.basedir}</air.main.basedir>
         <!-- Nessie version (matching to Iceberg release) must be bumped along with Iceberg version bump to avoid compatibility issues -->
-        <dep.nessie.version>0.71.0</dep.nessie.version>
+        <dep.nessie.version>0.77.1</dep.nessie.version>
     </properties>
 
     <dependencies>
@@ -239,12 +239,32 @@
             <groupId>org.projectnessie.nessie</groupId>
             <artifactId>nessie-client</artifactId>
             <version>${dep.nessie.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-annotations</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.projectnessie.nessie</groupId>
             <artifactId>nessie-model</artifactId>
             <version>${dep.nessie.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-annotations</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
@@ -356,14 +376,14 @@
         <dependency>
             <groupId>org.apache.httpcomponents.client5</groupId>
             <artifactId>httpclient5</artifactId>
-            <version>5.2.1</version>
+            <version>5.3.1</version>
             <scope>runtime</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.httpcomponents.core5</groupId>
             <artifactId>httpcore5</artifactId>
-            <version>5.2.1</version>
+            <version>5.2.4</version>
             <scope>runtime</scope>
         </dependency>
 

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieCatalogModule.java

@@ -22,9 +22,9 @@
 import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider;
 import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory;
 import org.apache.iceberg.nessie.NessieIcebergClient;
+import org.projectnessie.client.NessieClientBuilder;
 import org.projectnessie.client.api.NessieApiV1;
 import org.projectnessie.client.auth.BearerAuthenticationProvider;
-import org.projectnessie.client.http.HttpClientBuilder;
 
 import static io.airlift.configuration.ConfigBinder.configBinder;
 import static java.lang.Math.toIntExact;
@@ -47,7 +47,7 @@ protected void setup(Binder binder)
     @Singleton
     public static NessieIcebergClient createNessieIcebergClient(IcebergNessieCatalogConfig icebergNessieCatalogConfig)
     {
-        HttpClientBuilder builder = HttpClientBuilder.builder()
+        NessieClientBuilder builder = NessieClientBuilder.createClientBuilderFromSystemSettings()
                 .withUri(icebergNessieCatalogConfig.getServerUri())
                 .withDisableCompression(!icebergNessieCatalogConfig.isCompressionEnabled())
                 .withReadTimeout(toIntExact(icebergNessieCatalogConfig.getReadTimeout().toMillis()))

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieTableOperations.java

@@ -109,7 +109,9 @@ protected void commitNewTable(TableMetadata metadata)
     {
         verify(version.isEmpty(), "commitNewTable called on a table which already exists");
         try {
-            nessieClient.commitTable(null, metadata, writeNewMetadata(metadata, 0), table, toKey(new SchemaTableName(database, this.tableName)));
+            String contentId = table == null ? null : table.getId();
+            nessieClient.commitTable(null, metadata, writeNewMetadata(metadata, 0), contentId,
+                    toKey(new SchemaTableName(database, this.tableName)));
         }
         catch (NessieNotFoundException e) {
             throw new TrinoException(ICEBERG_COMMIT_ERROR, format("Cannot commit: ref '%s' no longer exists", nessieClient.refName()), e);
@@ -126,7 +128,11 @@ protected void commitToExistingTable(TableMetadata base, TableMetadata metadata)
     {
         verify(version.orElseThrow() >= 0, "commitToExistingTable called on a new table");
         try {
-            nessieClient.commitTable(base, metadata, writeNewMetadata(metadata, version.getAsInt() + 1), table, toKey(new SchemaTableName(database, this.tableName)));
+            if (table == null) {
+                table = nessieClient.table(toIdentifier(new SchemaTableName(database, tableName)));
+            }
+            nessieClient.commitTable(base, metadata, writeNewMetadata(metadata, version.getAsInt() + 1), table.getId(),
+                    toKey(new SchemaTableName(database, this.tableName)));
         }
         catch (NessieNotFoundException e) {
             throw new TrinoException(ICEBERG_COMMIT_ERROR, format("Cannot commit: ref '%s' no longer exists", nessieClient.refName()), e);

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java

@@ -36,8 +36,8 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.parallel.Execution;
+import org.projectnessie.client.NessieClientBuilder;
 import org.projectnessie.client.api.NessieApiV1;
-import org.projectnessie.client.http.HttpClientBuilder;
 
 import java.io.File;
 import java.io.IOException;
@@ -94,7 +94,7 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations)
         TrinoFileSystemFactory fileSystemFactory = new HdfsFileSystemFactory(HDFS_ENVIRONMENT, HDFS_FILE_SYSTEM_STATS);
         IcebergNessieCatalogConfig icebergNessieCatalogConfig = new IcebergNessieCatalogConfig()
                 .setServerUri(URI.create(nessieContainer.getRestApiUri()));
-        NessieApiV1 nessieApi = HttpClientBuilder.builder()
+        NessieApiV1 nessieApi = NessieClientBuilder.createClientBuilderFromSystemSettings()
                 .withUri(nessieContainer.getRestApiUri())
                 .build(NessieApiV1.class);
         NessieIcebergClient nessieClient = new NessieIcebergClient(nessieApi, icebergNessieCatalogConfig.getDefaultReferenceName(), null, ImmutableMap.of());
@@ -118,7 +118,7 @@ public void testDefaultLocation()
         IcebergNessieCatalogConfig icebergNessieCatalogConfig = new IcebergNessieCatalogConfig()
                 .setDefaultWarehouseDir(tmpDirectory.toAbsolutePath().toString())
                 .setServerUri(URI.create(nessieContainer.getRestApiUri()));
-        NessieApiV1 nessieApi = HttpClientBuilder.builder()
+        NessieApiV1 nessieApi = NessieClientBuilder.createClientBuilderFromSystemSettings()
                 .withUri(nessieContainer.getRestApiUri())
                 .build(NessieApiV1.class);
         NessieIcebergClient nessieClient = new NessieIcebergClient(nessieApi, icebergNessieCatalogConfig.getDefaultReferenceName(), null, ImmutableMap.of());

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergTrinoRestCatalogConnectorSmokeTest.java

@@ -185,14 +185,6 @@ public void testDropTableWithMissingManifestListFile()
                 .hasMessageContaining("Table location should not exist");
     }
 
-    @Test
-    @Override
-    public void testDropTableWithMissingDataFile()
-    {
-        assertThatThrownBy(super::testDropTableWithMissingDataFile)
-                .hasMessageContaining("Table location should not exist");
-    }
-
     @Test
     @Override
     public void testDropTableWithNonExistentTableLocation()

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java

@@ -309,14 +309,6 @@ public void testDropTableWithMissingManifestListFile()
                 .hasMessageContaining("Table location should not exist");
     }
 
-    @Test
-    @Override
-    public void testDropTableWithMissingDataFile()
-    {
-        assertThatThrownBy(super::testDropTableWithMissingDataFile)
-                .hasMessageContaining("Table location should not exist");
-    }
-
     @Test
     @Override
     public void testDropTableWithNonExistentTableLocation()

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java

@@ -28,9 +28,9 @@ public class NessieContainer
 {
     private static final Logger log = Logger.get(NessieContainer.class);
 
-    public static final String DEFAULT_IMAGE = "projectnessie/nessie:0.71.0";
+    public static final String DEFAULT_IMAGE = "ghcr.io/projectnessie/nessie:0.77.1";
     public static final String DEFAULT_HOST_NAME = "nessie";
-    public static final String VERSION_STORE_TYPE = "INMEMORY";
+    public static final String VERSION_STORE_TYPE = "IN_MEMORY";
 
     public static final int PORT = 19121;
 

plugin/trino-iceberg/src/test/java/org/apache/iceberg/jdbc/TestingTrinoIcebergJdbcUtil.java

@@ -16,8 +16,8 @@
 
 public final class TestingTrinoIcebergJdbcUtil
 {
-    public static final String CREATE_CATALOG_TABLE = JdbcUtil.CREATE_CATALOG_TABLE;
-    public static final String CREATE_NAMESPACE_PROPERTIES_TABLE = JdbcUtil.CREATE_NAMESPACE_PROPERTIES_TABLE;
+    public static final String CREATE_CATALOG_TABLE = JdbcUtil.CREATE_CATALOG_SQL;
+    public static final String CREATE_NAMESPACE_PROPERTIES_TABLE = JdbcUtil.CREATE_NAMESPACE_PROPERTIES_TABLE_SQL;
 
     private TestingTrinoIcebergJdbcUtil() {}
 }

pom.xml

@@ -188,7 +188,7 @@
         <dep.errorprone.version>2.24.1</dep.errorprone.version>
         <dep.flyway.version>10.7.2</dep.flyway.version>
         <dep.google.http.client.version>1.44.1</dep.google.http.client.version>
-        <dep.iceberg.version>1.4.3</dep.iceberg.version>
+        <dep.iceberg.version>1.5.0</dep.iceberg.version>
         <dep.jna.version>5.14.0</dep.jna.version>
         <dep.joda.version>2.12.7</dep.joda.version>
         <dep.jsonwebtoken.version>0.12.5</dep.jsonwebtoken.version>
@@ -2224,6 +2224,14 @@
         </dependencies>
     </dependencyManagement>
 
+    <repositories>
+        <repository>
+            <id>iceberg-release-candidate</id>
+            <name>Iceberg Release Candidate</name>
+            <url>https://repository.apache.org/content/repositories/orgapacheiceberg-1151/</url>
+        </repository>
+    </repositories>
+
     <build>
         <pluginManagement>
             <plugins>

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java

@@ -43,7 +43,7 @@ public class EnvSinglenodeSparkIcebergNessie
 
     private static final int SPARK_THRIFT_PORT = 10213;
     private static final int NESSIE_PORT = 19120;
-    private static final String NESSIE_VERSION = "0.71.0";
+    private static final String NESSIE_VERSION = "0.77.1";
     private static final String SPARK = "spark";
 
     private final DockerFiles dockerFiles;
@@ -99,8 +99,8 @@ private DockerContainer createSparkContainer()
 
     private DockerContainer createNessieContainer()
     {
-        DockerContainer container = new DockerContainer("projectnessie/nessie:" + NESSIE_VERSION, "nessie-server")
-                .withEnv("NESSIE_VERSION_STORE_TYPE", "INMEMORY")
+        DockerContainer container = new DockerContainer("ghcr.io/projectnessie/nessie:" + NESSIE_VERSION, "nessie-server")
+                .withEnv("NESSIE_VERSION_STORE_TYPE", "IN_MEMORY")
                 .withEnv("QUARKUS_HTTP_PORT", Integer.valueOf(NESSIE_PORT).toString())
                 .withStartupCheckStrategy(new IsRunningStartupCheckStrategy())
                 .waitingFor(forSelectedPorts(NESSIE_PORT));