From 8a646306334b2b18ac3cedc4ef94412e43ce6301 Mon Sep 17 00:00:00 2001 From: Manuel Soulier Date: Fri, 1 Nov 2024 23:44:00 +0100 Subject: [PATCH] fix: oom parquet export --- .../arc/core/service/p6export/parquet/ParquetDao.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/arc-core/src/main/java/fr/insee/arc/core/service/p6export/parquet/ParquetDao.java b/arc-core/src/main/java/fr/insee/arc/core/service/p6export/parquet/ParquetDao.java index 4e3f1e30..a2cf1511 100644 --- a/arc-core/src/main/java/fr/insee/arc/core/service/p6export/parquet/ParquetDao.java +++ b/arc-core/src/main/java/fr/insee/arc/core/service/p6export/parquet/ParquetDao.java @@ -11,6 +11,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; +import java.util.Properties; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; @@ -83,8 +84,11 @@ public void exportToParquet(List tables, String outputDirectory // load duckdb extension loadDuckdb(); - - try (Connection connection = DriverManager.getConnection("jdbc:duckdb:")) { + + Properties connectionProperties = new Properties(); + connectionProperties.setProperty("preserve_insertion_order", "false"); + + try (Connection connection = DriverManager.getConnection("jdbc:duckdb:",connectionProperties)) { // unzip extensions unzipExtensions(connection); @@ -279,7 +283,6 @@ private void attachPostgresDatabasesToDuckdb(Connection connection) + query.quoteText(encryptionKey.getValue()) + ");"); } } - executeQuery(connection, query); }