Merge pull request #62 from InseeFrLab/0.4.2

0.4.2
InseeFrLab · Dec 24, 2023 · 186bc85 · 186bc85
2 parents 70a1cd7 + 5ed1087
commit 186bc85
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 25 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -15,10 +15,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 11
-        uses: actions/setup-java@v1
+      - name: Set up Maven Central Repository
+        uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
+          distribution: "adopt"
       - name: Set current version
         run: mvn -B versions:set -DnewVersion=${GITHUB_REF##*/} -DprocessAllModules -DgenerateBackupPoms=false
       - uses: s4u/[email protected]

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM openjdk:11-jre-slim
+FROM openjdk:17-jdk-slim
 EXPOSE 8080
 
 COPY target/lib /lib/
@@ -13,4 +13,4 @@ COPY target/lib/postgis-jdbc-*.jar /postgis-jdbc.jar
 
 COPY target/trevas-lab*.jar.original /lib/trevas-lab.jar
 
-ENTRYPOINT ["java", "-cp", "/lib/*", "fr.insee.trevas.lab.TrevasLabApplication"]
+ENTRYPOINT ["java", "-cp", "/lib/*", "--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED", "fr.insee.trevas.lab.TrevasLabApplication"]
diff --git a/pom.xml b/pom.xml
@@ -5,12 +5,12 @@
     <parent>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-starter-parent</artifactId>
-        <version>2.4.1</version>
+        <version>2.7.14</version>
         <relativePath/> <!-- lookup parent from repository -->
     </parent>
     <groupId>fr.insee</groupId>
     <artifactId>trevas-lab</artifactId>
-    <version>0.4.1</version>
+    <version>0.4.2</version>
     <name>Trevas Lab</name>
     <description>VTL API calling Trevas</description>
 
@@ -22,16 +22,17 @@
     </licenses>
 
     <properties>
-        <java.version>11</java.version>
-        <trevas.version>1.1.1</trevas.version>
+        <java.version>17</java.version>
+        <trevas.version>1.2.0</trevas.version>
+        <jackson.version>2.16.0</jackson.version>
     </properties>
 
     <dependencyManagement>
         <dependencies>
             <dependency>
                 <groupId>org.codehaus.janino</groupId>
                 <artifactId>janino</artifactId>
-                <version>3.0.16</version>
+                <version>3.1.10</version>
             </dependency>
         </dependencies>
     </dependencyManagement>
@@ -117,13 +118,19 @@
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-kubernetes_2.12</artifactId>
-            <version>3.3.1</version>
+            <version>3.5.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.logging.log4j</groupId>
+                    <artifactId>log4j-slf4j2-impl</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.postgresql</groupId>
             <artifactId>postgresql</artifactId>
-            <version>42.6.0</version>
+            <version>42.7.0</version>
         </dependency>
         <dependency>
             <groupId>net.postgis</groupId>
@@ -132,35 +139,35 @@
         </dependency>
 
         <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
-            <version>8.0.33</version>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
+            <version>8.2.0</version>
         </dependency>
 
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>
-            <version>2.15.2</version>
+            <version>${jackson.version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.15.2</version>
+            <version>${jackson.version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
-            <version>2.15.2</version>
+            <version>${jackson.version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.module</groupId>
             <artifactId>jackson-module-scala_2.12</artifactId>
-            <version>2.15.2</version>
+            <version>${jackson.version}</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.datatype</groupId>
             <artifactId>jackson-datatype-jsr310</artifactId>
-            <version>2.15.2</version>
+            <version>${jackson.version}</version>
         </dependency>
 
 
@@ -216,6 +223,11 @@
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>3.2.2</version>
+            </plugin>
             <plugin>
                 <artifactId>maven-dependency-plugin</artifactId>
                 <executions>
@@ -230,6 +242,17 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.11.0</version>
+                <configuration>
+                    <compilerArgs>
+                        <arg>--add-exports</arg>
+                        <arg>java.base/sun.nio.ch=ALL-UNNAMED</arg>
+                    </compilerArgs>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

diff --git a/src/main/java/fr/insee/trevas/lab/utils/Utils.java b/src/main/java/fr/insee/trevas/lab/utils/Utils.java
@@ -4,6 +4,7 @@
 import fr.insee.trevas.lab.model.QueriesForBindingsToSave;
 import fr.insee.trevas.lab.model.S3ForBindings;
 import fr.insee.vtl.model.InMemoryDataset;
+import fr.insee.vtl.model.PersistentDataset;
 import fr.insee.vtl.spark.SparkDataset;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -40,7 +41,7 @@ public static ScriptEngine initEngineWithSpark(Bindings bindings, SparkSession s
     public static Bindings getBindings(Bindings input) {
         Bindings output = new SimpleBindings();
         input.forEach((k, v) -> {
-            if (!k.startsWith("$")) output.put(k, v);
+            if (v instanceof PersistentDataset) output.put(k, v);
         });
         return output;
     }
@@ -68,17 +69,18 @@ public static SparkConf loadSparkConfig(String stringPath) {
     public static Bindings getSparkBindings(Bindings input, Integer limit) {
         Bindings output = new SimpleBindings();
         input.forEach((k, v) -> {
-            if (!k.startsWith("$")) {
-                if (v instanceof SparkDataset) {
-                    Dataset<Row> sparkDs = ((SparkDataset) v).getSparkDataset();
+            if (v instanceof PersistentDataset) {
+                fr.insee.vtl.model.Dataset ds = ((PersistentDataset) v).getDelegate();
+                if (ds instanceof SparkDataset) {
+                    Dataset<Row> sparkDs = ((SparkDataset) ds).getSparkDataset();
                     if (limit != null) {
                         SparkDataset sparkDataset = new SparkDataset(sparkDs.limit(limit));
                         InMemoryDataset im = new InMemoryDataset(
                                 sparkDataset.getDataPoints(),
                                 sparkDataset.getDataStructure());
                         output.put(k, im);
                     } else output.put(k, new SparkDataset(sparkDs)); // useless
-                } else output.put(k, v);
+                }
             }
         });
         return output;