Merge branch 'hotfix/github-62'

dimajix · Apr 6, 2020 · 01e8e6c · 01e8e6c
2 parents 0f9c501 + f771a96
commit 01e8e6c
Show file tree

Hide file tree

Showing 19 changed files with 29 additions and 18 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,9 @@
-# Version 0.12.1 - 
+# Version 0.12.2 - 2020-04-06
+
+* Fix type coercion of DecimalTypes
+
+
+# Version 0.12.1 - 2020-01-15
 
 * Improve support for Swagger Schema
 * Fix infinite loop in recursiveSql

diff --git a/docker/pom.xml b/docker/pom.xml
@@ -10,7 +10,7 @@
     <parent>
         <groupId>com.dimajix.flowman</groupId>
         <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>..</relativePath>
     </parent>
 

diff --git a/flowman-core/pom.xml b/flowman-core/pom.xml
@@ -9,7 +9,7 @@
     <parent>
 	    <groupId>com.dimajix.flowman</groupId>
 	    <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>..</relativePath>
     </parent>
 

diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/types/SchemaUtils.scala b/flowman-core/src/main/scala/com/dimajix/flowman/types/SchemaUtils.scala
@@ -49,7 +49,8 @@ object SchemaUtils {
                 val rFractionalPrecision = rs
                 val totalIntegerPrecision = scala.math.max(lIntegerPrecision, rIntegerPrecision)
                 val totalFractionalPrecision = scala.math.max(lFractionalPrecision, rFractionalPrecision)
-                DecimalType(totalIntegerPrecision + totalFractionalPrecision, totalFractionalPrecision)
+                val totalPrecision = scala.math.min(totalIntegerPrecision + totalFractionalPrecision, DecimalType.MAX_PRECISION)
+                DecimalType(totalPrecision, totalFractionalPrecision)
 
             // Integrals are promoted to bigger type
             case (IntegralType(_, lp), IntegralType(_, rp)) =>

diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/types/SchemaUtilsTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/types/SchemaUtilsTest.scala
@@ -138,6 +138,11 @@ class SchemaUtilsTest extends FlatSpec with Matchers {
         SchemaUtils.coerce(DecimalType(4,2), DecimalType(10,1)) should be (DecimalType(11,2))
     }
 
+    it should "limit DecimlTypes to supported precisions" in {
+        SchemaUtils.coerce(DecimalType(38,2), DecimalType(38,10)) should be (DecimalType(38,10))
+        SchemaUtils.coerce(DecimalType(38,2), DecimalType(38,10)).sparkType should be (org.apache.spark.sql.types.DecimalType(38,10))
+    }
+
     it should "coerce VarChar and Char types" in {
         SchemaUtils.coerce(VarcharType(10), VarcharType(10)) should be (VarcharType(10))
         SchemaUtils.coerce(VarcharType(20), VarcharType(10)) should be (VarcharType(20))

diff --git a/flowman-dist/pom.xml b/flowman-dist/pom.xml
@@ -10,7 +10,7 @@
     <parent>
 	    <groupId>com.dimajix.flowman</groupId>
 	    <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>..</relativePath>
     </parent>
 

diff --git a/flowman-plugins/aws/pom.xml b/flowman-plugins/aws/pom.xml
@@ -10,7 +10,7 @@
     <parent>
         <groupId>com.dimajix.flowman</groupId>
         <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>../..</relativePath>
     </parent>
 

diff --git a/flowman-plugins/azure/pom.xml b/flowman-plugins/azure/pom.xml
@@ -10,7 +10,7 @@
     <parent>
         <groupId>com.dimajix.flowman</groupId>
         <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>../..</relativePath>
     </parent>
 

diff --git a/flowman-plugins/example/pom.xml b/flowman-plugins/example/pom.xml
@@ -9,7 +9,7 @@
     <parent>
 	    <groupId>com.dimajix.flowman</groupId>
 	    <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>../..</relativePath>
     </parent>
 

diff --git a/flowman-plugins/impala/pom.xml b/flowman-plugins/impala/pom.xml
@@ -9,7 +9,7 @@
     <parent>
 	    <groupId>com.dimajix.flowman</groupId>
 	    <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>../..</relativePath>
     </parent>
 

diff --git a/flowman-plugins/kafka/pom.xml b/flowman-plugins/kafka/pom.xml
@@ -9,7 +9,7 @@
     <parent>
 	    <groupId>com.dimajix.flowman</groupId>
 	    <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>../..</relativePath>
     </parent>
 

diff --git a/flowman-plugins/mariadb/pom.xml b/flowman-plugins/mariadb/pom.xml
@@ -10,7 +10,7 @@
     <parent>
         <groupId>com.dimajix.flowman</groupId>
         <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>../..</relativePath>
     </parent>
 

diff --git a/flowman-server/pom.xml b/flowman-server/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <artifactId>flowman-root</artifactId>
         <groupId>com.dimajix.flowman</groupId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>..</relativePath>
     </parent>
 

diff --git a/flowman-spark-sources/pom.xml b/flowman-spark-sources/pom.xml
@@ -9,7 +9,7 @@
     <parent>
 	    <groupId>com.dimajix.flowman</groupId>
 	    <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>..</relativePath>
     </parent>
 

diff --git a/flowman-spark-testing/pom.xml b/flowman-spark-testing/pom.xml
@@ -9,7 +9,7 @@
     <parent>
 	    <groupId>com.dimajix.flowman</groupId>
 	    <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>..</relativePath>
     </parent>
 

diff --git a/flowman-testing/pom.xml b/flowman-testing/pom.xml
@@ -9,7 +9,7 @@
     <parent>
 	    <groupId>com.dimajix.flowman</groupId>
 	    <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>..</relativePath>
     </parent>
 

diff --git a/flowman-tools/pom.xml b/flowman-tools/pom.xml
@@ -9,7 +9,7 @@
     <parent>
 	    <groupId>com.dimajix.flowman</groupId>
 	    <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>..</relativePath>
     </parent>
 

diff --git a/flowman-ui/pom.xml b/flowman-ui/pom.xml
@@ -9,7 +9,7 @@
     <parent>
 	    <groupId>com.dimajix.flowman</groupId>
 	    <artifactId>flowman-root</artifactId>
-        <version>0.12.1</version>
+        <version>0.12.2</version>
         <relativePath>..</relativePath>
     </parent>
 

diff --git a/pom.xml b/pom.xml
@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.dimajix.flowman</groupId>
     <artifactId>flowman-root</artifactId>
-    <version>0.12.1</version>
+    <version>0.12.2</version>
     <packaging>pom</packaging>
     <name>Flowman root pom</name>
     <description>A Spark based ETL tool</description>