Fix failing functional tests for Vertica 10 (#440)

* Introduced vertica table abstraction

* Added complex_type table

* Added types table

* introduced selectWhereExpectOne

* updated unit tests

* schemaToolsV10 and unit tests updated

* ignore Array[Row] functional test

* comments added to VerticaTable

* refactored SchemaTools

* Added primitive type isSigned

* refactored SchemaTools function piping.

* Added Json package

* Added comments

* Updated SchemaTools complex type flow

* Added support for scale and precision

* Fixing scala style

* fixed scalastyle

* Added json option unit test

* Added VerticaJsonScan test

* increased unit test coverage.

* Added struct read schema support

* Close spark session on test finished

* - Fixed nested array bugs.
- Added functional tests for numeric types

* Unite tests updated

* removed unused json option

* Added unit tests

* updated unit tests

* extracted cleanup code for read pipe

* refactored into separated files

* Increased test coverage

* updated to Spark 3.2.1
fixed failing test.

* Added unit tests

* Reorganized classes

* Reorganized classes

* Updated cleaner interface

* updated scalastyle

* reorganized classes

* reorganized classes

* updated interface

* Hooked up cleaner to PartitionReaderWrapper

* updated unit tests

* updated cleanupFiles() to iterate over indices.

* moved classes

* Added Json export statement

* Added Json export statement

* Added Json export statement

* Fix scalastyle

* Merged with main

* Fixed casting issue

* Fixed import issues

* Added json option

* Added json option to functional test

* Added binary type checking during before read json export.

* minor refactor

* Added BasicJsonReadTests

* Added global json option to BasicJsonReadTests

* Moved files to new packages and added new unit tests

* Added case for array[binary]

* Added support for row field name

* Renamed traits to conform to mixins.

* Updated traits name

* Fixed merge

* Fix imports

* updated comments

* Fix import

* Moved JsonReadSupport

* Json

* Fixed unit test expectations

* Added batch factory test.

* Added PartitionReaderWrapperFactoryTest

* Updated export build

* Functional tests updated

* updated struct read test

* updated complex types read test to check for values

* Removed complex_types query by one. Unit tests updated

* updated ComplexTypesSchemaTools as a class

* Added unit test for array[binary]

* Fixed build issue

* Unit tests now covers properly covers "row" branch

* Fixed double when

* Removed unused import

* Added handling for unrecognized type kinds in complex_types table

* Added version checking and automatic complex type detection

* Refactored ComplexTypesSchemaTools recursion

* Moved listToEither to ErrorHandling

* Updated query tables to build rows using column names instead of indices.

* - Removed unused table queries.
- Added unit tests to increase coverage.

* Removed scala magic numbers

* Added unit tests for json version check and automatic

* Removed json options in complex type tests

* updated magic number

* Updated PlaceholderError

* Removed place holder error

* Added debug info

* now properly checks for all schema

* Updated unit tests

* removed todo

* Updated -v10 functional tests option

* Removed comments

functional-tests/src/main/scala/Main.scala

@@ -283,7 +283,7 @@ object Main extends App {
     )
 
     testSuites = if (options.v10) testSuites :+ new ComplexTypeTestsV10(readOpts, writeOpts, jdbcConfig, fileStoreConfig)
-    else testSuites :+ new ComplexTypeTests(readOpts, writeOpts, jdbcConfig, fileStoreConfig)
+    else testSuites ++ List(new ComplexTypeTests(readOpts, writeOpts, jdbcConfig, fileStoreConfig), new BasicJsonReadTests(readOpts, writeOpts, jdbcConfig, fileStoreConfig))
 
     if (options.large) testSuites = testSuites :+ new LargeDataTests(readOpts, writeOpts, jdbcConfig, fileStoreConfig)
 

functional-tests/src/main/scala/com/vertica/spark/functests/endtoend/ComplexTypeTestsV10.scala

@@ -172,6 +172,8 @@ class ComplexTypeTestsV10(readOpts: Map[String, String], writeOpts: Map[String,
       }
       case Success(_) => fail("Expected failure")
     }
+
+    TestUtils.fsCleanup(fsLayer, fsConfig.address)
   }
 
   it should "error on reading complex types from external table in Vertica 10" in {
@@ -215,5 +217,7 @@ class ComplexTypeTestsV10(readOpts: Map[String, String], writeOpts: Map[String,
       }
       case Success(_) => fail("Expected failure")
     }
+
+    TestUtils.fsCleanup(fsLayer, fsConfig.address)
   }
 }