Merge pull request #1820 from cdapio/CDAP-20810-fix-data-cacher

CDAP-20810 fix data cacher storage level
cdapio · Sep 13, 2023 · 64a2b65 · 64a2b65
2 parents 339807e + 0a56d8f
commit 64a2b65
Showing 1 changed file with 2 additions and 11 deletions.
diff --git a/spark-plugins/src/main/java/io/cdap/plugin/spark/DataCacher.java b/spark-plugins/src/main/java/io/cdap/plugin/spark/DataCacher.java
@@ -58,17 +58,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
 
   @Override
   public JavaRDD<StructuredRecord> transform(SparkExecutionPluginContext sparkExecutionPluginContext,
-                                             JavaRDD<StructuredRecord> javaRDD) throws Exception {
-
-    StorageLevel storageLevel = StorageLevel.MEMORY_AND_DISK();
-    if (config.storageLevel.isEmpty()) {
-      StorageLevel.fromString(config.storageLevel);
-    }
-    if (storageLevel == StorageLevel.NONE()) {
-      throw new RuntimeException(
-        String.format("Invalid storage level '%s'. Please select a valid value", config.storageLevel));
-    }
-
+                                             JavaRDD<StructuredRecord> javaRDD) {
+    StorageLevel storageLevel = StorageLevel.fromString(config.storageLevel);
     javaRDD.persist(storageLevel);
     return javaRDD;
   }