From 71cc6b8b05b27430a3d6582e6b9df24c46a6ea8e Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Thu, 12 Sep 2024 18:59:44 -0400 Subject: [PATCH 1/5] chore: fix behavior of init files --- .../microsoft/azure/synapse/ml/codegen/PyCodegen.scala | 5 +++-- .../azure/synapse/ml/core/env/FileUtilities.scala | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala index 425d7314f6..47a0f1e8df 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala @@ -42,9 +42,10 @@ object PyCodegen { val initFile = new File(dir, "__init__.py") if (packageFolder != "/cognitive"){ if (packageFolder != "") { - writeFile(initFile, conf.packageHelp(importStrings)) + prependToFile(initFile, conf.packageHelp(importStrings)) } else if (initFile.exists()) { - initFile.delete() + throw new IllegalArgumentException(initFile.toString) +// initFile.delete() } } dir.listFiles().filter(_.isDirectory).foreach(f => diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/core/env/FileUtilities.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/core/env/FileUtilities.scala index baecf3d8bd..1a7363a9ab 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/core/env/FileUtilities.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/core/env/FileUtilities.scala @@ -6,7 +6,7 @@ package com.microsoft.azure.synapse.ml.core.env import org.apache.hadoop.fs.Path import java.io.File -import java.nio.file.{Files, StandardCopyOption} +import java.nio.file.{Files, StandardCopyOption, StandardOpenOption} import scala.io.{BufferedSource, Source} object FileUtilities { @@ -61,6 +61,13 @@ object FileUtilities { () } + def prependToFile(file: File, stuff: Any): Unit = { + val existingContent = new String(Files.readAllBytes(file.toPath)) + val newContent = stuff.toString + existingContent + Files.write(file.toPath, newContent.getBytes(), StandardOpenOption.TRUNCATE_EXISTING) + () + } + def copyFile(from: File, toDir: File, overwrite: Boolean = false): Unit = { Files.copy(from.toPath, new File(toDir, from.getName).toPath, (if (overwrite) Seq(StandardCopyOption.REPLACE_EXISTING) From 33dfeef0bbffa2827b40742973515be09e933842 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Thu, 12 Sep 2024 19:02:54 -0400 Subject: [PATCH 2/5] chore: fix behavior of init files --- .../microsoft/azure/synapse/ml/core/env/FileUtilities.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/core/env/FileUtilities.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/core/env/FileUtilities.scala index 1a7363a9ab..544d8bd1db 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/core/env/FileUtilities.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/core/env/FileUtilities.scala @@ -6,7 +6,7 @@ package com.microsoft.azure.synapse.ml.core.env import org.apache.hadoop.fs.Path import java.io.File -import java.nio.file.{Files, StandardCopyOption, StandardOpenOption} +import java.nio.file.{Files, StandardCopyOption} import scala.io.{BufferedSource, Source} object FileUtilities { @@ -64,7 +64,7 @@ object FileUtilities { def prependToFile(file: File, stuff: Any): Unit = { val existingContent = new String(Files.readAllBytes(file.toPath)) val newContent = stuff.toString + existingContent - Files.write(file.toPath, newContent.getBytes(), StandardOpenOption.TRUNCATE_EXISTING) + Files.write(file.toPath, newContent.getBytes()) () } From 0d76c2e8b31b8589e94f474c6135a44fe1ea712d Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Thu, 12 Sep 2024 19:04:11 -0400 Subject: [PATCH 3/5] chore: fix behavior of init files --- .../com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala index 47a0f1e8df..baae44e0e3 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala @@ -44,7 +44,7 @@ object PyCodegen { if (packageFolder != "") { prependToFile(initFile, conf.packageHelp(importStrings)) } else if (initFile.exists()) { - throw new IllegalArgumentException(initFile.toString) + throw new IllegalArgumentException("HEREEEEE!" + initFile.toString) // initFile.delete() } } From ee87e5e28396c268fe26f04a90d69af97fd21f09 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Thu, 12 Sep 2024 19:05:52 -0400 Subject: [PATCH 4/5] chore: fix behavior of init files --- .../com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala index baae44e0e3..adea625d2f 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala @@ -44,8 +44,9 @@ object PyCodegen { if (packageFolder != "") { prependToFile(initFile, conf.packageHelp(importStrings)) } else if (initFile.exists()) { - throw new IllegalArgumentException("HEREEEEE!" + initFile.toString) -// initFile.delete() + if (initFile.length() == 0) { + initFile.delete() + } } } dir.listFiles().filter(_.isDirectory).foreach(f => From 5029dd71c1dfdaff51cb625935db6d40416689a5 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Thu, 12 Sep 2024 19:08:20 -0400 Subject: [PATCH 5/5] chore: fix behavior of init files --- .../com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala index adea625d2f..b6cbef3a81 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/PyCodegen.scala @@ -42,7 +42,11 @@ object PyCodegen { val initFile = new File(dir, "__init__.py") if (packageFolder != "/cognitive"){ if (packageFolder != "") { - prependToFile(initFile, conf.packageHelp(importStrings)) + if (initFile.exists()) { + prependToFile(initFile, conf.packageHelp(importStrings)) + } else { + writeFile(initFile, conf.packageHelp(importStrings)) + } } else if (initFile.exists()) { if (initFile.length() == 0) { initFile.delete()