From c31aa8f3302fc5995b61e1d7917858756e826057 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 25 Jul 2023 10:35:29 +0200 Subject: [PATCH 1/8] install executable script for python programs --- .../generator/python/PythonGenerator.java | 32 ++++++++---- .../generator/python/PythonInfoGenerator.java | 52 ------------------- 2 files changed, 22 insertions(+), 62 deletions(-) delete mode 100644 core/src/main/java/org/lflang/generator/python/PythonInfoGenerator.java diff --git a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java index b8b3a74282..70a00db411 100644 --- a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java +++ b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java @@ -37,6 +37,7 @@ import org.eclipse.emf.ecore.resource.Resource; import org.eclipse.xtext.xbase.lib.Exceptions; import org.lflang.AttributeUtils; +import org.lflang.FileConfig; import org.lflang.Target; import org.lflang.TargetProperty; import org.lflang.ast.ASTUtils; @@ -100,8 +101,7 @@ public PythonGenerator(LFGeneratorContext context) { "lib/python_time.c", "lib/pythontarget.c"), PythonGenerator::setUpMainTarget, - "install(TARGETS)" // No-op - )); + generateCmakeInstall(context.getFileConfig()))); } private PythonGenerator( @@ -396,15 +396,10 @@ public void doGenerate(Resource resource, LFGeneratorContext context) { codeMaps.putAll(codeMapsForFederate); copyTargetFiles(); new PythonValidator(fileConfig, messageReporter, codeMaps, protoNames).doValidate(context); - if (targetConfig.noCompile) { - messageReporter.nowhere().info(PythonInfoGenerator.generateSetupInfo(fileConfig)); - } } catch (Exception e) { //noinspection ConstantConditions throw Exceptions.sneakyThrow(e); } - - messageReporter.nowhere().info(PythonInfoGenerator.generateRunInfo(fileConfig, lfModuleName)); } if (messageReporter.getErrorsOccurred()) { @@ -578,7 +573,7 @@ private static String setUpMainTarget( add_subdirectory(core) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}) set(LF_MAIN_TARGET ) - find_package(Python 3.7.0...<3.11.0 COMPONENTS Interpreter Development) + find_package(Python 3.7.0...<3.12.0 REQUIRED COMPONENTS Interpreter Development) Python_add_library( ${LF_MAIN_TARGET} MODULE @@ -598,11 +593,28 @@ private static String setUpMainTarget( target_link_libraries(${LF_MAIN_TARGET} PRIVATE ${Python_LIBRARIES}) target_compile_definitions(${LF_MAIN_TARGET} PUBLIC MODULE_NAME=) """) - .replace("", generatePythonModuleName(executableName)) - .replace("executableName", executableName); + .replace("", generatePythonModuleName(executableName)); // The use of fileConfig.name will break federated execution, but that's fine } + private static String generateCmakeInstall(FileConfig fileConfig) { + return """ + file(GENERATE OUTPUT CONTENT + "#!/bin/sh\n\\ + ${Python_EXECUTABLE} " + ) + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/ DESTINATION ${CMAKE_INSTALL_BINDIR}) + """ + .replace("", fileConfig.name) + .replace( + "", + fileConfig + .getSrcGenPath() + .resolve(fileConfig.name + ".py") + .toAbsolutePath() + .toString()); + } + /** * Generate a ({@code key}, {@code val}) tuple pair for the {@code define_macros} field of the * Extension class constructor from setuptools. diff --git a/core/src/main/java/org/lflang/generator/python/PythonInfoGenerator.java b/core/src/main/java/org/lflang/generator/python/PythonInfoGenerator.java deleted file mode 100644 index 01cadbcc5c..0000000000 --- a/core/src/main/java/org/lflang/generator/python/PythonInfoGenerator.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.lflang.generator.python; - -import java.io.File; -import org.lflang.FileConfig; - -public class PythonInfoGenerator { - /** - * Print information about necessary steps to install the supporting Python C extension for the - * generated program. - * - * @note Only needed if no-compile is set to true - */ - public static String generateSetupInfo(FileConfig fileConfig) { - return String.join( - "\n", - "", - "#####################################", - "To compile and install the generated code, do:", - " ", - " cd " + fileConfig.getSrcGenPath() + File.separator, - " python3 -m pip install --force-reinstall .", - ""); - } - - /** Print information on how to execute the generated program. */ - public static String generateRunInfo(FileConfig fileConfig, String lfModuleName) { - return String.join( - "\n", - "", - "#####################################", - "To run the generated program, use:", - " ", - " python3 " + fileConfig.getSrcGenPath() + File.separator + lfModuleName + ".py", - "", - "#####################################", - ""); - } - - /** Print information on how to execute the generated federation. */ - public static String generateFedRunInfo(FileConfig fileConfig) { - return String.join( - "\n", - "", - "#####################################", - "To run the generated program, run:", - " ", - " bash " + fileConfig.binPath + "/" + fileConfig.name, - "", - "#####################################", - ""); - } -} From e1b7e0a36f13fd67af14d9d56e1b6d221e3c90d2 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 25 Jul 2023 10:39:48 +0200 Subject: [PATCH 2/8] also install batch script on Windows --- .../generator/python/PythonGenerator.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java index 70a00db411..fcd86ed2a1 100644 --- a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java +++ b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java @@ -599,12 +599,21 @@ private static String setUpMainTarget( private static String generateCmakeInstall(FileConfig fileConfig) { return """ - file(GENERATE OUTPUT CONTENT - "#!/bin/sh\n\\ - ${Python_EXECUTABLE} " - ) - install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/ DESTINATION ${CMAKE_INSTALL_BINDIR}) - """ + if(WIN32) + file(GENERATE OUTPUT .bat CONTENT + "@echo off\n\\ + ${Python_EXECUTABLE} \n\\ + pause" + ) + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/.bat DESTINATION ${CMAKE_INSTALL_BINDIR}) + else() + file(GENERATE OUTPUT CONTENT + "#!/bin/sh\n\\ + ${Python_EXECUTABLE} " + ) + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/ DESTINATION ${CMAKE_INSTALL_BINDIR}) + endif() + """ .replace("", fileConfig.name) .replace( "", From da74101e35ed14c595ef965c757c30b5b21788bd Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 25 Jul 2023 10:59:28 +0200 Subject: [PATCH 3/8] fix escapes --- .../org/lflang/generator/python/PythonGenerator.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java index fcd86ed2a1..60a687691a 100644 --- a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java +++ b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java @@ -601,15 +601,14 @@ private static String generateCmakeInstall(FileConfig fileConfig) { return """ if(WIN32) file(GENERATE OUTPUT .bat CONTENT - "@echo off\n\\ - ${Python_EXECUTABLE} \n\\ - pause" + "@echo off\n\ + ${Python_EXECUTABLE} " ) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/.bat DESTINATION ${CMAKE_INSTALL_BINDIR}) else() file(GENERATE OUTPUT CONTENT - "#!/bin/sh\n\\ - ${Python_EXECUTABLE} " + "#!/bin/sh\\n\\ + ${Python_EXECUTABLE} " ) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/ DESTINATION ${CMAKE_INSTALL_BINDIR}) endif() @@ -621,7 +620,8 @@ private static String generateCmakeInstall(FileConfig fileConfig) { .getSrcGenPath() .resolve(fileConfig.name + ".py") .toAbsolutePath() - .toString()); + .toString() + .replace("\\", "\\\\")); } /** From 83853af76e6624ac49cd05dd64df8f149f0926a0 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 25 Jul 2023 11:26:06 +0200 Subject: [PATCH 4/8] require Python 3.10 --- .../main/java/org/lflang/generator/python/PythonGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java index 60a687691a..34241798f6 100644 --- a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java +++ b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java @@ -573,7 +573,7 @@ private static String setUpMainTarget( add_subdirectory(core) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}) set(LF_MAIN_TARGET ) - find_package(Python 3.7.0...<3.12.0 REQUIRED COMPONENTS Interpreter Development) + find_package(Python 3.10.0...<3.11.0 REQUIRED COMPONENTS Interpreter Development) Python_add_library( ${LF_MAIN_TARGET} MODULE From f3b7df445fb0dc0c5b99377a8f9851e678a48853 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 25 Jul 2023 12:46:41 +0200 Subject: [PATCH 5/8] use the script in bin for testing --- .../lflang/generator/python/PyFileConfig.java | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/python/PyFileConfig.java b/core/src/main/java/org/lflang/generator/python/PyFileConfig.java index 7c67b92493..3927b6d02e 100644 --- a/core/src/main/java/org/lflang/generator/python/PyFileConfig.java +++ b/core/src/main/java/org/lflang/generator/python/PyFileConfig.java @@ -2,10 +2,9 @@ import java.io.IOException; import java.nio.file.Path; -import java.util.List; import org.eclipse.emf.ecore.resource.Resource; +import org.lflang.generator.GeneratorUtils; import org.lflang.generator.c.CFileConfig; -import org.lflang.util.LFCommand; public class PyFileConfig extends CFileConfig { public PyFileConfig(Resource resource, Path srcGenBasePath, boolean useHierarchicalBin) @@ -13,19 +12,8 @@ public PyFileConfig(Resource resource, Path srcGenBasePath, boolean useHierarchi super(resource, srcGenBasePath, useHierarchicalBin); } - @Override - public LFCommand getCommand() { - return LFCommand.get( - "python3", List.of(srcPkgPath.relativize(getExecutable()).toString()), true, srcPkgPath); - } - - @Override - public Path getExecutable() { - return srcGenPath.resolve(name + getExecutableExtension()); - } - @Override protected String getExecutableExtension() { - return ".py"; + return GeneratorUtils.isHostWindows() ? ".bat" : ""; } } From e132dda561d3b0f19a5014456d18347cb6492575 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 25 Jul 2023 12:47:14 +0200 Subject: [PATCH 6/8] fix regex to also work on Windows --- .../integrationTest/java/org/lflang/tests/RunSingleTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/integrationTest/java/org/lflang/tests/RunSingleTest.java b/core/src/integrationTest/java/org/lflang/tests/RunSingleTest.java index 424568109a..29e62e835b 100644 --- a/core/src/integrationTest/java/org/lflang/tests/RunSingleTest.java +++ b/core/src/integrationTest/java/org/lflang/tests/RunSingleTest.java @@ -49,7 +49,7 @@ public class RunSingleTest { private static final Pattern TEST_FILE_PATTERN = - Pattern.compile("(test/(\\w+))/src/([^/]++/)*(\\w+.lf)"); + Pattern.compile("(test\\W(\\w+))\\Wsrc\\W(\\w++\\W)*(\\w+.lf)"); @Test public void runSingleTest() throws FileNotFoundException { From 4e64dbe3601d618a0dcc19f9785c3155fc43e25e Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 25 Jul 2023 12:59:14 +0200 Subject: [PATCH 7/8] also use bin scripts in federated launch script --- .../federated/launcher/PyBuildConfig.java | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/lflang/federated/launcher/PyBuildConfig.java b/core/src/main/java/org/lflang/federated/launcher/PyBuildConfig.java index 09f7fac34f..98f1c16ad6 100644 --- a/core/src/main/java/org/lflang/federated/launcher/PyBuildConfig.java +++ b/core/src/main/java/org/lflang/federated/launcher/PyBuildConfig.java @@ -12,26 +12,12 @@ public PyBuildConfig( } @Override - public String localExecuteCommand() { - return "python3 " - + fileConfig.getSrcGenPath() - + "/" - + federate.name - + "/" - + federate.name - + ".py -i $FEDERATION_ID"; + public String remoteExecuteCommand() { + return "bin/" + fileConfig.name + "_" + federate.name + " -i '$FEDERATION_ID'"; } @Override - public String remoteExecuteCommand() { - return "python3 src-gen/" - + fileConfig.name - + "/" - + federate.name - + "/" - + fileConfig.name - + "_" - + federate.name - + " -i '$FEDERATION_ID'"; + public String localExecuteCommand() { + return fileConfig.getFedBinPath().resolve(federate.name) + " -i $FEDERATION_ID"; } } From c226c66252ee5b09f5a02d950bf28d02f21d4dc2 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 25 Jul 2023 15:58:46 +0200 Subject: [PATCH 8/8] forward all arguments to the Python script --- .../generator/python/PythonGenerator.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java index 34241798f6..16615bb59e 100644 --- a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java +++ b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java @@ -598,30 +598,27 @@ private static String setUpMainTarget( } private static String generateCmakeInstall(FileConfig fileConfig) { + final var pyMainPath = + fileConfig.getSrcGenPath().resolve(fileConfig.name + ".py").toAbsolutePath(); + // need to replace '\' with '\\' on Windwos for proper escaping in cmake + final var pyMainName = pyMainPath.toString().replace("\\", "\\\\"); return """ if(WIN32) file(GENERATE OUTPUT .bat CONTENT "@echo off\n\ - ${Python_EXECUTABLE} " + ${Python_EXECUTABLE} %*" ) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/.bat DESTINATION ${CMAKE_INSTALL_BINDIR}) else() file(GENERATE OUTPUT CONTENT "#!/bin/sh\\n\\ - ${Python_EXECUTABLE} " + ${Python_EXECUTABLE} \\\"$@\\\"" ) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/ DESTINATION ${CMAKE_INSTALL_BINDIR}) endif() """ .replace("", fileConfig.name) - .replace( - "", - fileConfig - .getSrcGenPath() - .resolve(fileConfig.name + ".py") - .toAbsolutePath() - .toString() - .replace("\\", "\\\\")); + .replace("", pyMainName); } /**