From d94d31a73e080e5669d8cdece5b63283948bd295 Mon Sep 17 00:00:00 2001 From: depetrol Date: Thu, 4 Jul 2024 17:13:29 -0700 Subject: [PATCH 1/2] feat: out-of-band serialization in LF 0.8 --- .../launcher/FedLauncherGenerator.java | 2 + .../FedNativePythonSerialization.java | 46 +++++++++---------- core/src/main/resources/lib/c/reactor-c | 2 +- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/lflang/federated/launcher/FedLauncherGenerator.java b/core/src/main/java/org/lflang/federated/launcher/FedLauncherGenerator.java index 332ee003c6..85e4e43b8f 100644 --- a/core/src/main/java/org/lflang/federated/launcher/FedLauncherGenerator.java +++ b/core/src/main/java/org/lflang/federated/launcher/FedLauncherGenerator.java @@ -280,6 +280,7 @@ private String getSetupCode() { "# Set a trap to kill all background jobs on error or control-C", "# Use two distinct traps so we can see which signal causes this.", "cleanup() {", + " tmux kill-session -t plasma", " if [ \"$EXITED_SUCCESSFULLY\" = true ] ; then", " exit 0", " else", @@ -293,6 +294,7 @@ private String getSetupCode() { "}", "", "trap 'cleanup; exit' EXIT", + "tmux new -d -s plasma plasma_store -m 4000000000 -s /tmp/plasma", "", "# Create a random 48-byte text ID for this federation.", "# The likelihood of two federations having the same ID is 1/16,777,216 (1/2^24).", diff --git a/core/src/main/java/org/lflang/federated/serialization/FedNativePythonSerialization.java b/core/src/main/java/org/lflang/federated/serialization/FedNativePythonSerialization.java index 971f37b879..a32ecd173b 100644 --- a/core/src/main/java/org/lflang/federated/serialization/FedNativePythonSerialization.java +++ b/core/src/main/java/org/lflang/federated/serialization/FedNativePythonSerialization.java @@ -63,28 +63,23 @@ public StringBuilder generateNetworkSerializerCode(String varName, String origin serializerCode.append( "if (global_pickler == NULL) lf_print_error_and_exit(\"The pickle module is not" + " loaded.\");\n"); - // Define the serialized PyObject - serializerCode - .append( - "PyObject* serialized_pyobject = PyObject_CallMethod(global_pickler, \"dumps\", \"O\"," - + " ") - .append(varName) - .append(");\n"); - - // Error check - serializerCode.append("if (serialized_pyobject == NULL) {\n"); - serializerCode.append(" if (PyErr_Occurred()) PyErr_Print();\n"); + // Check that global_serializer is not null serializerCode.append( - " lf_print_error_and_exit(\"Could not serialize serialized_pyobject.\");\n"); - serializerCode.append("}\n"); - - serializerCode.append("Py_buffer " + serializedVarName + ";\n"); + "if (global_serializer == NULL) lf_print_error_and_exit(\"The globalserializer module is not" + + " loaded.\");\n"); + // Define the serialized PyObject serializerCode.append( - "int returnValue = PyBytes_AsStringAndSize(serialized_pyobject, (char**)&" - + serializedVarName - + ".buf, &" - + serializedVarName - + ".len);\n"); + "PyObject *serializer_serialize = PyObject_GetAttrString(global_serializer, \"serialize\");\n" + + "PyObject *args = PyTuple_Pack(1, "+varName+");\n" + + "PyObject *serialized_pyobject = PyObject_CallObject(serializer_serialize, args);\n" + + "if (serialized_pyobject == NULL) {\n" + + " if (PyErr_Occurred()) PyErr_Print();\n" + + " lf_print_error_and_exit(\"Could not serialize object.\");\n" + + "}\n" + + "Py_buffer "+serializedVarName+";\n" + + "int returnValue = PyBytes_AsStringAndSize(serialized_pyobject, (char**)&"+serializedVarName+".buf, &"+serializedVarName+".len);\n" + + "" + ); // Error check serializerCode.append("if (returnValue == -1) {\n"); serializerCode.append(" if (PyErr_Occurred()) PyErr_Print();\n"); @@ -105,11 +100,16 @@ public StringBuilder generateNetworkDeserializerCode(String varName, String targ .append("->token->value, ") .append(varName) .append("->token->length);\n"); + // Check that global_serializer is not null + deserializerCode.append( + "if (global_serializer == NULL) lf_print_error_and_exit(\"The globalserializer module is not" + + " loaded.\");\n"); // Deserialize using Pickle deserializerCode.append( - "PyObject* " - + deserializedVarName - + " = PyObject_CallMethod(global_pickler, \"loads\", \"O\", message_byte_array);\n"); + "PyObject *serializer_deserialize = PyObject_GetAttrString(global_serializer, \"deserialize\");\n" + + "PyObject *args = PyTuple_Pack(1, message_byte_array);\n" + + "PyObject *"+deserializedVarName+" = PyObject_CallObject(serializer_deserialize, args);\n" + ); // Error check deserializerCode.append("if (" + deserializedVarName + " == NULL) {\n"); deserializerCode.append(" if (PyErr_Occurred()) PyErr_Print();\n"); diff --git a/core/src/main/resources/lib/c/reactor-c b/core/src/main/resources/lib/c/reactor-c index 8fe0436ebf..d6ae43b2fc 160000 --- a/core/src/main/resources/lib/c/reactor-c +++ b/core/src/main/resources/lib/c/reactor-c @@ -1 +1 @@ -Subproject commit 8fe0436ebf8bffb53dfc3373c093c0a80e88a6a2 +Subproject commit d6ae43b2fc7d8638c8b8131bdb2d7309c0e90412 From 94079c7239cd5ac9cae775003be6177e4723f7d5 Mon Sep 17 00:00:00 2001 From: depetrol Date: Thu, 4 Jul 2024 21:57:14 -0700 Subject: [PATCH 2/2] fix: update nightly build action secret & ./gradlew :core:spotlessApply --- .github/workflows/nightly-build.yml | 2 +- .../FedNativePythonSerialization.java | 48 +++++++++++-------- core/src/main/resources/lib/c/reactor-c | 2 +- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 199f15ba23..8d5d2237d2 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -35,7 +35,7 @@ jobs: - name: Deploy nightly release uses: marvinpinto/action-automatic-releases@latest with: - repo_token: "${{ secrets.NIGHTLY_BUILD }}" + repo_token: "${{ secrets.GITHUB_TOKEN }}" automatic_release_tag: 'nightly' prerelease: true title: "Lingua Franca Nightly" diff --git a/core/src/main/java/org/lflang/federated/serialization/FedNativePythonSerialization.java b/core/src/main/java/org/lflang/federated/serialization/FedNativePythonSerialization.java index a32ecd173b..67a60710c8 100644 --- a/core/src/main/java/org/lflang/federated/serialization/FedNativePythonSerialization.java +++ b/core/src/main/java/org/lflang/federated/serialization/FedNativePythonSerialization.java @@ -65,21 +65,29 @@ public StringBuilder generateNetworkSerializerCode(String varName, String origin + " loaded.\");\n"); // Check that global_serializer is not null serializerCode.append( - "if (global_serializer == NULL) lf_print_error_and_exit(\"The globalserializer module is not" - + " loaded.\");\n"); + "if (global_serializer == NULL) lf_print_error_and_exit(\"The globalserializer module is" + + " not loaded.\");\n"); // Define the serialized PyObject serializerCode.append( - "PyObject *serializer_serialize = PyObject_GetAttrString(global_serializer, \"serialize\");\n" + - "PyObject *args = PyTuple_Pack(1, "+varName+");\n" + - "PyObject *serialized_pyobject = PyObject_CallObject(serializer_serialize, args);\n" + - "if (serialized_pyobject == NULL) {\n" + - " if (PyErr_Occurred()) PyErr_Print();\n" + - " lf_print_error_and_exit(\"Could not serialize object.\");\n" + - "}\n" + - "Py_buffer "+serializedVarName+";\n" + - "int returnValue = PyBytes_AsStringAndSize(serialized_pyobject, (char**)&"+serializedVarName+".buf, &"+serializedVarName+".len);\n" + - "" - ); + "PyObject *serializer_serialize = PyObject_GetAttrString(global_serializer," + + " \"serialize\");\n" + + "PyObject *args = PyTuple_Pack(1, " + + varName + + ");\n" + + "PyObject *serialized_pyobject = PyObject_CallObject(serializer_serialize, args);\n" + + "if (serialized_pyobject == NULL) {\n" + + " if (PyErr_Occurred()) PyErr_Print();\n" + + " lf_print_error_and_exit(\"Could not serialize object.\");\n" + + "}\n" + + "Py_buffer " + + serializedVarName + + ";\n" + + "int returnValue = PyBytes_AsStringAndSize(serialized_pyobject, (char**)&" + + serializedVarName + + ".buf, &" + + serializedVarName + + ".len);\n" + + ""); // Error check serializerCode.append("if (returnValue == -1) {\n"); serializerCode.append(" if (PyErr_Occurred()) PyErr_Print();\n"); @@ -102,14 +110,16 @@ public StringBuilder generateNetworkDeserializerCode(String varName, String targ .append("->token->length);\n"); // Check that global_serializer is not null deserializerCode.append( - "if (global_serializer == NULL) lf_print_error_and_exit(\"The globalserializer module is not" - + " loaded.\");\n"); + "if (global_serializer == NULL) lf_print_error_and_exit(\"The globalserializer module is" + + " not loaded.\");\n"); // Deserialize using Pickle deserializerCode.append( - "PyObject *serializer_deserialize = PyObject_GetAttrString(global_serializer, \"deserialize\");\n" + - "PyObject *args = PyTuple_Pack(1, message_byte_array);\n" + - "PyObject *"+deserializedVarName+" = PyObject_CallObject(serializer_deserialize, args);\n" - ); + "PyObject *serializer_deserialize = PyObject_GetAttrString(global_serializer," + + " \"deserialize\");\n" + + "PyObject *args = PyTuple_Pack(1, message_byte_array);\n" + + "PyObject *" + + deserializedVarName + + " = PyObject_CallObject(serializer_deserialize, args);\n"); // Error check deserializerCode.append("if (" + deserializedVarName + " == NULL) {\n"); deserializerCode.append(" if (PyErr_Occurred()) PyErr_Print();\n"); diff --git a/core/src/main/resources/lib/c/reactor-c b/core/src/main/resources/lib/c/reactor-c index d6ae43b2fc..0b88a5e9fb 160000 --- a/core/src/main/resources/lib/c/reactor-c +++ b/core/src/main/resources/lib/c/reactor-c @@ -1 +1 @@ -Subproject commit d6ae43b2fc7d8638c8b8131bdb2d7309c0e90412 +Subproject commit 0b88a5e9fb391acc061e1aa2447c4cd3540cb15e