Skip to content

Commit

Permalink
Save network to a binary buffer (#672)
Browse files Browse the repository at this point in the history
Save network to a binary buffer

Signed-off-by: Geoffroy Jamgotchian <[email protected]>
  • Loading branch information
geofjamg authored Nov 13, 2023
1 parent 6c803de commit bdf4b7c
Show file tree
Hide file tree
Showing 11 changed files with 240 additions and 39 deletions.
7 changes: 5 additions & 2 deletions cpp/src/bindings.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -326,10 +326,13 @@ PYBIND11_MODULE(_pypowsybl, m) {
m.def("load_network_from_binary_buffers", &pypowsybl::loadNetworkFromBinaryBuffers, "Load a network from a list of binary buffer", py::call_guard<py::gil_scoped_release>(),
py::arg("buffers"), py::arg("parameters"), py::arg("reporter"));

m.def("dump_network", &pypowsybl::dumpNetwork, "Dump network to a file in a given format", py::call_guard<py::gil_scoped_release>(),
m.def("save_network", &pypowsybl::saveNetwork, "Save network to a file in a given format", py::call_guard<py::gil_scoped_release>(),
py::arg("network"), py::arg("file"),py::arg("format"), py::arg("parameters"), py::arg("reporter"));

m.def("dump_network_to_string", &pypowsybl::dumpNetworkToString, "Dump network in a given format", py::call_guard<py::gil_scoped_release>(),
m.def("save_network_to_string", &pypowsybl::saveNetworkToString, "Save network in a given format to a string", py::call_guard<py::gil_scoped_release>(),
py::arg("network"), py::arg("format"), py::arg("parameters"), py::arg("reporter"));

m.def("save_network_to_binary_buffer", &pypowsybl::saveNetworkToBinaryBuffer, "Save network in a given format to a binary byffer", py::call_guard<py::gil_scoped_release>(),
py::arg("network"), py::arg("format"), py::arg("parameters"), py::arg("reporter"));

m.def("reduce_network", &pypowsybl::reduceNetwork, "Reduce network", py::call_guard<py::gil_scoped_release>(),
Expand Down
26 changes: 22 additions & 4 deletions cpp/src/pypowsybl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -664,7 +664,7 @@ JavaHandle loadNetworkFromBinaryBuffers(std::vector<py::buffer> byteBuffers, con
return networkHandle;
}

void dumpNetwork(const JavaHandle& network, const std::string& file, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter) {
void saveNetwork(const JavaHandle& network, const std::string& file, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter) {
std::vector<std::string> parameterNames;
std::vector<std::string> parameterValues;
parameterNames.reserve(parameters.size());
Expand All @@ -675,11 +675,11 @@ void dumpNetwork(const JavaHandle& network, const std::string& file, const std::
}
ToCharPtrPtr parameterNamesPtr(parameterNames);
ToCharPtrPtr parameterValuesPtr(parameterValues);
callJava(::dumpNetwork, network, (char*) file.data(), (char*) format.data(), parameterNamesPtr.get(), parameterNames.size(),
callJava(::saveNetwork, network, (char*) file.data(), (char*) format.data(), parameterNamesPtr.get(), parameterNames.size(),
parameterValuesPtr.get(), parameterValues.size(), (reporter == nullptr) ? nullptr : *reporter);
}

std::string dumpNetworkToString(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter) {
std::string saveNetworkToString(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter) {
std::vector<std::string> parameterNames;
std::vector<std::string> parameterValues;
parameterNames.reserve(parameters.size());
Expand All @@ -690,10 +690,28 @@ std::string dumpNetworkToString(const JavaHandle& network, const std::string& fo
}
ToCharPtrPtr parameterNamesPtr(parameterNames);
ToCharPtrPtr parameterValuesPtr(parameterValues);
return toString(callJava<char*>(::dumpNetworkToString, network, (char*) format.data(), parameterNamesPtr.get(), parameterNames.size(),
return toString(callJava<char*>(::saveNetworkToString, network, (char*) format.data(), parameterNamesPtr.get(), parameterNames.size(),
parameterValuesPtr.get(), parameterValues.size(), (reporter == nullptr) ? nullptr : *reporter));
}

py::bytes saveNetworkToBinaryBuffer(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter) {
std::vector<std::string> parameterNames;
std::vector<std::string> parameterValues;
parameterNames.reserve(parameters.size());
parameterValues.reserve(parameters.size());
for (std::pair<std::string, std::string> p : parameters) {
parameterNames.push_back(p.first);
parameterValues.push_back(p.second);
}
ToCharPtrPtr parameterNamesPtr(parameterNames);
ToCharPtrPtr parameterValuesPtr(parameterValues);
array* byteArray = callJava<array*>(::saveNetworkToBinaryBuffer, network, (char*) format.data(), parameterNamesPtr.get(), parameterNames.size(),
parameterValuesPtr.get(), parameterValues.size(), reporter == nullptr ? nullptr : *reporter);
py::bytes bytes((char*) byteArray->ptr, byteArray->length);
callJava<>(::freeNetworkBinaryBuffer, byteArray);
return bytes;
}

void reduceNetwork(const JavaHandle& network, double v_min, double v_max, const std::vector<std::string>& ids,
const std::vector<std::string>& vls, const std::vector<int>& depths, bool withDangLingLines) {
ToCharPtrPtr elementIdPtr(ids);
Expand Down
6 changes: 4 additions & 2 deletions cpp/src/pypowsybl.h
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ JavaHandle loadNetworkFromString(const std::string& fileName, const std::string&

JavaHandle loadNetworkFromBinaryBuffers(std::vector<py::buffer> byteBuffer, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);

void dumpNetwork(const JavaHandle& network, const std::string& file, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);
void saveNetwork(const JavaHandle& network, const std::string& file, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);

LoadFlowParameters* createLoadFlowParameters();

Expand All @@ -364,7 +364,9 @@ SensitivityAnalysisParameters* createSensitivityAnalysisParameters();

std::vector<std::string> getSensitivityAnalysisProviderParametersNames(const std::string& sensitivityAnalysisProvider);

std::string dumpNetworkToString(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);
std::string saveNetworkToString(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);

py::bytes saveNetworkToBinaryBuffer(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);

void reduceNetwork(const JavaHandle& network, const double v_min, const double v_max, const std::vector<std::string>& ids, const std::vector<std::string>& vls, const std::vector<int>& depths, bool withDangLingLines);

Expand Down
10 changes: 8 additions & 2 deletions docs/user_guide/network.rst
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,19 @@ Networks can be written to the filesystem, using one of the available export for

.. code-block:: python
network.dump('network.xiidm', format='XIIDM')
network.save('network.xiidm', format='XIIDM')
You can also serialize networks to a string:

.. code-block:: python
xiidm_str = network.dump_to_string('XIIDM')
xiidm_str = network.save_to_string('XIIDM')
And also to a zip file as a (io.BytesIO) binary buffer.

.. code-block:: python
zipped_xiidm = network.save_to_binary_buffer('XIIDM')
The supported formats are:

Expand Down
4 changes: 4 additions & 0 deletions java/src/main/java/com/powsybl/python/commons/CTypeUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ public static CCharPointer toCharPtr(String str) {
}
// pybind11 convert std::string and char* to python utf-8 string
byte[] bytes = str.getBytes(StandardCharsets.UTF_8);
return toCharPtr(bytes);
}

public static CCharPointer toCharPtr(byte[] bytes) {
CCharPointer charPtr = UnmanagedMemory.calloc((bytes.length + 1) * SizeOf.get(CCharPointer.class));
for (int i = 0; i < bytes.length; ++i) {
charPtr.write(i, bytes[i]);
Expand Down
5 changes: 5 additions & 0 deletions java/src/main/java/com/powsybl/python/commons/Util.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.powsybl.python.dataframe.CDataframeHandler;
import org.graalvm.nativeimage.UnmanagedMemory;
import org.graalvm.nativeimage.c.struct.SizeOf;
import org.graalvm.nativeimage.c.type.CCharPointer;
import org.graalvm.nativeimage.c.type.CCharPointerPointer;
import org.graalvm.nativeimage.c.type.CDoublePointer;
import org.graalvm.nativeimage.c.type.CIntPointer;
Expand Down Expand Up @@ -156,6 +157,10 @@ public static ArrayPointer<CIntPointer> createIntegerArray(List<Integer> integer
return allocArrayPointer(intListPtr, integerList.size());
}

public static ArrayPointer<CCharPointer> createByteArray(byte[] bytes) {
return allocArrayPointer(CTypeUtil.toCharPtr(bytes), bytes.length);
}

public static int convert(SeriesDataType type) {
return switch (type) {
case STRING -> CDataframeHandler.STRING_SERIES_TYPE;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.util.*;
import java.util.zip.ZipOutputStream;

import static com.powsybl.python.commons.CTypeUtil.toStringList;
import static com.powsybl.python.commons.PyPowsyblApiHeader.*;
Expand Down Expand Up @@ -210,8 +211,8 @@ public static ObjectHandle loadNetworkFromBinaryBuffers(IsolateThread thread, CC
});
}

@CEntryPoint(name = "dumpNetwork")
public static void dumpNetwork(IsolateThread thread, ObjectHandle networkHandle, CCharPointer file, CCharPointer format,
@CEntryPoint(name = "saveNetwork")
public static void saveNetwork(IsolateThread thread, ObjectHandle networkHandle, CCharPointer file, CCharPointer format,
CCharPointerPointer parameterNamesPtrPtr, int parameterNamesCount,
CCharPointerPointer parameterValuesPtrPtr, int parameterValuesCount,
ObjectHandle reporterHandle, ExceptionHandlerPointer exceptionHandlerPtr) {
Expand All @@ -228,8 +229,8 @@ public static void dumpNetwork(IsolateThread thread, ObjectHandle networkHandle,
});
}

@CEntryPoint(name = "dumpNetworkToString")
public static CCharPointer dumpNetworkToString(IsolateThread thread, ObjectHandle networkHandle, CCharPointer format,
@CEntryPoint(name = "saveNetworkToString")
public static CCharPointer saveNetworkToString(IsolateThread thread, ObjectHandle networkHandle, CCharPointer format,
CCharPointerPointer parameterNamesPtrPtr, int parameterNamesCount,
CCharPointerPointer parameterValuesPtrPtr, int parameterValuesCount,
ObjectHandle reporterHandle, ExceptionHandlerPointer exceptionHandlerPtr) {
Expand Down Expand Up @@ -260,6 +261,40 @@ public static CCharPointer dumpNetworkToString(IsolateThread thread, ObjectHandl
});
}

@CEntryPoint(name = "saveNetworkToBinaryBuffer")
public static ArrayPointer<CCharPointer> saveNetworkToBinaryBuffer(IsolateThread thread, ObjectHandle networkHandle, CCharPointer format,
CCharPointerPointer parameterNamesPtrPtr, int parameterNamesCount,
CCharPointerPointer parameterValuesPtrPtr, int parameterValuesCount,
ObjectHandle reporterHandle, ExceptionHandlerPointer exceptionHandlerPtr) {
return doCatch(exceptionHandlerPtr, () -> {
Network network = ObjectHandles.getGlobal().get(networkHandle);
String formatStr = CTypeUtil.toString(format);
Properties parameters = createParameters(parameterNamesPtrPtr, parameterNamesCount, parameterValuesPtrPtr, parameterValuesCount);
var exporter = Exporter.find(formatStr);
if (exporter == null) {
throw new PowsyblException("No exporter found for '" + formatStr + "' to export as a string");
}
Reporter reporter = ReportCUtils.getReporter(reporterHandle);
// to support all kind of export: simple file or multiple to an archive,
// best is to write to a zip file
ByteArrayOutputStream bos = new ByteArrayOutputStream();
try (ZipOutputStream zos = new ZipOutputStream(bos)) {
DataSource dataSource = new ZipMemDataSource("file", zos);
exporter.export(network, parameters, dataSource, reporter);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
byte[] bytes = bos.toByteArray();
return Util.createByteArray(bytes);
});
}

@CEntryPoint(name = "freeNetworkBinaryBuffer")
public static void freeNetworkBinaryBuffer(IsolateThread thread, PyPowsyblApiHeader.ArrayPointer<CCharPointer> byteArrayPtr,
PyPowsyblApiHeader.ExceptionHandlerPointer exceptionHandlerPtr) {
doCatch(exceptionHandlerPtr, () -> freeArrayPointer(byteArrayPtr));
}

@CEntryPoint(name = "reduceNetwork")
public static void reduceNetwork(IsolateThread thread, ObjectHandle networkHandle,
double vMin, double vMax,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/**
* Copyright (c) 2023, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
* SPDX-License-Identifier: MPL-2.0
*/
package com.powsybl.python.network;

import com.powsybl.commons.datasource.DataSource;
import com.powsybl.commons.datasource.DataSourceUtil;
import com.powsybl.commons.io.ForwardingOutputStream;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Objects;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
* @author Geoffroy Jamgotchian {@literal <geoffroy.jamgotchian at rte-france.com>}
*/
public class ZipMemDataSource implements DataSource {

private final String baseName;

private final ZipOutputStream zos;

public ZipMemDataSource(String baseName, ZipOutputStream zos) {
this.baseName = Objects.requireNonNull(baseName);
this.zos = Objects.requireNonNull(zos);
}

@Override
public OutputStream newOutputStream(String fileName, boolean append) throws IOException {
Objects.requireNonNull(fileName);
if (append) {
throw new UnsupportedOperationException("append not supported in zip file data source");
}
zos.putNextEntry(new ZipEntry(fileName));
return new ForwardingOutputStream<OutputStream>(zos) {
@Override
public void close() throws IOException {
zos.closeEntry();
}
};
}

@Override
public OutputStream newOutputStream(String suffix, String ext, boolean append) throws IOException {
return newOutputStream(DataSourceUtil.getFileName(baseName, suffix, ext), append);
}

@Override
public String getBaseName() {
return baseName;
}

@Override
public boolean exists(String suffix, String ext) {
throw new UnsupportedOperationException();
}

@Override
public boolean exists(String fileName) {
throw new UnsupportedOperationException();
}

@Override
public InputStream newInputStream(String suffix, String ext) {
throw new UnsupportedOperationException();
}

@Override
public InputStream newInputStream(String fileName) {
throw new UnsupportedOperationException();
}

@Override
public Set<String> listNames(String regex) {
throw new UnsupportedOperationException();
}
}
5 changes: 3 additions & 2 deletions pypowsybl/_pypowsybl.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -553,8 +553,9 @@ def get_extensions_names() -> List[str]: ...
def get_extensions_information() -> SeriesArray: ...
def create_security_analysis() -> JavaHandle: ...
def create_sensitivity_analysis() -> JavaHandle: ...
def dump_network(network: JavaHandle, file: str, format: str, parameters: Dict[str,str], report: Optional[JavaHandle]) -> None: ...
def dump_network_to_string(network: JavaHandle, format: str, parameters: Dict[str,str], report: Optional[JavaHandle]) -> str: ...
def save_network(network: JavaHandle, file: str, format: str, parameters: Dict[str,str], report: Optional[JavaHandle]) -> None: ...
def save_network_to_string(network: JavaHandle, format: str, parameters: Dict[str,str], report: Optional[JavaHandle]) -> str: ...
def save_network_to_binary_buffer(network: JavaHandle, format: str, parameters: Dict[str,str], report: Optional[JavaHandle]) -> bytes: ...
def get_branch_flows_sensitivity_matrix(sensitivity_analysis_result_context: JavaHandle, matrix_id: str, contingency_id: str) -> Matrix: ...
def get_branch_results(result: JavaHandle) -> SeriesArray: ...
def get_bus_breaker_view_buses(network: JavaHandle, voltage_level: str) -> SeriesArray: ...
Expand Down
Loading

0 comments on commit bdf4b7c

Please sign in to comment.