Merge branch 'main' into open_rao_integration

.github/workflows/release-ci.yml

@@ -4,6 +4,7 @@ on:
   release:
     types:
       - created
+      - released
 
 permissions:
   contents: write
@@ -82,6 +83,12 @@ jobs:
             name: darwin,
             os: macos-13,
           }
+          - {
+            name: darwin-arm64,
+            os: macos-14,
+            macosx_deployment_target: "11", # first arm64 version of macosx
+            bdist_wheel_args: "--plat-name macosx-11.0-arm64", # needed to avoid the wheel to be named -universal2
+          }
           - {
             name: windows,
             os: windows-2022,

cpp/CMakeLists.txt

@@ -37,6 +37,7 @@ endif()
 
 if(BUILD_PYTHON_BINDINGS)
   #Build pythin bindings
-  add_subdirectory(lib/pybind11)
+  set(PYBIND11_FINDPYTHON ON)
+  find_package(pybind11 CONFIG REQUIRED)
   add_subdirectory(pypowsybl-cpp)
 endif()

java/pom.xml

@@ -63,6 +63,8 @@
     <properties>
         <java.version>17</java.version>
         <commons-collections4.version>4.4</commons-collections4.version>
+        <commons-csv.version>1.11.0</commons-csv.version>
+
         <graalvm.version>23.0.0</graalvm.version>
         <janino.version>3.1.0</janino.version>
         <junit-jupiter.version>5.10.0</junit-jupiter.version>
@@ -211,6 +213,11 @@
             <artifactId>logback-core</artifactId>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-csv</artifactId>
+            <version>${commons-csv.version}</version>
+        </dependency>
         <!-- workaround for https://github.com/oracle/graal/issues/1943 -->
         <dependency>
             <groupId>org.codehaus.janino</groupId>

java/src/main/java/com/powsybl/dataframe/network/NetworkDataframes.java

@@ -24,6 +24,7 @@
 
 import java.util.*;
 import java.util.function.*;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 import static com.powsybl.dataframe.MappingUtils.*;
@@ -437,7 +438,9 @@ static NetworkDataframeMapper shuntsNonLinear() {
                         .filter(sc -> sc.getModelType() == ShuntCompensatorModelType.NON_LINEAR)
                         .flatMap(shuntCompensator -> {
                             ShuntCompensatorNonLinearModel model = (ShuntCompensatorNonLinearModel) shuntCompensator.getModel();
-                            return model.getAllSections().stream().map(section -> Triple.of(shuntCompensator, section, model.getAllSections().indexOf(section)));
+                            // careful: shunt section number starts at 1, but position in array starts at 0
+                            var allSections = model.getAllSections();
+                            return IntStream.range(0, allSections.size()).mapToObj(i -> Triple.of(shuntCompensator, allSections.get(i), i + 1));
                         });
         return NetworkDataframeMapperBuilder.ofStream(nonLinearShunts, NetworkDataframes::getShuntSectionNonlinear)
                 .stringsIndex("id", triple -> triple.getLeft().getId())
@@ -457,7 +460,12 @@ static Triple<ShuntCompensator, ShuntCompensatorNonLinearModel.Section, Integer>
         } else {
             int section = dataframe.getIntValue("section", index)
                     .orElseThrow(() -> new PowsyblException("section is missing"));
-            return Triple.of(shuntCompensator, shuntNonLinear.getAllSections().get(section), section);
+            // careful: shunt section number starts at 1, but position in array starts at 0
+            List<ShuntCompensatorNonLinearModel.Section> allSections = shuntNonLinear.getAllSections();
+            if (section < 1 || section > allSections.size()) {
+                throw new PowsyblException(String.format("Section number must be between 1 and %d, inclusive", allSections.size()));
+            }
+            return Triple.of(shuntCompensator, allSections.get(section - 1), section);
         }
     }
 

java/src/main/java/com/powsybl/python/network/Dataframes.java

@@ -310,18 +310,19 @@ private static List<NodeContext> getNodeBreakerViewNodes(VoltageLevel.NodeBreake
         return nodes.stream().map(node -> {
             Terminal terminal = nodeBreakerView.getTerminal(node);
             if (terminal == null) {
-                return new NodeContext(node, null);
+                return new NodeContext(node, null, null);
             } else {
-                return new NodeContext(node, terminal.getConnectable().getId());
+                return new NodeContext(node, terminal.getConnectable().getId(), terminal.getConnectable().getType());
             }
-        }).collect(Collectors.toList());
+        }).toList();
     }
 
     private static DataframeMapper<VoltageLevel.NodeBreakerView, Void> createNodeBreakerViewNodes() {
         return new DataframeMapperBuilder<VoltageLevel.NodeBreakerView, NodeContext, Void>()
                 .itemsProvider(Dataframes::getNodeBreakerViewNodes)
-                .intsIndex("node", NodeContext::getNode)
-                .strings("connectable_id", node -> Objects.toString(node.getConnectableId(), ""))
+                .intsIndex("node", NodeContext::node)
+                .strings("connectable_id", node -> Objects.toString(node.connectableId(), ""))
+                .strings("connectable_type", node -> Objects.toString(node.connectableType(), ""))
                 .build();
     }
 

java/src/main/java/com/powsybl/python/network/NodeContext.java

@@ -7,26 +7,10 @@
  */
 package com.powsybl.python.network;
 
-import javax.annotation.Nullable;
-import java.util.Objects;
+import com.powsybl.iidm.network.IdentifiableType;
 
 /**
  * @author Etienne Lesot {@literal <etienne.lesot at rte-france.com>}
  */
-public class NodeContext {
-    private final int node;
-    private final String connectableId;
-
-    public NodeContext(int node, @Nullable String connectableId) {
-        this.node = Objects.requireNonNull(node);
-        this.connectableId = connectableId;
-    }
-
-    public int getNode() {
-        return node;
-    }
-
-    public String getConnectableId() {
-        return connectableId;
-    }
+public record NodeContext(int node, String connectableId, IdentifiableType connectableType) {
 }

java/src/main/resources/META-INF/native-image/com.fasterxml.jackson.core/jackson-databind/reflect-config.json

@@ -6,5 +6,8 @@
 {
   "name":"com.fasterxml.jackson.databind.ext.Java7SupportImpl",
   "methods":[{"name":"<init>","parameterTypes":[] }]
+},
+{
+  "name":"[Lcom.fasterxml.jackson.databind.deser.Deserializers;"
 }
 ]

java/src/main/resources/META-INF/native-image/com.powsybl/powsybl-iidm-geodata/reflect-config.json

@@ -0,0 +1,9 @@
+[
+{
+  "name":"com.powsybl.iidm.geodata.odre.OdreGeoDataAdderPostProcessor"
+},
+{
+  "name":"com.powsybl.iidm.geodata.utils.GeoShapeDeserializer",
+  "methods":[{"name":"<init>","parameterTypes":[] }]
+}
+]

pypowsybl/network/impl/node_breaker_topology.py

@@ -53,7 +53,8 @@ def nodes(self) -> DataFrame:
 
         The dataframe includes the following columns:
 
-        - **connectable_id**: Connected element, if any.
+        - **connectable_id**: Connected element ID, if any.
+        - **connectable_type**: Connected element type, if any.
 
         This dataframe is indexed by the id of the nodes.
         """
@@ -76,7 +77,10 @@ def create_graph(self) -> _nx.Graph:
         Representation of the topology as a networkx graph.
         """
         graph = _nx.Graph()
-        graph.add_nodes_from(self._nodes.index.tolist())
-        graph.add_edges_from(self._switchs[['node1', 'node2']].values.tolist())
+        for (index, row) in self.nodes.iterrows():
+            graph.add_node(index, connectable_id=row['connectable_id'], connectable_type=row['connectable_type'])
+        for (index, row) in self._switchs.iterrows():
+            graph.add_edge(row['node1'], row['node2'], id=index, name=row['name'], kind=row['kind'],
+                           open=row['open'], retained=row['retained'])
         graph.add_edges_from(self._internal_connections[['node1', 'node2']].values.tolist())
         return graph

requirements.txt

@@ -1,10 +1,11 @@
 # actual dependencies
-pandas==2.2.2; python_version >= "3.9"
+pandas==2.2.3; python_version >= "3.9"
 pandas==2.0.3; python_version <= "3.8"
 prettytable==3.11.0 # last version supporting python 3.8
 networkx
 matplotlib==3.9.2; python_version >= "3.9"
 matplotlib==3.7.5; python_version <= "3.8"
+pybind11[global]==2.13.6
 
 # optional dependencies
 pandapower==2.14.11
@@ -14,7 +15,7 @@ sphinx==7.1.2
 furo==2024.1.29
 
 # CI dependencies
-setuptools==73.0.1
+setuptools==75.3.0
 wheel==0.44.0
 coverage==7.3.2
 pytest>=8.3.3

setup.py

@@ -12,13 +12,14 @@
 import zipfile
 import glob
 
-from setuptools import setup, Extension
-from setuptools.command.build_ext import build_ext
+from pybind11.setup_helpers import Pybind11Extension, build_ext
+from setuptools import setup
 from packaging.version import parse
 
-class PyPowsyblExtension(Extension):
+class PyPowsyblExtension(Pybind11Extension):
     def __init__(self):
-        Extension.__init__(self, 'pypowsybl._pypowsybl', sources=[])
+        Pybind11Extension.__init__(self, 'pypowsybl._pypowsybl',
+                                   ["cpp/*"])
 
 
 class PyPowsyblBuild(build_ext):
@@ -44,7 +45,7 @@ def build_extension(self, ext):
         if not extdir.endswith(os.path.sep):
             extdir += os.path.sep
         cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
-                      '-DPYTHON_EXECUTABLE=' + sys.executable]
+                      '-DPython_EXECUTABLE=' + sys.executable]
 
         cfg = 'Debug' if self.debug else 'Release'
         build_args = ['--config', cfg]

tests/test_network.py

@@ -1423,24 +1423,24 @@ def test_busbar_sections():
 def test_non_linear_shunt():
     n = util.create_non_linear_shunt_network()
     non_linear_shunt_sections = n.get_non_linear_shunt_compensator_sections()
-    pd.testing.assert_series_equal(non_linear_shunt_sections.loc[('SHUNT', 0)],
-                                   pd.Series(data={'g': 0.0, 'b': 0.00001},
-                                             name=('SHUNT', 0)), check_dtype=False)
     pd.testing.assert_series_equal(non_linear_shunt_sections.loc[('SHUNT', 1)],
-                                   pd.Series(data={'g': 0.3, 'b': 0.0200},
+                                   pd.Series(data={'g': 0.0, 'b': 0.00001},
                                              name=('SHUNT', 1)), check_dtype=False)
-    update = pd.DataFrame(index=pd.MultiIndex.from_tuples([('SHUNT', 0), ('SHUNT', 1)], names=['id', 'section']),
+    pd.testing.assert_series_equal(non_linear_shunt_sections.loc[('SHUNT', 2)],
+                                   pd.Series(data={'g': 0.3, 'b': 0.0200},
+                                             name=('SHUNT', 2)), check_dtype=False)
+    update = pd.DataFrame(index=pd.MultiIndex.from_tuples([('SHUNT', 1), ('SHUNT', 2)], names=['id', 'section']),
                           columns=['g', 'b'],
                           data=[[0.1, 0.00002],
                                 [0.4, 0.03]])
     n.update_non_linear_shunt_compensator_sections(update)
     non_linear_shunt_sections = n.get_non_linear_shunt_compensator_sections()
-    pd.testing.assert_series_equal(non_linear_shunt_sections.loc[('SHUNT', 0)],
-                                   pd.Series(data={'g': 0.1, 'b': 0.00002},
-                                             name=('SHUNT', 0)), check_dtype=False)
     pd.testing.assert_series_equal(non_linear_shunt_sections.loc[('SHUNT', 1)],
-                                   pd.Series(data={'g': 0.4, 'b': 0.03},
+                                   pd.Series(data={'g': 0.1, 'b': 0.00002},
                                              name=('SHUNT', 1)), check_dtype=False)
+    pd.testing.assert_series_equal(non_linear_shunt_sections.loc[('SHUNT', 2)],
+                                   pd.Series(data={'g': 0.4, 'b': 0.03},
+                                             name=('SHUNT', 2)), check_dtype=False)
 
 
 def test_voltage_levels():
@@ -1469,11 +1469,25 @@ def test_voltage_levels():
     pd.testing.assert_frame_equal(expected, n.get_voltage_levels(), check_dtype=False)
 
 
-def test_update_with_keywords():
+def test_update_non_linear_shunt_with_keywords():
+    n = util.create_non_linear_shunt_network()
+    n.update_non_linear_shunt_compensator_sections(id='SHUNT', section=1, g=0.2, b=0.000001)
+    n.update_non_linear_shunt_compensator_sections(id='SHUNT', section=2, g=0.3, b=0.000002)
+    sections = n.get_non_linear_shunt_compensator_sections()
+    assert 0.2 == sections.loc['SHUNT', 1]['g']
+    assert 0.000001 == sections.loc['SHUNT', 1]['b']
+    assert 0.3 == sections.loc['SHUNT', 2]['g']
+    assert 0.000002 == sections.loc['SHUNT', 2]['b']
+
+
+def test_update_non_linear_shunt_wrong_section():
     n = util.create_non_linear_shunt_network()
-    n.update_non_linear_shunt_compensator_sections(id='SHUNT', section=0, g=0.2, b=0.000001)
-    assert 0.2 == n.get_non_linear_shunt_compensator_sections().loc['SHUNT', 0]['g']
-    assert 0.000001 == n.get_non_linear_shunt_compensator_sections().loc['SHUNT', 0]['b']
+    with pytest.raises(PyPowsyblError) as exc:
+        n.update_non_linear_shunt_compensator_sections(id='SHUNT', section=0, g=0.2, b=0.000001)
+    assert exc.match('Section number must be between 1 and 2, inclusive')
+    with pytest.raises(PyPowsyblError) as exc:
+        n.update_non_linear_shunt_compensator_sections(id='SHUNT', section=3, g=0.2, b=0.000001)
+    assert exc.match('Section number must be between 1 and 2, inclusive')
 
 
 def test_update_generators_with_keywords():
@@ -1539,6 +1553,8 @@ def test_node_breaker_view():
     assert 0 == switches.loc['S4VL1_BBS_LINES3S4_DISCONNECTOR']['node1']
     assert 5 == switches.loc['S4VL1_BBS_LINES3S4_DISCONNECTOR']['node2']
     assert 7 == len(nodes)
+    assert 'S4VL1_BBS' == nodes.iloc[0]['connectable_id']
+    assert 'BUSBAR_SECTION' == nodes.iloc[0]['connectable_type']
     assert topology.internal_connections.empty
 
     with pytest.raises(PyPowsyblError) as exc:
@@ -1551,7 +1567,14 @@ def test_graph():
     network_topology = n.get_node_breaker_topology('S4VL1')
     graph = network_topology.create_graph()
     assert 7 == len(graph.nodes)
+    assert [0, 1, 2, 3, 4, 5, 6] == list(graph.nodes)
+    assert {'connectable_id': 'S4VL1_BBS', 'connectable_type': 'BUSBAR_SECTION'} == graph.nodes[0]
     assert [(0, 5), (0, 1), (0, 3), (1, 2), (3, 4), (5, 6)] == list(graph.edges)
+    assert {'id': 'S4VL1_BBS_LINES3S4_DISCONNECTOR',
+            'kind': 'DISCONNECTOR',
+            'name': 'S4VL1_BBS_LINES3S4_DISCONNECTOR',
+            'open': False,
+            'retained': False} == graph.edges[0, 5]
 
 
 @unittest.skip("plot graph skipping")

tests/test_network_elements_creation.py

@@ -446,16 +446,16 @@ def test_non_linear_shunt():
     assert shunt.b == 2
 
     model1 = n.get_non_linear_shunt_compensator_sections().loc['SHUNT1']
-    section1 = model1.loc[0]
-    section2 = model1.loc[1]
+    section1 = model1.loc[1]
+    section2 = model1.loc[2]
     assert section1.g == 1
     assert section1.b == 2
     assert section2.g == 3
     assert section2.b == 4
 
     model2 = n.get_non_linear_shunt_compensator_sections().loc['SHUNT2']
-    section1 = model2.loc[0]
-    section2 = model2.loc[1]
+    section1 = model2.loc[1]
+    section2 = model2.loc[2]
     assert section1.g == 5
     assert section1.b == 6
     assert section2.g == 7

tests/test_network_modification.py

@@ -527,16 +527,16 @@ def test_add_non_linear_shunt_bay():
     assert shunt.b == 2
 
     model1 = n.get_non_linear_shunt_compensator_sections().loc['shunt1']
-    section1 = model1.loc[0]
-    section2 = model1.loc[1]
+    section1 = model1.loc[1]
+    section2 = model1.loc[2]
     assert section1.g == 1
     assert section1.b == 2
     assert section2.g == 3
     assert section2.b == 4
 
     model2 = n.get_non_linear_shunt_compensator_sections().loc['shunt2']
-    section1 = model2.loc[0]
-    section2 = model2.loc[1]
+    section1 = model2.loc[1]
+    section2 = model2.loc[2]
     assert section1.g == 5
     assert section1.b == 6
     assert section2.g == 7
@@ -581,16 +581,16 @@ def test_add_non_linear_shunt_bay_bus_breaker():
     assert shunt.b == 2
 
     model1 = n.get_non_linear_shunt_compensator_sections().loc['shunt1']
-    section1 = model1.loc[0]
-    section2 = model1.loc[1]
+    section1 = model1.loc[1]
+    section2 = model1.loc[2]
     assert section1.g == 1
     assert section1.b == 2
     assert section2.g == 3
     assert section2.b == 4
 
     model2 = n.get_non_linear_shunt_compensator_sections().loc['shunt2']
-    section1 = model2.loc[0]
-    section2 = model2.loc[1]
+    section1 = model2.loc[1]
+    section2 = model2.loc[2]
     assert section1.g == 5
     assert section1.b == 6
     assert section2.g == 7