Merge pull request #218 from alphaville/feature/pyo3-interface

Python-to-Rust interface using Pyo3

docs/example_estimation_py.md

@@ -219,7 +219,11 @@ meta = og.config.OptimizerMeta()               \
     .with_optimizer_name("estimator")
 builder = og.builder.OpEnOptimizerBuilder(problem, meta, build_config)
 builder.build()
+```
+
+The generated solver can be consumed over its TCP interface:
 
+```python
 # Use TCP server
 # ------------------------------------
 mng = og.tcp.OptimizerTcpManager('python_test_build/estimator')

docs/example_rosenbrock_py.md

@@ -42,11 +42,11 @@ problem = og.builder.Problem(u, p, phi) \
     .with_penalty_constraints(c)        \
     .with_constraints(bounds)
 build_config = og.config.BuildConfiguration()  \
-    .with_build_directory("python_test_build") \
+    .with_build_directory("my_optimizers") \
     .with_build_mode("debug")                  \
     .with_tcp_interface_config()
 meta = og.config.OptimizerMeta()                   \
-    .with_optimizer_name("tcp_enabled_optimizer")
+    .with_optimizer_name("rosenbrock")
 solver_config = og.config.SolverConfiguration()    \
     .with_tolerance(1e-5)                          \
     .with_delta_tolerance(1e-4)                    \
@@ -58,12 +58,16 @@ builder.build()
 
 # Use TCP server
 # ------------------------------------
-mng = og.tcp.OptimizerTcpManager('python_test_build/tcp_enabled_optimizer')
+mng = og.tcp.OptimizerTcpManager('my_optimizers/rosenbrock')
 mng.start()
 
 mng.ping()
-solution = mng.call([1.0, 50.0])
-print(solution)
+server_response = mng.call([1.0, 50.0])
+
+if server_response.is_ok():
+    solution = server_response.get()
+    u_star = solution.solution
+    status = solution.exit_status
 
 mng.kill()
 ```

docs/python-bindings.md

@@ -0,0 +1,133 @@
+---
+id: python-bindings
+title: Direct interface
+description: How to access the auto-generated optimizer from Python
+---
+
+<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script>
+<script type="text/javascript" async src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
+
+<div class="alert alert-warning">
+<b>Info:</b> The functionality presented here was introduced in <code>opengen</code> version <code>0.6.4</code>.
+The API is still young and is likely to change in versions <code>0.7.X</code>.
+</div>
+
+
+As we [discussed previously](python-interface#about) there are various ways
+you can use the auto-generated optimizer. You can use the generated Rust code,
+access it over a TCP socket, and so on. In this section we will focus on 
+how to access it *directly*.
+
+The idea is that OpEn can generate a Python module that you can `import`.
+
+## Generate a Python module for your optimizer
+Consider the following [parametric optimization problem](example_rosenbrock_py):
+<div class="math">
+\[
+    \begin{align}
+    \operatorname*{Minimize}_{\|u\|\leq r}& \sum_{i=1}^{n_u - 1} b (u_{i+1} - u_{i}^2)^2 + (a-u_i)^2
+    \\
+    \text{subject to: }& 1.5 u_1 - u_2 = 0
+    \\
+    &u_3 - u_4 + 0.1 \leq 0
+    \end{align}
+\]</div>
+
+```python
+import opengen as og
+import casadi.casadi as cs
+
+u = cs.SX.sym("u", 5)
+p = cs.SX.sym("p", 2)
+phi = og.functions.rosenbrock(u, p)
+c = cs.vertcat(1.5 * u[0] - u[1],
+               cs.fmax(0.0, u[2] - u[3] + 0.1))
+bounds = og.constraints.Ball2(radius=1.5)
+problem = og.builder.Problem(u, p, phi) \
+    .with_penalty_constraints(c) \
+    .with_constraints(bounds)
+build_config = og.config.BuildConfiguration() \
+    .with_build_directory("my_optimizers") \
+    .with_build_mode("debug") \
+    .with_build_python_bindings()
+meta = og.config.OptimizerMeta() \
+    .with_optimizer_name("rosenbrock")
+builder = og.builder.OpEnOptimizerBuilder(problem, meta,
+                                          build_config)
+builder.build()
+```
+
+Note that we have used `with_build_python_bindings()`.
+
+This will allow us to **import the auto-generated optimizer** as a Python 
+module!
+
+
+
+## Use the generated module
+
+The above code generates an optimizer which is stored in `my_optimizers/rosenbrock`.
+In that directory you can find a file called `rosenbrock.so` (or `rosenbrock.pyd` on Windows).
+This can be loaded as an autogenerated Python module.
+However, mind that this directory is most likely not in your Python path, 
+so you will have to add it before you can import the optimizer.
+This can be done very easily:
+
+```python
+sys.path.insert(1, './my_optimizers/rosenbrock')
+import rosenbrock
+```
+
+Then you will be able to use it as follows:
+
+```python
+solver = rosenbrock.solver()
+result = solver.run(p=[20., 1.])
+u_star = result.solution
+```
+
+In the first line, `solver = rosenbrock.solver()`, we obtain an instance of 
+`Solver`, which can be used to solve parametric optimization problems.
+In the second line, `result = solver.run(p=[20., 1.])`, we call the solver
+with parameter $p=(20, 1)$. Method `run` accepts another three optional
+arguments, namely:
+
+- `initial_guess` (can be either a list or a numpy array),
+- `initial_lagrange_multipliers`, and 
+- `initial_penalty`
+
+The solver returns an object of type `OptimizerSolution` with the following 
+properties:
+
+
+| Property                 | Explanation                                 |
+|--------------------------|---------------------------------------------|
+| `exit_status`             | Exit status; can be (i) `Converged` or (ii) `NotConvergedIterations`, if the maximum number of iterations was reached, therefore, the algorithm did not converge up to the specified tolerances, or (iii) `NotConvergedOutOfTime`, if the solver did not have enough time to converge |
+| `num_outer_iterations`    | Number of outer iterations   |
+| `num_inner_iterations`    | Total number of inner iterations (for all inner problems)    |
+| `last_problem_norm_fpr`   | Norm of the fixed-point residual of the last inner problem; this is a measure of the solution quality of the inner problem      |
+| `f1_infeasibility`       | Euclidean norm of $c^{-1}(y^+-y)$, which is equal to the distance between $F_1(u, p)$ and $C$ at the solution   |
+| `f2_norm`                 | Euclidean norm of $F_2(u, p)$ at the solution|
+| `solve_time_ms`           | Total execution time in milliseconds |
+| `penalty`                 | Last value of the penalty parameter |
+| `solution`                | Solution |
+| `cost`                    | Cost function at solution |
+| `lagrange_multipliers`    | Vector of Lagrange multipliers (if $n_1 > 0$) or an empty vector, otherwise |
+
+These are the same properties as those of `opengen.tcp.SolverStatus`.
+
+
+## Importing optimizer with variable name
+
+Previously we used `import rosenbrock` to import the auto-generated module.
+
+The limitation of this syntax is that it makes it difficult to change the name of the optimizer, i.e., `rosenbrock` is hard-coded.
+
+A better syntax would be:
+
+```python
+optimizers_dir = "my_optimizers"
+optimizer_name = "rosenbrock"
+sys.path.insert(1, os.path.join(optimizers_dir, optimizer_name))
+rosenbrock = __import__(optimizer_name)
+```

docs/python-interface.md

@@ -26,6 +26,7 @@ of the following ways:
 - Directly in **Rust** (you can include it in you Rust project as a dependency)
 - Over a **TCP socket** based on JSON (which can be accessed easily from any programming language)
 - In **Python** (using the TCP/IP interface in the background)
+- In [**Python**](python-bindings) by accessing the Rust-based auto-generated optimizer directly
 - In [**C** or **C++**](python-c) using auto-generated C/C++ bindings (header files and static or shared libraries)
 - In [**ROS**](python-ros) using auto-generated ROS packages
 

open-codegen/CHANGELOG.md

@@ -8,6 +8,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 Note: This is the Changelog file of `opengen` - the Python interface of OpEn
 
 
+## [0.6.4] - 2020-10-21
+
+### Added
+
+* Accessing Rust from Python directly using PyO3 
+
+### Fixed
+
+* List of authors in `Cargo.toml` is generated properly
+
+# Changed
+
+* `enable_tcp_interface` previously gave a `DeprecationWarning`; now it raises it. In a future version, it will be removed. 
+
+
 ## [0.6.3] - 2020-10-06
 
 ### Fixed
@@ -77,6 +92,7 @@ Note: This is the Changelog file of `opengen` - the Python interface of OpEn
 * Project-specific `tcp_iface` TCP interface
 * Fixed `lbfgs` typo
 
+[0.6.2]: https://github.com/alphaville/optimization-engine/compare/opengen-0.6.3...opengen-0.6.4
 [0.6.2]: https://github.com/alphaville/optimization-engine/compare/opengen-0.6.2...opengen-0.6.3
 [0.6.2]: https://github.com/alphaville/optimization-engine/compare/opengen-0.6.1...opengen-0.6.2
 [0.6.1]: https://github.com/alphaville/optimization-engine/compare/opengen-v0.6.0...opengen-0.6.1

open-codegen/VERSION

@@ -1 +1 @@
-0.6.3
+0.6.4

open-codegen/opengen/builder/optimizer_builder.py

@@ -11,13 +11,15 @@
 import jinja2
 import logging
 import pkg_resources
+import sys
 
 from .ros_builder import RosBuilder
 
 _AUTOGEN_COST_FNAME = 'auto_casadi_cost.c'
 _AUTOGEN_GRAD_FNAME = 'auto_casadi_grad.c'
 _AUTOGEN_PNLT_CONSTRAINTS_FNAME = 'auto_casadi_mapping_f2.c'
 _AUTOGEN_ALM_MAPPING_F1_FNAME = 'auto_casadi_mapping_f1.c'
+_PYTHON_BINDINGS_PREFIX = 'python_bindings_'
 _TCP_IFACE_PREFIX = 'tcp_iface_'
 _ICASADI_PREFIX = 'icasadi_'
 _ROS_PREFIX = 'ros_node_'
@@ -241,7 +243,8 @@ def __generate_cargo_toml(self):
             meta=self.__meta,
             build_config=self.__build_config,
             activate_tcp_server=self.__build_config.tcp_interface_config is not None,
-            activate_clib_generation=self.__build_config.build_c_bindings)
+            activate_clib_generation=self.__build_config.build_c_bindings,
+            timestamp_created=datetime.datetime.now())
         cargo_toml_path = os.path.abspath(os.path.join(target_dir, "Cargo.toml"))
         with open(cargo_toml_path, "w") as fh:
             fh.write(cargo_output_template)
@@ -421,6 +424,36 @@ def __build_optimizer(self):
         if process_completion != 0:
             raise Exception('Rust build failed')
 
+    def __build_python_bindings(self):
+        self.__logger.info("Building Python bindings")
+        target_dir = os.path.abspath(self.__target_dir())
+        optimizer_name = self.__meta.optimizer_name
+        python_bindings_dir_name = _PYTHON_BINDINGS_PREFIX + optimizer_name
+        python_bindings_dir = os.path.join(target_dir, python_bindings_dir_name)
+        command = self.__make_build_command()
+        p = subprocess.Popen(command, cwd=python_bindings_dir)
+        process_completion = p.wait()
+        if process_completion != 0:
+            raise Exception('Rust build of Python bindings failed')
+
+        if self.__build_config.build_mode.lower() == 'release':
+            build_dir = os.path.join(python_bindings_dir, 'target', 'release')
+        else:
+            build_dir = os.path.join(python_bindings_dir, 'target', 'debug')
+
+        pltform_extension_dict = {'linux': ('.so', '.so'),
+                                  'darwin': ('.dylib', '.so'),
+                                  'win32': ('.dll', '.pyd')}
+        (original_lib_extension, target_lib_extension) = pltform_extension_dict[sys.platform]
+        generated_bindings = os.path.join(build_dir, f"lib{optimizer_name}{original_lib_extension}")
+        target_bindings = os.path.join(target_dir, f"{optimizer_name}{target_lib_extension}")
+        shutil.copyfile(generated_bindings, target_bindings)
+        self.__logger.info(f"To use the Python bindings do:\n\n"
+                           f"       import sys\n"
+                           f"       sys.path.insert(1, \"{target_dir}\")\n"
+                           f"       import {optimizer_name}\n"
+                           f"       solver = {optimizer_name}.solver()")
+
     def __build_tcp_iface(self):
         self.__logger.info("Building the TCP interface")
         target_dir = os.path.abspath(self.__target_dir())
@@ -439,6 +472,43 @@ def __initialize(self):
     def __check_user_provided_parameters(self):
         self.__logger.info("Checking user parameters")
 
+    def __generate_code_python_bindings(self):
+        self.__logger.info("Generating code for Python bindings")
+        target_dir = self.__target_dir()
+        python_bindings_dir_name = _PYTHON_BINDINGS_PREFIX + self.__meta.optimizer_name
+        python_bindings_dir = os.path.join(target_dir, python_bindings_dir_name)
+        python_bindings_source_dir = os.path.join(python_bindings_dir, "src")
+        self.__logger.info(f"Generating code for Python bindings in {python_bindings_dir}")
+
+        # make python_bindings/ and python_bindings/src
+        make_dir_if_not_exists(python_bindings_dir)
+        make_dir_if_not_exists(python_bindings_source_dir)
+
+        # generate tcp_server.rs for python_bindings
+        python_rs_template = OpEnOptimizerBuilder.__get_template('python_bindings.rs', 'python')
+        python_rs_output_template = python_rs_template.render(
+            meta=self.__meta,
+            timestamp_created=datetime.datetime.now())
+        target_python_rs_path = os.path.join(python_bindings_source_dir, "lib.rs")
+        with open(target_python_rs_path, "w") as fh:
+            fh.write(python_rs_output_template)
+
+        # generate Cargo.toml for python_bindings
+        python_rs_template = OpEnOptimizerBuilder.__get_template('python_bindings_cargo.toml', 'python')
+        python_rs_output_template = python_rs_template.render(
+            meta=self.__meta,
+            build_config=self.__build_config,
+            timestamp_created=datetime.datetime.now())
+        target_python_rs_path = os.path.join(python_bindings_dir, "Cargo.toml")
+        with open(target_python_rs_path, "w") as fh:
+            fh.write(python_rs_output_template)
+
+        # move cargo_config into .cargo/config
+        target_cargo_config_dir = os.path.join(python_bindings_dir, '.cargo')
+        make_dir_if_not_exists(target_cargo_config_dir)
+        cargo_config_file = os.path.join(og_dfn.templates_dir(), 'python', 'cargo_config')
+        shutil.copy(cargo_config_file, os.path.join(target_cargo_config_dir, 'config'))
+
     def __generate_code_tcp_interface(self):
         self.__logger.info("Generating code for TCP/IP interface (tcp_iface/src/main.rs)")
         self.__logger.info("TCP server will bind at %s:%d",
@@ -518,9 +588,9 @@ def __generate_yaml_data_file(self):
 
     def enable_tcp_interface(self,
                              tcp_server_configuration=og_cfg.TcpServerConfiguration()):
-        warnings.warn("deprecated (use BuildConfiguration.with_tcp_interface_config instead)",
-                      DeprecationWarning)
-        self.__build_config.with_tcp_interface_config(tcp_server_configuration)
+        # This method should not be used!
+        raise DeprecationWarning(
+            "deprecated (use BuildConfiguration.with_tcp_interface_config instead)")
 
     def __generate_c_bindings_example(self):
         self.__logger.info("Generating example_optimizer.c (C bindings example for your convenience)")
@@ -582,9 +652,16 @@ def build(self):
                 self.__build_tcp_iface()
 
         if self.__build_config.build_c_bindings:
+            self.__logger.info("Generating C/C++ bindings")
             self.__generate_c_bindings_example()
             self.__generate_c_bindings_makefile()
 
+        if self.__build_config.build_python_bindings:
+            self.__logger.info("Generating Python bindings")
+            self.__generate_code_python_bindings()
+            if not self.__generate_not_build:
+                self.__build_python_bindings()
+
         if self.__build_config.ros_config is not None:
             ros_builder = RosBuilder(
                 self.__meta,

open-codegen/opengen/config/build_config.py

@@ -36,6 +36,7 @@ def __init__(self, build_dir="."):
         self.__build_dir = build_dir
         self.__open_version = None
         self.__build_c_bindings = False
+        self.__build_python_bindings = False
         self.__ros_config = None
         self.__tcp_interface_config = None
         self.__local_path = None
@@ -103,6 +104,10 @@ def local_path(self):
     def build_c_bindings(self):
         return self.__build_c_bindings
 
+    @property
+    def build_python_bindings(self):
+        return self.__build_python_bindings
+
     @property
     def tcp_interface_config(self):
         return self.__tcp_interface_config
@@ -204,6 +209,21 @@ def with_build_c_bindings(self, build_c_bindings=True):
         self.__build_c_bindings = build_c_bindings
         return self
 
+    def with_build_python_bindings(self, build_python_bindings=True):
+        """
+        If activated, OpEn will generate python bindings for the
+        auto-generated solver
+
+        :param build_python_bindings: whether to build python bindings for
+        auto-generated solver; default: `True`, i.e., it suffices
+        to call `build_config.with_build_python_bindings()` instead of
+        `build_config.with_build_python_bindings(True)`
+
+        :return: current instance of BuildConfiguration
+        """
+        self.__build_python_bindings = build_python_bindings
+        return self
+
     def with_ros(self, ros_config: RosConfiguration):
         """
         Activates the generation of a ROS package. The caller must provide an