From 91301590141d9f8ffda1a5ed77c7ad3a6b825641 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Wed, 16 Sep 2020 21:38:53 +0100 Subject: [PATCH 01/10] docs: updated python examples --- docs/example_estimation_py.md | 4 ++++ docs/example_rosenbrock_py.md | 14 +++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/example_estimation_py.md b/docs/example_estimation_py.md index db16d6e0..a16bcac8 100644 --- a/docs/example_estimation_py.md +++ b/docs/example_estimation_py.md @@ -219,7 +219,11 @@ meta = og.config.OptimizerMeta() \ .with_optimizer_name("estimator") builder = og.builder.OpEnOptimizerBuilder(problem, meta, build_config) builder.build() +``` + +The generated solver can be consumed over its TCP interface: +```python # Use TCP server # ------------------------------------ mng = og.tcp.OptimizerTcpManager('python_test_build/estimator') diff --git a/docs/example_rosenbrock_py.md b/docs/example_rosenbrock_py.md index b23b4ec3..25112cec 100644 --- a/docs/example_rosenbrock_py.md +++ b/docs/example_rosenbrock_py.md @@ -42,11 +42,11 @@ problem = og.builder.Problem(u, p, phi) \ .with_penalty_constraints(c) \ .with_constraints(bounds) build_config = og.config.BuildConfiguration() \ - .with_build_directory("python_test_build") \ + .with_build_directory("my_optimizers") \ .with_build_mode("debug") \ .with_tcp_interface_config() meta = og.config.OptimizerMeta() \ - .with_optimizer_name("tcp_enabled_optimizer") + .with_optimizer_name("rosenbrock") solver_config = og.config.SolverConfiguration() \ .with_tolerance(1e-5) \ .with_delta_tolerance(1e-4) \ @@ -58,12 +58,16 @@ builder.build() # Use TCP server # ------------------------------------ -mng = og.tcp.OptimizerTcpManager('python_test_build/tcp_enabled_optimizer') +mng = og.tcp.OptimizerTcpManager('my_optimizers/rosenbrock') mng.start() mng.ping() -solution = mng.call([1.0, 50.0]) -print(solution) +server_response = mng.call([1.0, 50.0]) + +if server_response.is_ok(): + solution = server_response.get() + u_star = solution.solution + status = solution.exit_status mng.kill() ``` From 7287ccbba62d869b84b5ab0625ebbe2b77a90aa3 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Thu, 15 Oct 2020 02:56:40 +0100 Subject: [PATCH 02/10] empty commit (starting new branch) From 30c7201a708da97222477ca38bb4d673d0684811 Mon Sep 17 00:00:00 2001 From: David Rusu Date: Fri, 16 Oct 2020 16:56:04 -0400 Subject: [PATCH 03/10] Integrate pyo3 generated bindings --- .../opengen/builder/optimizer_builder.py | 60 ++++++++ open-codegen/opengen/config/build_config.py | 20 +++ open-codegen/opengen/main.py | 51 ++++--- .../templates/python/python_bindings.rs | 142 ++++++++++++++++++ .../python/python_bindings_cargo.toml | 40 +++++ 5 files changed, 291 insertions(+), 22 deletions(-) create mode 100644 open-codegen/opengen/templates/python/python_bindings.rs create mode 100644 open-codegen/opengen/templates/python/python_bindings_cargo.toml diff --git a/open-codegen/opengen/builder/optimizer_builder.py b/open-codegen/opengen/builder/optimizer_builder.py index 0d310926..15fb431c 100644 --- a/open-codegen/opengen/builder/optimizer_builder.py +++ b/open-codegen/opengen/builder/optimizer_builder.py @@ -18,6 +18,7 @@ _AUTOGEN_GRAD_FNAME = 'auto_casadi_grad.c' _AUTOGEN_PNLT_CONSTRAINTS_FNAME = 'auto_casadi_mapping_f2.c' _AUTOGEN_ALM_MAPPING_F1_FNAME = 'auto_casadi_mapping_f1.c' +_PYTHON_BINDINGS_PREFIX = 'python_bindings_' _TCP_IFACE_PREFIX = 'tcp_iface_' _ICASADI_PREFIX = 'icasadi_' _ROS_PREFIX = 'ros_node_' @@ -421,6 +422,27 @@ def __build_optimizer(self): if process_completion != 0: raise Exception('Rust build failed') + def __build_python_bindings(self): + self.__logger.info("Building Python bindings") + target_dir = os.path.abspath(self.__target_dir()) + optimizer_name = self.__meta.optimizer_name + python_bindings_dir_name = _PYTHON_BINDINGS_PREFIX + optimizer_name + python_bindings_dir = os.path.join(target_dir, python_bindings_dir_name) + command = self.__make_build_command() + p = subprocess.Popen(command, cwd=python_bindings_dir) + process_completion = p.wait() + if process_completion != 0: + raise Exception('Rust build of Python bindings failed') + + if self.__build_config.build_mode.lower() == 'release': + build_dir = os.path.join(python_bindings_dir, 'target', 'release') + else: + build_dir = os.path.join(python_bindings_dir, 'target', 'debug') + + generated_bindings = os.path.join(build_dir, f"lib{optimizer_name}.so") + assert os.path.isfile(generated_bindings) + shutil.copyfile(generated_bindings, os.path.join(os.getcwd(), f"{optimizer_name}.so")) + def __build_tcp_iface(self): self.__logger.info("Building the TCP interface") target_dir = os.path.abspath(self.__target_dir()) @@ -439,6 +461,37 @@ def __initialize(self): def __check_user_provided_parameters(self): self.__logger.info("Checking user parameters") + def __generate_code_python_bindings(self): + self.__logger.info("Generating code for Python bindings") + target_dir = self.__target_dir() + python_bindings_dir_name = _PYTHON_BINDINGS_PREFIX + self.__meta.optimizer_name + python_bindings_dir = os.path.join(target_dir, python_bindings_dir_name) + python_bindings_source_dir = os.path.join(python_bindings_dir, "src") + self.__logger.info(f"Generating code for Python bindings in {python_bindings_dir}") + + # make python_bindings/ and python_bindings/src + make_dir_if_not_exists(python_bindings_dir) + make_dir_if_not_exists(python_bindings_source_dir) + + # generate tcp_server.rs for python_bindings + python_rs_template = OpEnOptimizerBuilder.__get_template('python_bindings.rs', 'python') + python_rs_output_template = python_rs_template.render( + meta=self.__meta, + timestamp_created=datetime.datetime.now()) + target_python_rs_path = os.path.join(python_bindings_source_dir, "lib.rs") + with open(target_python_rs_path, "w") as fh: + fh.write(python_rs_output_template) + + # generate Cargo.toml for python_bindings + python_rs_template = OpEnOptimizerBuilder.__get_template('python_bindings_cargo.toml', 'python') + python_rs_output_template = python_rs_template.render( + meta=self.__meta, + build_config=self.__build_config, + timestamp_created=datetime.datetime.now()) + target_python_rs_path = os.path.join(python_bindings_dir, "Cargo.toml") + with open(target_python_rs_path, "w") as fh: + fh.write(python_rs_output_template) + def __generate_code_tcp_interface(self): self.__logger.info("Generating code for TCP/IP interface (tcp_iface/src/main.rs)") self.__logger.info("TCP server will bind at %s:%d", @@ -582,9 +635,16 @@ def build(self): self.__build_tcp_iface() if self.__build_config.build_c_bindings: + self.__logger.info("Generating C/C++ bindings") self.__generate_c_bindings_example() self.__generate_c_bindings_makefile() + if self.__build_config.build_python_bindings: + self.__logger.info("Generating Python bindings") + self.__generate_code_python_bindings() + if not self.__generate_not_build: + self.__build_python_bindings() + if self.__build_config.ros_config is not None: ros_builder = RosBuilder( self.__meta, diff --git a/open-codegen/opengen/config/build_config.py b/open-codegen/opengen/config/build_config.py index 29bd1662..b3ebfeda 100644 --- a/open-codegen/opengen/config/build_config.py +++ b/open-codegen/opengen/config/build_config.py @@ -36,6 +36,7 @@ def __init__(self, build_dir="."): self.__build_dir = build_dir self.__open_version = None self.__build_c_bindings = False + self.__build_python_bindings = False self.__ros_config = None self.__tcp_interface_config = None self.__local_path = None @@ -103,6 +104,10 @@ def local_path(self): def build_c_bindings(self): return self.__build_c_bindings + @property + def build_python_bindings(self): + return self.__build_python_bindings + @property def tcp_interface_config(self): return self.__tcp_interface_config @@ -204,6 +209,21 @@ def with_build_c_bindings(self, build_c_bindings=True): self.__build_c_bindings = build_c_bindings return self + def with_build_python_bindings(self, build_python_bindings=True): + """ + If activated, OpEn will generate python bindings for the + auto-generated solver + + :param build_python_bindings: whether to build python bindings for + auto-generated solver; default: `True`, i.e., it suffices + to call `build_config.with_build_python_bindings()` instead of + `build_config.with_build_python_bindings(True)` + + :return: current instance of BuildConfiguration + """ + self.__build_python_bindings = build_python_bindings + return self + def with_ros(self, ros_config: RosConfiguration): """ Activates the generation of a ROS package. The caller must provide an diff --git a/open-codegen/opengen/main.py b/open-codegen/opengen/main.py index 88e8de65..513228b5 100644 --- a/open-codegen/opengen/main.py +++ b/open-codegen/opengen/main.py @@ -22,7 +22,8 @@ build_config = og.config.BuildConfiguration() \ .with_build_directory('my_optimizers') \ .with_build_mode(og.config.BuildConfiguration.DEBUG_MODE) \ - .with_tcp_interface_config() + .with_build_python_bindings() + # .with_tcp_interface_config() builder = og.builder.OpEnOptimizerBuilder(problem, meta, @@ -30,24 +31,30 @@ og.config.SolverConfiguration()) builder.build() -all_managers = [] -for i in range(10): - all_managers += [og.tcp.OptimizerTcpManager( - optimizer_path='my_optimizers/halfspace_optimizer', - ip='0.0.0.0', - port=3311+i)] - -for m in all_managers: - m.start() - -time.sleep(4) - -for m in all_managers: - print(m.details) - resp = m.call(p=[1., 2.]) - print(resp.get().solution) - -# mng.kill() -time.sleep(6) -for m in all_managers: - m.kill() +import halfspace_optimizer + +solver = halfspace_optimizer.build_solver() +result = solver.run([1., 2.]) +print(result.solution) + +# all_managers = [] +# for i in range(10): +# all_managers += [og.tcp.OptimizerTcpManager( +# optimizer_path='my_optimizers/halfspace_optimizer', +# ip='0.0.0.0', +# port=3311+i)] +# +# for m in all_managers: +# m.start() +# +# time.sleep(4) +# +# for m in all_managers: +# print(m.details) +# resp = m.call(p=[1., 2.]) +# print(resp.get().solution) +# +# # mng.kill() +# time.sleep(6) +# for m in all_managers: +# m.kill() diff --git a/open-codegen/opengen/templates/python/python_bindings.rs b/open-codegen/opengen/templates/python/python_bindings.rs new file mode 100644 index 00000000..6fba7974 --- /dev/null +++ b/open-codegen/opengen/templates/python/python_bindings.rs @@ -0,0 +1,142 @@ +/// +/// Auto-generated python bindings for optimizer: {{ meta.optimizer_name }} +/// Generated at: {{timestamp_created}} +/// +use optimization_engine::alm::*; + +use pyo3::prelude::*; +use pyo3::wrap_pyfunction; + +use {{ meta.optimizer_name }}::*; + +#[pymodule] +fn {{ meta.optimizer_name }}(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(build_solver, m)?)?; + m.add_class::()?; + m.add_class::()?; + Ok(()) +} + +#[pyfunction] +fn build_solver() -> PyResult { + let cache = initialize_solver(); + Ok(Solver { cache }) +} + +/// Solution and solution status of optimizer +#[pyclass] +struct OptimizerSolution { + #[pyo3(get)] + exit_status: String, + #[pyo3(get)] + num_outer_iterations: usize, + #[pyo3(get)] + num_inner_iterations: usize, + #[pyo3(get)] + last_problem_norm_fpr: f64, + #[pyo3(get)] + delta_y_norm_over_c: f64, + #[pyo3(get)] + f2_norm: f64, + #[pyo3(get)] + solve_time_ms: f64, + #[pyo3(get)] + penalty: f64, + #[pyo3(get)] + solution: Vec, + #[pyo3(get)] + lagrange_multipliers: Vec, + #[pyo3(get)] + cost: f64, +} + +#[pyclass] +struct Solver { + cache: AlmCache, +} + +#[pymethods] +impl Solver { + /// Run solver + fn run( + &mut self, + parameter: Vec, + initial_guess: Option>, + initial_lagrange_multipliers: Option>, + initial_penalty: Option, + ) -> PyResult> { + let mut u = [0.0; {{meta.optimizer_name|upper}}_NUM_DECISION_VARIABLES]; + + // ---------------------------------------------------- + // Set initial value + // ---------------------------------------------------- + if let Some(u0) = initial_guess { + if u0.len() != {{meta.optimizer_name|upper}}_NUM_DECISION_VARIABLES { + println!( + "1600 -> Initial guess has incompatible dimensions: {} != {}", + u0.len(), + {{meta.optimizer_name|upper}}_NUM_DECISION_VARIABLES + ); + return Ok(None); + } + u.copy_from_slice(&u0); + } + + // ---------------------------------------------------- + // Check lagrange multipliers + // ---------------------------------------------------- + if let Some(y0) = &initial_lagrange_multipliers { + if y0.len() != {{meta.optimizer_name|upper}}_N1 { + println!( + "1700 -> wrong dimension of Langrange multipliers: {} != {}", + y0.len(), + {{meta.optimizer_name|upper}}_N1 + ); + return Ok(None); + } + } + + // ---------------------------------------------------- + // Check parameter + // ---------------------------------------------------- + if parameter.len() != {{meta.optimizer_name|upper}}_NUM_PARAMETERS { + println!( + "3003 -> wrong number of parameters: {} != {}", + parameter.len(), + {{meta.optimizer_name|upper}}_NUM_PARAMETERS + ); + return Ok(None); + } + + // ---------------------------------------------------- + // Run solver + // ---------------------------------------------------- + let solver_status = solve( + ¶meter, + &mut self.cache, + &mut u, + &initial_lagrange_multipliers, + &initial_penalty, + ); + + match solver_status { + Ok(status) => Ok(Some(OptimizerSolution { + exit_status: format!("{:?}", status.exit_status()), + num_outer_iterations: status.num_outer_iterations(), + num_inner_iterations: status.num_inner_iterations(), + last_problem_norm_fpr: status.last_problem_norm_fpr(), + delta_y_norm_over_c: status.delta_y_norm_over_c(), + f2_norm: status.f2_norm(), + penalty: status.penalty(), + lagrange_multipliers: status.lagrange_multipliers().clone().unwrap_or_default(), + solve_time_ms: (status.solve_time().as_nanos() as f64) / 1e6, + solution: u.to_vec(), + cost: status.cost(), + })), + Err(_) => { + println!("2000 -> Problem solution failed (solver error)"); + Ok(None) + } + } + } +} diff --git a/open-codegen/opengen/templates/python/python_bindings_cargo.toml b/open-codegen/opengen/templates/python/python_bindings_cargo.toml new file mode 100644 index 00000000..7a90aef4 --- /dev/null +++ b/open-codegen/opengen/templates/python/python_bindings_cargo.toml @@ -0,0 +1,40 @@ + +# ----------------------------------------------------------------- +# +# Autogenerated Cargo.toml configuration file for python bindings +# This file was generated by OptimizationEngine +# +# Python bindings for {{meta.optimizer_name}} v{{meta.version}} +# +# See https://alphaville.github.io/optimization-engine/ +# +# Generated at: {{timestamp_created}} +# +# ----------------------------------------------------------------- + +[package] +name = "python_bindings_{{meta.optimizer_name}}" +version = "0.0.1" +license = "MIT" +authors = ["John Smith"] +edition = "2018" +publish=false + + +[lib] +name = "{{meta.optimizer_name}}" +crate-type = ["cdylib"] + +[dependencies] +{% if build_config.local_path is not none -%} +optimization_engine = {path = "{{build_config.local_path}}"} +{% else -%} +optimization_engine = "{{build_config.open_version or '*'}}" +{% endif %} + +{{meta.optimizer_name}} = { path = "../" } + + +[dependencies.pyo3] +version = "0.12.3" +features = ["extension-module"] From e163c1b6388142b676aebff7a845149f23faa0d1 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Sat, 17 Oct 2020 13:43:28 +0100 Subject: [PATCH 04/10] deprecation of enable_tcp_interface --- open-codegen/CHANGELOG.md | 5 +++++ open-codegen/opengen/builder/optimizer_builder.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/open-codegen/CHANGELOG.md b/open-codegen/CHANGELOG.md index c2c0c736..843f031a 100644 --- a/open-codegen/CHANGELOG.md +++ b/open-codegen/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/). Note: This is the Changelog file of `opengen` - the Python interface of OpEn +## [0.6.3] - 2020-10-17 + +# Changed + +* `enable_tcp_interface` previously gave a `DeprecationWarning`; now it raises it. In a future version, it will be removed. ## [0.6.3] - 2020-10-06 diff --git a/open-codegen/opengen/builder/optimizer_builder.py b/open-codegen/opengen/builder/optimizer_builder.py index 0d310926..77fbdf45 100644 --- a/open-codegen/opengen/builder/optimizer_builder.py +++ b/open-codegen/opengen/builder/optimizer_builder.py @@ -518,9 +518,9 @@ def __generate_yaml_data_file(self): def enable_tcp_interface(self, tcp_server_configuration=og_cfg.TcpServerConfiguration()): - warnings.warn("deprecated (use BuildConfiguration.with_tcp_interface_config instead)", - DeprecationWarning) - self.__build_config.with_tcp_interface_config(tcp_server_configuration) + # This method should not be used! + raise DeprecationWarning( + "deprecated (use BuildConfiguration.with_tcp_interface_config instead)") def __generate_c_bindings_example(self): self.__logger.info("Generating example_optimizer.c (C bindings example for your convenience)") From 1d573b9c39dfa8fd973ac26279324e3680dd29c5 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Sun, 18 Oct 2020 21:17:08 +0100 Subject: [PATCH 05/10] renamed build_solver to solver introduced test in test.py --- .../templates/python/python_bindings.rs | 4 +-- open-codegen/opengen/test/test.py | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/open-codegen/opengen/templates/python/python_bindings.rs b/open-codegen/opengen/templates/python/python_bindings.rs index 6fba7974..0281aebb 100644 --- a/open-codegen/opengen/templates/python/python_bindings.rs +++ b/open-codegen/opengen/templates/python/python_bindings.rs @@ -11,14 +11,14 @@ use {{ meta.optimizer_name }}::*; #[pymodule] fn {{ meta.optimizer_name }}(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(build_solver, m)?)?; + m.add_function(wrap_pyfunction!(solver, m)?)?; m.add_class::()?; m.add_class::()?; Ok(()) } #[pyfunction] -fn build_solver() -> PyResult { +fn solver() -> PyResult { let cache = initialize_solver(); Ok(Solver { cache }) } diff --git a/open-codegen/opengen/test/test.py b/open-codegen/opengen/test/test.py index f44e6070..ee10a17b 100644 --- a/open-codegen/opengen/test/test.py +++ b/open-codegen/opengen/test/test.py @@ -28,6 +28,27 @@ def solverConfig(cls): .with_cbfgs_parameters(1.5, 1e-10, 1e-12) return solver_config + @classmethod + def setUpPythonBindings(cls): + u = cs.MX.sym("u", 5) # decision variable (nu = 5) + p = cs.MX.sym("p", 2) # parameter (np = 2) + phi = og.functions.rosenbrock(u, p) # cost function + bounds = og.constraints.Ball2(None, 1.5) # ball centered at origin + meta = og.config.OptimizerMeta() \ + .with_optimizer_name("python_bindings") + problem = og.builder.Problem(u, p, phi) \ + .with_constraints(bounds) + build_config = og.config.BuildConfiguration() \ + .with_open_version(RustBuildTestCase.OPEN_RUSTLIB_VERSION) \ + .with_build_directory(RustBuildTestCase.TEST_DIR) \ + .with_build_mode(og.config.BuildConfiguration.DEBUG_MODE)\ + .with_build_python_bindings() + og.builder.OpEnOptimizerBuilder(problem, + metadata=meta, + build_configuration=build_config, + solver_configuration=cls.solverConfig()) \ + .build() + @classmethod def setUpOnlyF1(cls): u = cs.MX.sym("u", 5) # decision variable (nu = 5) @@ -186,12 +207,25 @@ def setUpHalfspace(cls): @classmethod def setUpClass(cls): + cls.setUpPythonBindings() cls.setUpRosPackageGeneration() cls.setUpOnlyF1() cls.setUpOnlyF2() cls.setUpPlain() cls.setUpOnlyParametricF2() cls.setUpHalfspace() + pass + + def test_python_bindings(self): + import sys + import os + sys.path.insert(1, os.getcwd()) # include the CWD into the path! + + import python_bindings # import python_bindings.so + solver = python_bindings.solver() + result = solver.run([1., 2.]) # returns object of type OptimizerSolution + self.assertIsNotNone(result.solution) + def test_rectangle_empty(self): xmin = [-1, 2] From 387ccb315b10cb8a0dd9e5ba309933b451c77ba9 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Sun, 18 Oct 2020 23:04:38 +0100 Subject: [PATCH 06/10] adding target dir to path library file (.so) is stored in target dir --- open-codegen/opengen/builder/optimizer_builder.py | 15 +++++++++++---- open-codegen/opengen/test/test.py | 1 - 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/open-codegen/opengen/builder/optimizer_builder.py b/open-codegen/opengen/builder/optimizer_builder.py index 15fb431c..8e6af296 100644 --- a/open-codegen/opengen/builder/optimizer_builder.py +++ b/open-codegen/opengen/builder/optimizer_builder.py @@ -11,6 +11,7 @@ import jinja2 import logging import pkg_resources +import sys from .ros_builder import RosBuilder @@ -242,7 +243,8 @@ def __generate_cargo_toml(self): meta=self.__meta, build_config=self.__build_config, activate_tcp_server=self.__build_config.tcp_interface_config is not None, - activate_clib_generation=self.__build_config.build_c_bindings) + activate_clib_generation=self.__build_config.build_c_bindings, + timestamp_created=datetime.datetime.now()) cargo_toml_path = os.path.abspath(os.path.join(target_dir, "Cargo.toml")) with open(cargo_toml_path, "w") as fh: fh.write(cargo_output_template) @@ -439,9 +441,14 @@ def __build_python_bindings(self): else: build_dir = os.path.join(python_bindings_dir, 'target', 'debug') - generated_bindings = os.path.join(build_dir, f"lib{optimizer_name}.so") - assert os.path.isfile(generated_bindings) - shutil.copyfile(generated_bindings, os.path.join(os.getcwd(), f"{optimizer_name}.so")) + pltform_extension_dict = {'linux': ('.so', '.so'), + 'darwin': ('.dylib', '.so'), + 'win32': ('.dll', '.pyd')} + (original_lib_extension, target_lib_extension) = pltform_extension_dict[sys.platform] + generated_bindings = os.path.join(build_dir, f"lib{optimizer_name}{original_lib_extension}") + target_bindings = os.path.join(target_dir, f"{optimizer_name}{target_lib_extension}") + shutil.copyfile(generated_bindings, target_bindings) + sys.path.insert(1, target_dir) # add target_dir to path def __build_tcp_iface(self): self.__logger.info("Building the TCP interface") diff --git a/open-codegen/opengen/test/test.py b/open-codegen/opengen/test/test.py index ee10a17b..951c5c63 100644 --- a/open-codegen/opengen/test/test.py +++ b/open-codegen/opengen/test/test.py @@ -214,7 +214,6 @@ def setUpClass(cls): cls.setUpPlain() cls.setUpOnlyParametricF2() cls.setUpHalfspace() - pass def test_python_bindings(self): import sys From 66c16d5aed149b3176daf0064704a095dfb6352b Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Sun, 18 Oct 2020 23:29:45 +0100 Subject: [PATCH 07/10] including .cargo/config into python bindings dir using tojson in Jinja templates for list of authors fixes bug with encoding of quotes --- open-codegen/opengen/builder/optimizer_builder.py | 6 ++++++ open-codegen/opengen/templates/optimizer_cargo.toml | 2 +- open-codegen/opengen/templates/python/cargo_config | 5 +++++ .../opengen/templates/python/python_bindings_cargo.toml | 7 +++---- 4 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 open-codegen/opengen/templates/python/cargo_config diff --git a/open-codegen/opengen/builder/optimizer_builder.py b/open-codegen/opengen/builder/optimizer_builder.py index 8e6af296..3f1b03e8 100644 --- a/open-codegen/opengen/builder/optimizer_builder.py +++ b/open-codegen/opengen/builder/optimizer_builder.py @@ -499,6 +499,12 @@ def __generate_code_python_bindings(self): with open(target_python_rs_path, "w") as fh: fh.write(python_rs_output_template) + # move cargo_config into .cargo/config + target_cargo_config_dir = os.path.join(python_bindings_dir, '.cargo') + make_dir_if_not_exists(target_cargo_config_dir) + cargo_config_file = os.path.join(og_dfn.templates_dir(), 'python', 'cargo_config') + shutil.copy(cargo_config_file, os.path.join(target_cargo_config_dir, 'config')) + def __generate_code_tcp_interface(self): self.__logger.info("Generating code for TCP/IP interface (tcp_iface/src/main.rs)") self.__logger.info("TCP server will bind at %s:%d", diff --git a/open-codegen/opengen/templates/optimizer_cargo.toml b/open-codegen/opengen/templates/optimizer_cargo.toml index f4e48976..a92fc34a 100644 --- a/open-codegen/opengen/templates/optimizer_cargo.toml +++ b/open-codegen/opengen/templates/optimizer_cargo.toml @@ -12,7 +12,7 @@ name = "{{meta.optimizer_name}}" version = "{{meta.version}}" license = "{{meta.licence}}" -authors = ["{{meta.authors|join('", "')}}"] +authors = {{meta.authors | tojson }} edition = "2018" publish=false diff --git a/open-codegen/opengen/templates/python/cargo_config b/open-codegen/opengen/templates/python/cargo_config new file mode 100644 index 00000000..15d5d329 --- /dev/null +++ b/open-codegen/opengen/templates/python/cargo_config @@ -0,0 +1,5 @@ +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] diff --git a/open-codegen/opengen/templates/python/python_bindings_cargo.toml b/open-codegen/opengen/templates/python/python_bindings_cargo.toml index 7a90aef4..7c7cdd4a 100644 --- a/open-codegen/opengen/templates/python/python_bindings_cargo.toml +++ b/open-codegen/opengen/templates/python/python_bindings_cargo.toml @@ -1,4 +1,3 @@ - # ----------------------------------------------------------------- # # Autogenerated Cargo.toml configuration file for python bindings @@ -14,9 +13,9 @@ [package] name = "python_bindings_{{meta.optimizer_name}}" -version = "0.0.1" -license = "MIT" -authors = ["John Smith"] +version = "{{meta.version}}" +license = "{{meta.licence}}" +authors = {{meta.authors | tojson }} edition = "2018" publish=false From 48aaf3fbe10ea913445842b4a6fd0a8f85b2f46b Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Mon, 19 Oct 2020 19:46:54 +0100 Subject: [PATCH 08/10] renamed delta_y_norm_over_c to f1_infeasibility not adding target dir to path (the user has to do that) additional log messages in builder documentation on the website --- docs/python-bindings.md | 118 ++++++++++++++++++ docs/python-interface.md | 1 + open-codegen/CHANGELOG.md | 11 ++ open-codegen/VERSION | 2 +- .../opengen/builder/optimizer_builder.py | 6 +- .../templates/python/python_bindings.rs | 14 ++- website/sidebars.json | 1 + 7 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 docs/python-bindings.md diff --git a/docs/python-bindings.md b/docs/python-bindings.md new file mode 100644 index 00000000..9c88422d --- /dev/null +++ b/docs/python-bindings.md @@ -0,0 +1,118 @@ +--- +id: python-bindings +title: Direct interface +description: How to access the auto-generated optimizer from Python +--- + + + + +
+Info: The functionality presented here was introduced in opengen version 0.6.4. +The API is still young and is likely to change in versions 0.7.X. +
+ + +As we [discussed previously](python-interface#about) there are various ways +you can use the auto-generated optimizer. You can use the generated Rust code, +access it over a TCP socket, and so on. In this section we will focus on +how to access it *directly*. + +The idea is that OpEn can generate a Python module that you can `import`. + +## Generate a Python module for your optimizer +Consider the following [parametric optimization problem](example_rosenbrock_py): +
+\[ + \begin{align} + \operatorname*{Minimize}_{\|u\|\leq r}& \sum_{i=1}^{n_u - 1} b (u_{i+1} - u_{i}^2)^2 + (a-u_i)^2 + \\ + \text{subject to: }& 1.5 u_1 - u_2 = 0 + \\ + &u_3 - u_4 + 0.1 \leq 0 + \end{align} +\]
+ +```python +import opengen as og +import casadi.casadi as cs + +u = cs.SX.sym("u", 5) +p = cs.SX.sym("p", 2) +phi = og.functions.rosenbrock(u, p) +c = cs.vertcat(1.5 * u[0] - u[1], + cs.fmax(0.0, u[2] - u[3] + 0.1)) +bounds = og.constraints.Ball2(radius=1.5) +problem = og.builder.Problem(u, p, phi) \ + .with_penalty_constraints(c) \ + .with_constraints(bounds) +build_config = og.config.BuildConfiguration() \ + .with_build_directory("my_optimizers") \ + .with_build_mode("debug") \ + .with_build_python_bindings() +meta = og.config.OptimizerMeta() \ + .with_optimizer_name("rosenbrock") +builder = og.builder.OpEnOptimizerBuilder(problem, meta, + build_config) +builder.build() +``` + +Note that we have used `with_build_python_bindings()`. + +This will allow us to **import the auto-generated optimizer** as a Python +module! + + + +## Use the generated module + +The above code generates an optimizer which is stored in `my_optimizers/rosenbrock`. +In that directory you can find a file called `rosenbrock.so` (or `rosenbrock.pyd` on Windows). +This can be loaded as an autogenerated Python module. +However, mind that this directory is most likely not in your Python path, +so you will have to add it before you can import the optimizer. +This can be done very easily: + +```python +sys.path.insert(1, './my_optimizers/rosenbrock') +import rosenbrock +``` + +Then you will be able to use it as follows: + +```python +solver = rosenbrock.solver() +result = solver.run(p=[20., 1.]) +u_star = result.solution +``` + +In the first line, `solver = rosenbrock.solver()`, we obtain an instance of +`Solver`, which can be used to solve parametric optimization problems. +In the second line, `result = solver.run(p=[20., 1.])`, we call the solver +with parameter $p=(20, 1)$. Method `run` accepts another three optional +arguments, namely: + +- `initial_guess` (can be either a list or a numpy array), +- `initial_lagrange_multipliers`, and +- `initial_penalty` + +The solver returns an object of type `OptimizerSolution` with the following +properties: + + +| Property | Explanation | +|--------------------------|---------------------------------------------| +| `exit_status` | Exit status; can be (i) `Converged` or (ii) `NotConvergedIterations`, if the maximum number of iterations was reached, therefore, the algorithm did not converge up to the specified tolerances, or (iii) `NotConvergedOutOfTime`, if the solver did not have enough time to converge | +| `num_outer_iterations` | Number of outer iterations | +| `num_inner_iterations` | Total number of inner iterations (for all inner problems) | +| `last_problem_norm_fpr` | Norm of the fixed-point residual of the last inner problem; this is a measure of the solution quality of the inner problem | +| `f1_infeasibility` | Euclidean norm of $c^{-1}(y^+-y)$, which is equal to the distance between $F_1(u, p)$ and $C$ at the solution | +| `f2_norm` | Euclidean norm of $F_2(u, p)$ at the solution| +| `solve_time_ms` | Total execution time in milliseconds | +| `penalty` | Last value of the penalty parameter | +| `solution` | Solution | +| `cost` | Cost function at solution | +| `lagrange_multipliers` | Vector of Lagrange multipliers (if $n_1 > 0$) or an empty vector, otherwise | + +These are the same properties as those of `opengen.tcp.SolverStatus`. + diff --git a/docs/python-interface.md b/docs/python-interface.md index 2188462a..6434b77a 100644 --- a/docs/python-interface.md +++ b/docs/python-interface.md @@ -26,6 +26,7 @@ of the following ways: - Directly in **Rust** (you can include it in you Rust project as a dependency) - Over a **TCP socket** based on JSON (which can be accessed easily from any programming language) - In **Python** (using the TCP/IP interface in the background) +- In [**Python**](python-bindings) by accessing the Rust-based auto-generated optimizer directly - In [**C** or **C++**](python-c) using auto-generated C/C++ bindings (header files and static or shared libraries) - In [**ROS**](python-ros) using auto-generated ROS packages diff --git a/open-codegen/CHANGELOG.md b/open-codegen/CHANGELOG.md index c2c0c736..8463c85b 100644 --- a/open-codegen/CHANGELOG.md +++ b/open-codegen/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/). Note: This is the Changelog file of `opengen` - the Python interface of OpEn +## [0.6.4] - 2020-10-21 + +### Added + +* Accessing Rust from Python directly using PyO3 + +### Fixed + +* List of authors in `Cargo.toml` is generated properly + ## [0.6.3] - 2020-10-06 @@ -77,6 +87,7 @@ Note: This is the Changelog file of `opengen` - the Python interface of OpEn * Project-specific `tcp_iface` TCP interface * Fixed `lbfgs` typo +[0.6.2]: https://github.com/alphaville/optimization-engine/compare/opengen-0.6.3...opengen-0.6.4 [0.6.2]: https://github.com/alphaville/optimization-engine/compare/opengen-0.6.2...opengen-0.6.3 [0.6.2]: https://github.com/alphaville/optimization-engine/compare/opengen-0.6.1...opengen-0.6.2 [0.6.1]: https://github.com/alphaville/optimization-engine/compare/opengen-v0.6.0...opengen-0.6.1 diff --git a/open-codegen/VERSION b/open-codegen/VERSION index 844f6a91..d2b13eb6 100644 --- a/open-codegen/VERSION +++ b/open-codegen/VERSION @@ -1 +1 @@ -0.6.3 +0.6.4 diff --git a/open-codegen/opengen/builder/optimizer_builder.py b/open-codegen/opengen/builder/optimizer_builder.py index 3f1b03e8..ab7a37bf 100644 --- a/open-codegen/opengen/builder/optimizer_builder.py +++ b/open-codegen/opengen/builder/optimizer_builder.py @@ -448,7 +448,11 @@ def __build_python_bindings(self): generated_bindings = os.path.join(build_dir, f"lib{optimizer_name}{original_lib_extension}") target_bindings = os.path.join(target_dir, f"{optimizer_name}{target_lib_extension}") shutil.copyfile(generated_bindings, target_bindings) - sys.path.insert(1, target_dir) # add target_dir to path + self.__logger.info(f"To use the Python bindings do:\n\n" + f" import sys\n" + f" sys.path.insert(1, \"{target_dir}\")\n" + f" import {optimizer_name}\n" + f" solver = {optimizer_name}.solver()") def __build_tcp_iface(self): self.__logger.info("Building the TCP interface") diff --git a/open-codegen/opengen/templates/python/python_bindings.rs b/open-codegen/opengen/templates/python/python_bindings.rs index 0281aebb..4bedda21 100644 --- a/open-codegen/opengen/templates/python/python_bindings.rs +++ b/open-codegen/opengen/templates/python/python_bindings.rs @@ -35,7 +35,7 @@ struct OptimizerSolution { #[pyo3(get)] last_problem_norm_fpr: f64, #[pyo3(get)] - delta_y_norm_over_c: f64, + f1_infeasibility: f64, #[pyo3(get)] f2_norm: f64, #[pyo3(get)] @@ -58,9 +58,11 @@ struct Solver { #[pymethods] impl Solver { /// Run solver + /// + #[text_signature = "($self, p, initial_guess, initial_y, initial_penalty)"] fn run( &mut self, - parameter: Vec, + p: Vec, initial_guess: Option>, initial_lagrange_multipliers: Option>, initial_penalty: Option, @@ -99,10 +101,10 @@ impl Solver { // ---------------------------------------------------- // Check parameter // ---------------------------------------------------- - if parameter.len() != {{meta.optimizer_name|upper}}_NUM_PARAMETERS { + if p.len() != {{meta.optimizer_name|upper}}_NUM_PARAMETERS { println!( "3003 -> wrong number of parameters: {} != {}", - parameter.len(), + p.len(), {{meta.optimizer_name|upper}}_NUM_PARAMETERS ); return Ok(None); @@ -112,7 +114,7 @@ impl Solver { // Run solver // ---------------------------------------------------- let solver_status = solve( - ¶meter, + &p, &mut self.cache, &mut u, &initial_lagrange_multipliers, @@ -125,7 +127,7 @@ impl Solver { num_outer_iterations: status.num_outer_iterations(), num_inner_iterations: status.num_inner_iterations(), last_problem_norm_fpr: status.last_problem_norm_fpr(), - delta_y_norm_over_c: status.delta_y_norm_over_c(), + f1_infeasibility: status.delta_y_norm_over_c(), f2_norm: status.f2_norm(), penalty: status.penalty(), lagrange_multipliers: status.lagrange_multipliers().clone().unwrap_or_default(), diff --git a/website/sidebars.json b/website/sidebars.json index ada12f84..2008c0fa 100644 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -8,6 +8,7 @@ "python-interface", "python-advanced", "python-c", + "python-bindings", "python-tcp-ip", "python-ros", "python-examples" From d97ff9c6e5392d3d8c5983400ec1baa7052f2109 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Mon, 19 Oct 2020 20:14:47 +0100 Subject: [PATCH 09/10] fixed test.py --- open-codegen/opengen/test/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/open-codegen/opengen/test/test.py b/open-codegen/opengen/test/test.py index 951c5c63..b1a598a0 100644 --- a/open-codegen/opengen/test/test.py +++ b/open-codegen/opengen/test/test.py @@ -218,9 +218,11 @@ def setUpClass(cls): def test_python_bindings(self): import sys import os - sys.path.insert(1, os.getcwd()) # include the CWD into the path! + # include the target directory into the path... + sys.path.insert(1, os.path.join(RustBuildTestCase.TEST_DIR, "python_bindings")) import python_bindings # import python_bindings.so + solver = python_bindings.solver() result = solver.run([1., 2.]) # returns object of type OptimizerSolution self.assertIsNotNone(result.solution) From aa129818ebb444af5689e72d907d0d1fca86fd74 Mon Sep 17 00:00:00 2001 From: Pantelis Sopasakis Date: Tue, 20 Oct 2020 12:39:37 +0100 Subject: [PATCH 10/10] docs: import optimizer with dynamic name main.py: using pyo3-generated module --- docs/python-bindings.md | 15 ++++++++++++ open-codegen/opengen/main.py | 47 +++++++++--------------------------- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/docs/python-bindings.md b/docs/python-bindings.md index 9c88422d..1d0e5044 100644 --- a/docs/python-bindings.md +++ b/docs/python-bindings.md @@ -116,3 +116,18 @@ properties: These are the same properties as those of `opengen.tcp.SolverStatus`. + +## Importing optimizer with variable name + +Previously we used `import rosenbrock` to import the auto-generated module. + +The limitation of this syntax is that it makes it difficult to change the name of the optimizer, i.e., `rosenbrock` is hard-coded. + +A better syntax would be: + +```python +optimizers_dir = "my_optimizers" +optimizer_name = "rosenbrock" +sys.path.insert(1, os.path.join(optimizers_dir, optimizer_name)) +rosenbrock = __import__(optimizer_name) +``` diff --git a/open-codegen/opengen/main.py b/open-codegen/opengen/main.py index 513228b5..bfb5ab2c 100644 --- a/open-codegen/opengen/main.py +++ b/open-codegen/opengen/main.py @@ -1,29 +1,25 @@ -import time -import logging +import sys +import os import casadi.casadi as cs import opengen as og -logging.getLogger().setLevel(5) +optimizers_dir = "my_optimizers" +optimizer_name = "rosenbrock" u = cs.SX.sym("u", 5) # decision variable (nu = 5) p = cs.SX.sym("p", 2) # parameter (np = 2) -phi = cs.dot(u, u) # cost function +phi = og.functions.rosenbrock(u, p) # cost function bounds = og.constraints.Halfspace([1., 2., 1., 5., 2.], -10.39) - -problem = og.builder.Problem(u, p, phi) \ - .with_constraints(bounds) - +problem = og.builder.Problem(u, p, phi).with_constraints(bounds) meta = og.config.OptimizerMeta() \ - .with_optimizer_name("halfspace_optimizer") \ - .with_authors(["P. Sopasakis", "S. Author"]).with_version("0.1.56") + .with_optimizer_name(optimizer_name) tcp_config = og.config.TcpServerConfiguration(bind_port=3305) build_config = og.config.BuildConfiguration() \ - .with_build_directory('my_optimizers') \ + .with_build_directory(optimizers_dir) \ .with_build_mode(og.config.BuildConfiguration.DEBUG_MODE) \ .with_build_python_bindings() - # .with_tcp_interface_config() builder = og.builder.OpEnOptimizerBuilder(problem, meta, @@ -31,30 +27,9 @@ og.config.SolverConfiguration()) builder.build() -import halfspace_optimizer +sys.path.insert(1, os.path.join(optimizers_dir, optimizer_name)) +rosenbrock = __import__(optimizer_name) -solver = halfspace_optimizer.build_solver() +solver = rosenbrock.solver() result = solver.run([1., 2.]) print(result.solution) - -# all_managers = [] -# for i in range(10): -# all_managers += [og.tcp.OptimizerTcpManager( -# optimizer_path='my_optimizers/halfspace_optimizer', -# ip='0.0.0.0', -# port=3311+i)] -# -# for m in all_managers: -# m.start() -# -# time.sleep(4) -# -# for m in all_managers: -# print(m.details) -# resp = m.call(p=[1., 2.]) -# print(resp.get().solution) -# -# # mng.kill() -# time.sleep(6) -# for m in all_managers: -# m.kill()