Update to ropt 0.11

Author: verveerpj

pyproject.toml

@@ -138,8 +138,8 @@ everest = [
     "decorator",
     "resdata",
     "colorama",
-    "ropt[pandas]>=0.1,<0.11",
-    "ropt-dakota>=0.1,<0.11",
+    "ropt[pandas]>=0.11,<0.12",
+    "ropt-dakota>=0.11,<0.12",
 ]
 
 [tool.setuptools]

src/ert/run_models/everest_run_model.py

@@ -190,6 +190,7 @@ def run_experiment(
         self.ever_storage = EverestStorage(
             output_dir=Path(self._everest_config.optimization_output_dir),
         )
+        self.ever_storage.init(self._everest_config)
         self.ever_storage.observe_optimizer(optimizer)
 
         # Run the optimization:

src/everest/everest_storage.py

@@ -350,80 +350,82 @@ def read_from_output_dir(self) -> None:
         exp = _OptimizerOnlyExperiment(self._output_dir)
         self.data.read_from_experiment(exp)
 
-    def observe_optimizer(
-        self,
-        optimizer: BasicOptimizer,
-    ) -> None:
-        optimizer.add_observer(
-            EventType.START_OPTIMIZER_STEP, self._on_start_optimization
-        )
+    def observe_optimizer(self, optimizer: BasicOptimizer) -> None:
         optimizer.add_observer(
             EventType.FINISHED_EVALUATION, self._on_batch_evaluation_finished
         )
         optimizer.add_observer(
-            EventType.FINISHED_OPTIMIZER_STEP,
-            self._on_optimization_finished,
+            EventType.FINISHED_OPTIMIZER_STEP, self._on_optimization_finished
         )
 
-    def _on_start_optimization(self, event: Event) -> None:
-        def _format_control_names(control_names):
-            converted_names = []
-            for name in control_names:
-                converted = f"{name[0]}_{name[1]}"
-                if len(name) > 2:
-                    converted += f"-{name[2]}"
-                converted_names.append(converted)
-
-            return converted_names
-
-        config = event.config
-
-        # Note: We probably do not have to store
-        # all of this information, consider removing.
+    def init(self, everest_config: EverestConfig) -> None:
         self.data.controls = polars.DataFrame(
             {
                 "control_name": polars.Series(
-                    _format_control_names(config.variables.names), dtype=polars.String
+                    everest_config.formatted_control_names, dtype=polars.String
                 ),
             }
         )
+
         # TODO: The weight and normalization keys are only used by the everest api,
         # with everviz. They should be removed in the long run.
+        weights = np.fromiter(
+            (
+                1.0 if obj.weight is None else obj.weight
+                for obj in everest_config.objective_functions
+            ),
+            dtype=np.float64,
+        )
         self.data.objective_functions = polars.DataFrame(
             {
-                "objective_name": config.objectives.names,
-                "weight": polars.Series(
-                    config.objectives.weights, dtype=polars.Float64
-                ),
-                "normalization": polars.Series(  # Q: Is this correct?
-                    [1.0 / s for s in config.objectives.scales],
+                "objective_name": everest_config.objective_names,
+                "weight": polars.Series(weights / sum(weights), dtype=polars.Float64),
+                "normalization": polars.Series(
+                    [
+                        1.0 if obj.normalization is None else obj.normalization
+                        for obj in everest_config.objective_functions
+                    ],
                     dtype=polars.Float64,
                 ),
             }
         )
 
-        if config.nonlinear_constraints is not None:
+        if everest_config.output_constraints is not None:
             self.data.nonlinear_constraints = polars.DataFrame(
                 {
-                    "constraint_name": config.nonlinear_constraints.names,
+                    "constraint_name": everest_config.constraint_names,
                 }
             )
+        else:
+            self.data.nonlinear_constraints = None
 
         self.data.realization_weights = polars.DataFrame(
             {
                 "realization": polars.Series(
-                    config.realizations.names, dtype=polars.UInt32
+                    everest_config.model.realizations, dtype=polars.UInt32
                 ),
             }
         )
 
     def _store_function_results(self, results: FunctionResults) -> _EvaluationResults:
+        names = {
+            "variable": self.data.controls["control_name"],
+            "objective": self.data.objective_functions["objective_name"],
+            "nonlinear_constraint": (
+                self.data.nonlinear_constraints["constraint_name"]
+                if self.data.nonlinear_constraints is not None
+                else None
+            ),
+            "realization": self.data.realization_weights["realization"],
+        }
+
         # We could select only objective values,
         # but we select all to also get the constraint values (if they exist)
         realization_objectives = polars.from_pandas(
             results.to_dataframe(
                 "evaluations",
                 select=["objectives", "evaluation_ids"],
+                names=names,
             ).reset_index(),
         ).select(
             "batch_id",
@@ -438,6 +440,7 @@ def _store_function_results(self, results: FunctionResults) -> _EvaluationResult
                 results.to_dataframe(
                     "evaluations",
                     select=["constraints", "evaluation_ids"],
+                    names=names,
                 ).reset_index(),
             ).select(
                 "batch_id",
@@ -452,7 +455,9 @@ def _store_function_results(self, results: FunctionResults) -> _EvaluationResult
             )
 
             batch_constraints = polars.from_pandas(
-                results.to_dataframe("functions", select=["constraints"]).reset_index()
+                results.to_dataframe(
+                    "functions", select=["constraints"], names=names
+                ).reset_index()
             ).select("batch_id", "nonlinear_constraint", "constraints")
 
             batch_constraints = batch_constraints.rename(
@@ -480,12 +485,13 @@ def _store_function_results(self, results: FunctionResults) -> _EvaluationResult
             results.to_dataframe(
                 "functions",
                 select=["objectives", "weighted_objective"],
+                names=names,
             ).reset_index()
         ).select("batch_id", "objective", "objectives", "weighted_objective")
 
         realization_controls = polars.from_pandas(
             results.to_dataframe(
-                "evaluations", select=["variables", "evaluation_ids"]
+                "evaluations", select=["variables", "evaluation_ids"], names=names
             ).reset_index()
         ).select(
             "batch_id",
@@ -535,8 +541,19 @@ def _store_function_results(self, results: FunctionResults) -> _EvaluationResult
         }
 
     def _store_gradient_results(self, results: GradientResults) -> _GradientResults:
+        names = {
+            "variable": self.data.controls["control_name"],
+            "objective": self.data.objective_functions["objective_name"],
+            "nonlinear_constraint": (
+                self.data.nonlinear_constraints["constraint_name"]
+                if self.data.nonlinear_constraints is not None
+                else None
+            ),
+            "realization": self.data.realization_weights["realization"],
+        }
+
         perturbation_objectives = polars.from_pandas(
-            results.to_dataframe("evaluations").reset_index()
+            results.to_dataframe("evaluations", names=names).reset_index()
         ).select(
             [
                 "batch_id",
@@ -560,7 +577,7 @@ def _store_gradient_results(self, results: GradientResults) -> _GradientResults:
 
         if results.gradients is not None:
             batch_objective_gradient = polars.from_pandas(
-                results.to_dataframe("gradients").reset_index()
+                results.to_dataframe("gradients", names=names).reset_index()
             ).select(
                 [
                     "batch_id",
@@ -678,7 +695,7 @@ def _on_batch_evaluation_finished(self, event: Event) -> None:
         logger.debug("Storing batch results dataframes")
 
         converted_results = tuple(
-            convert_to_maximize(result) for result in event.results
+            convert_to_maximize(result) for result in event.data.get("results", [])
         )
         results: list[FunctionResults | GradientResults] = []
 

src/everest/optimizer/everest2ropt.py

@@ -6,7 +6,6 @@
 from ropt.enums import ConstraintType, PerturbationType, VariableType
 
 from everest.config import (
-    ControlConfig,
     EverestConfig,
     InputConstraintConfig,
     ModelConfig,
@@ -49,15 +48,13 @@ def _parse_controls(controls: FlattenedControls, ropt_config):
         ]
     indices = [idx for idx, is_enabled in enumerate(controls.enabled) if is_enabled]
     ropt_config["variables"] = {
-        "names": controls.names,
         "types": None if all(item is None for item in control_types) else control_types,
         "initial_values": controls.initial_guesses,
         "lower_bounds": controls.lower_bounds,
         "upper_bounds": controls.upper_bounds,
         "offsets": offsets,
         "scales": scales,
         "indices": indices if indices else None,
-        "delimiters": "_-",
     }
 
     if "gradients" not in ropt_config:
@@ -81,62 +78,55 @@ def _parse_controls(controls: FlattenedControls, ropt_config):
     ]
 
     ropt_config["gradient"]["perturbation_types"] = [
-        PerturbationType.SCALED.value
-        if auto_scale
-        else PerturbationType[perturbation_type.upper()]
-        for perturbation_type, auto_scale in zip(
-            controls.perturbation_types, controls.auto_scales, strict=True
-        )
+        PerturbationType[perturbation_type.upper()]
+        for perturbation_type in controls.perturbation_types
     ]
 
 
 def _parse_objectives(objective_functions: list[ObjectiveFunctionConfig], ropt_config):
-    names: list[str] = []
     scales: list[float] = []
     auto_scale: list[bool] = []
     weights: list[float] = []
-    transform_indices: list[int] = []
-    transforms: list = []
+    function_estimator_indices: list[int] = []
+    function_estimators: list = []
 
     for objective in objective_functions:
         assert isinstance(objective.name, str)
-        names.append(objective.name)
         weights.append(objective.weight or 1.0)
         scales.append(1.0 / (objective.normalization or 1.0))
         auto_scale.append(objective.auto_normalize or False)
 
         # If any objective specifies an objective type, we have to specify
-        # function transforms in ropt to implement these types. This is done by
-        # supplying a list of transforms and for each objective an index into
+        # function estimators in ropt to implement these types. This is done by
+        # supplying a list of estimators and for each objective an index into
         # that list:
         objective_type = objective.type
         if objective_type is None:
             objective_type = "mean"
-        # Find the transform if it exists:
-        transform_idx = next(
+        # Find the estimator if it exists:
+        function_estimator_idx = next(
             (
                 idx
-                for idx, transform in enumerate(transforms)
-                if transform["method"] == objective_type
+                for idx, estimator in enumerate(function_estimators)
+                if estimator["method"] == objective_type
             ),
             None,
         )
-        # If not, make a new transform:
-        if transform_idx is None:
-            transform_idx = len(transforms)
-            transforms.append({"method": objective_type})
-        transform_indices.append(transform_idx)
+        # If not, make a new estimator:
+        if function_estimator_idx is None:
+            function_estimator_idx = len(function_estimators)
+            function_estimators.append({"method": objective_type})
+        function_estimator_indices.append(function_estimator_idx)
 
     ropt_config["objectives"] = {
-        "names": names,
         "weights": weights,
         "scales": scales,
         "auto_scale": auto_scale,
     }
-    if transforms:
+    if function_estimators:
         # Only needed if we specified at least one objective type:
-        ropt_config["objectives"]["function_transforms"] = transform_indices
-        ropt_config["function_transforms"] = transforms
+        ropt_config["objectives"]["function_estimators"] = function_estimator_indices
+        ropt_config["function_estimators"] = function_estimators
 
 
 def _parse_input_constraints(
@@ -197,7 +187,6 @@ def _parse_output_constraints(
     if not output_constraints:
         return
 
-    names: list[str] = []
     rhs_values: list[float] = []
     scales: list[float] = []
     auto_scale: list[bool] = []
@@ -207,8 +196,6 @@ def _add_output_constraint(
         rhs_value: float | None, constraint_type: ConstraintType, suffix=None
     ):
         if rhs_value is not None:
-            name = constr.name
-            names.append(name if suffix is None else f"{name}:{suffix}")
             rhs_values.append(rhs_value)
             scales.append(constr.scale if constr.scale is not None else 1.0)
             auto_scale.append(constr.auto_scale or False)
@@ -238,7 +225,6 @@ def _add_output_constraint(
         )
 
     ropt_config["nonlinear_constraints"] = {
-        "names": names,
         "rhs_values": rhs_values,
         "scales": scales,
         "auto_scale": auto_scale,
@@ -336,9 +322,9 @@ def _parse_optimization(
             # indices to any realization filters that should be applied. In this
             # case, we want all objectives and constraints to refer to the same
             # filter implementing cvar:
-            objective_count = len(ropt_config["objectives"]["names"])
+            objective_count = len(ropt_config["objectives"]["weights"])
             constraint_count = len(
-                ropt_config.get("nonlinear_constraints", {}).get("names", [])
+                ropt_config.get("nonlinear_constraints", {}).get("rhs_values", [])
             )
             ropt_config["objectives"]["realization_filters"] = objective_count * [0]
             if constraint_count > 0:
@@ -366,7 +352,6 @@ def _parse_model(
         ever_reals_weights = [1.0 / len(ever_reals)] * len(ever_reals)
 
     ropt_config["realizations"] = {
-        "names": ever_reals,
         "weights": ever_reals_weights,
     }
     min_real_succ = ever_opt.min_realizations_success if ever_opt else None

tests/everest/test_multiobjective.py

@@ -89,10 +89,8 @@ def test_multi_objectives2ropt(copy_mocked_test_data_to_tmp):
     enopt_config = EnOptConfig.model_validate(
         everest2ropt(EverestConfig.model_validate(config_dict))
     )
-    assert len(enopt_config.objectives.names) == 2
-    assert enopt_config.objectives.names[1] == ever_objs[1]["name"]
+    assert len(enopt_config.objectives.weights) == 2
     assert enopt_config.objectives.weights[1] == ever_objs[1]["weight"] / norm
-    assert enopt_config.objectives.names[0] == ever_objs[0]["name"]
     assert enopt_config.objectives.weights[0] == ever_objs[0]["weight"] / norm
     assert enopt_config.objectives.scales[0] == ever_objs[0]["normalization"]
 

tests/everest/test_output_constraints.py

@@ -222,7 +222,6 @@ def test_upper_bound_output_constraint_def(copy_mocked_test_data_to_tmp):
     }
 
     assert expected["scale"] == 1.0 / ropt_conf.nonlinear_constraints.scales[0]
-    assert expected["name"] == ropt_conf.nonlinear_constraints.names[0]
     assert expected["rhs_value"] == ropt_conf.nonlinear_constraints.rhs_values[0]
     assert expected["type"] == ropt_conf.nonlinear_constraints.types[0]