Refactor and update for ropt 0.11

pyproject.toml

@@ -138,8 +138,8 @@ everest = [
     "decorator",
     "resdata",
     "colorama",
-    "ropt[pandas]>=0.1,<0.11",
-    "ropt-dakota>=0.1,<0.11",
+    "ropt[pandas]>=0.11,<0.12",
+    "ropt-dakota>=0.11,<0.12",
 ]
 
 [tool.setuptools]

src/ert/run_models/everest_run_model.py

@@ -190,6 +190,7 @@ def run_experiment(
         self.ever_storage = EverestStorage(
             output_dir=Path(self._everest_config.optimization_output_dir),
         )
+        self.ever_storage.init(self._everest_config)
         self.ever_storage.observe_optimizer(optimizer)
 
         # Run the optimization:
@@ -210,92 +211,8 @@ def run_experiment(
                     self._exit_code = EverestExitCode.COMPLETED
 
     def _create_optimizer(self) -> BasicOptimizer:
-        RESULT_COLUMNS = {
-            "result_id": "ID",
-            "batch_id": "Batch",
-            "functions.weighted_objective": "Total-Objective",
-            "linear_constraints.violations": "IC-violation",
-            "nonlinear_constraints.violations": "OC-violation",
-            "functions.objectives": "Objective",
-            "functions.constraints": "Constraint",
-            "evaluations.variables": "Control",
-            "linear_constraints.values": "IC-diff",
-            "nonlinear_constraints.values": "OC-diff",
-            "functions.scaled_objectives": "Scaled-Objective",
-            "functions.scaled_constraints": "Scaled-Constraint",
-            "evaluations.scaled_variables": "Scaled-Control",
-            "nonlinear_constraints.scaled_values": "Scaled-OC-diff",
-            "nonlinear_constraints.scaled_violations": "Scaled-OC-violation",
-        }
-        GRADIENT_COLUMNS = {
-            "result_id": "ID",
-            "batch_id": "Batch",
-            "gradients.weighted_objective": "Total-Gradient",
-            "gradients.objectives": "Grad-objective",
-            "gradients.constraints": "Grad-constraint",
-        }
-        SIMULATION_COLUMNS = {
-            "result_id": "ID",
-            "batch_id": "Batch",
-            "realization": "Realization",
-            "evaluations.evaluation_ids": "Simulation",
-            "evaluations.variables": "Control",
-            "evaluations.objectives": "Objective",
-            "evaluations.constraints": "Constraint",
-            "evaluations.scaled_variables": "Scaled-Control",
-            "evaluations.scaled_objectives": "Scaled-Objective",
-            "evaluations.scaled_constraints": "Scaled-Constraint",
-        }
-        PERTURBATIONS_COLUMNS = {
-            "result_id": "ID",
-            "batch_id": "Batch",
-            "realization": "Realization",
-            "evaluations.perturbed_evaluation_ids": "Simulation",
-            "evaluations.perturbed_variables": "Control",
-            "evaluations.perturbed_objectives": "Objective",
-            "evaluations.perturbed_constraints": "Constraint",
-            "evaluations.scaled_perturbed_variables": "Scaled-Control",
-            "evaluations.scaled_perturbed_objectives": "Scaled-Objective",
-            "evaluations.scaled_perturbed_constraints": "Scaled-Constraint",
-        }
-        MIN_HEADER_LEN = 3
-
-        # Initialize the optimizer with output tables. `min_header_len` is set
-        # to ensure that all tables have the same number of header lines,
-        # simplifying code that reads them as fixed width tables. `maximize` is
-        # set because ropt reports minimization results, while everest wants
-        # maximization results, necessitating a conversion step.
-        ropt_output_folder = Path(self._everest_config.optimization_output_dir)
-        optimizer = (
-            BasicOptimizer(
-                enopt_config=self._ropt_config, evaluator=self._forward_model_evaluator
-            )
-            .add_table(
-                columns=RESULT_COLUMNS,
-                path=ropt_output_folder / "results.txt",
-                min_header_len=MIN_HEADER_LEN,
-                maximize=True,
-            )
-            .add_table(
-                columns=GRADIENT_COLUMNS,
-                path=ropt_output_folder / "gradients.txt",
-                table_type="gradients",
-                min_header_len=MIN_HEADER_LEN,
-                maximize=True,
-            )
-            .add_table(
-                columns=SIMULATION_COLUMNS,
-                path=ropt_output_folder / "simulations.txt",
-                min_header_len=MIN_HEADER_LEN,
-                maximize=True,
-            )
-            .add_table(
-                columns=PERTURBATIONS_COLUMNS,
-                path=ropt_output_folder / "perturbations.txt",
-                table_type="gradients",
-                min_header_len=MIN_HEADER_LEN,
-                maximize=True,
-            )
+        optimizer = BasicOptimizer(
+            enopt_config=self._ropt_config, evaluator=self._forward_model_evaluator
         )
 
         # Before each batch evaluation we check if we should abort:

src/everest/api/everest_data_api.py

@@ -3,7 +3,6 @@
 
 import polars
 import polars as pl
-from ropt.enums import ConstraintType
 
 from ert.storage import open_storage
 from everest.config import EverestConfig
@@ -53,35 +52,6 @@ def output_constraint_names(self):
             else []
         )
 
-    def input_constraint(self, control):
-        # Note: This function is weird, its existence is probably not well-justified
-        # consider removing!
-        initial_values = self._ever_storage.data.controls
-        control_spec = initial_values.filter(
-            pl.col("control_name") == control
-        ).to_dicts()[0]
-        return {
-            "min": control_spec.get("lower_bounds"),
-            "max": control_spec.get("upper_bounds"),
-        }
-
-    def output_constraint(self, constraint):
-        """
-        :return: a dictionary with two keys: "type" and "right_hand_side".
-                 "type" has three options:
-                     ["lower_bound", "upper_bound", "target"]
-                 "right_hand_side" is a constant real number that indicates
-                 the constraint bound/target.
-        """
-
-        constraint_dict = self._ever_storage.data.nonlinear_constraints.filter(
-            polars.col("constraint_name") == constraint
-        ).to_dicts()[0]
-        return {
-            "type": ConstraintType(constraint_dict["constraint_type"]).name.lower(),
-            "right_hand_side": constraint_dict["constraint_rhs_value"],
-        }
-
     @property
     def realizations(self):
         return sorted(

src/everest/config/everest_config.py

@@ -621,6 +621,20 @@ def control_names(self):
         controls = self.controls or []
         return [control.name for control in controls]
 
+    @property
+    def formatted_control_names(self) -> list[str]:
+        names = []
+        for control in self.controls:
+            for variable in control.variables:
+                if isinstance(variable, ControlVariableGuessListConfig):
+                    for index in range(1, len(variable.initial_guess) + 1):
+                        names.append(f"{control.name}_{variable.name}-{index}")
+                elif variable.index is not None:
+                    names.append(f"{control.name}_{variable.name}-{variable.index}")
+                else:
+                    names.append(f"{control.name}_{variable.name}")
+        return names
+
     @property
     def objective_names(self) -> list[str]:
         return [objective.name for objective in self.objective_functions]

src/everest/config/simulator_config.py

@@ -79,17 +79,13 @@ class SimulatorConfig(BaseModel, extra="forbid"):  # type: ignore
     If not specified, a default value of 1 will be used.""",
     )
     enable_cache: bool = Field(
-        default=False,
+        default=True,
         description="""Enable forward model result caching.
 
         If enabled, objective and constraint function results are cached for
         each realization. If the optimizer requests an evaluation that has
         already been done before, these cached values will be re-used without
-        running the forward model again.
-
-        This option is disabled by default, since it will not be necessary for
-        the most common use of a standard optimization with a continuous
-        optimizer.""",
+        running the forward model again.""",
     )
 
     @field_validator("queue_system", mode="before")

src/everest/config/utils.py

@@ -1,4 +1,4 @@
-from collections.abc import Generator, Iterator
+from collections.abc import Generator
 from typing import Any
 
 from .control_config import ControlConfig
@@ -129,17 +129,3 @@ def _inject_defaults(
             ]:
                 if var_dict.get(key) is None:
                     var_dict[key] = getattr(control, key)
-
-
-def control_tuples(
-    controls: list[ControlConfig],
-) -> Iterator[tuple[str, str, int] | tuple[str, str]]:
-    for control in controls:
-        for variable in control.variables:
-            if isinstance(variable, ControlVariableGuessListConfig):
-                for index in range(1, len(variable.initial_guess) + 1):
-                    yield (control.name, variable.name, index)
-            elif variable.index is not None:
-                yield (control.name, variable.name, variable.index)
-            else:
-                yield (control.name, variable.name)