Remove ErtConfig from EverestRunModel

src/ert/run_models/everest_run_model.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import collections
+import copy
 import datetime
 import functools
 import json
@@ -27,14 +29,32 @@
 from typing_extensions import TypedDict
 
 from _ert.events import EESnapshot, EESnapshotUpdate, Event
-from ert.config import ErtConfig, ExtParamConfig
+from ert.config import ExtParamConfig
+from ert.config.ensemble_config import EnsembleConfig
+from ert.config.ert_config import (
+    ErtConfig,
+    _substitutions_from_dict,
+    create_list_of_forward_model_steps_to_run,
+    installed_forward_model_steps_from_dict,
+    read_templates,
+    uppercase_subkeys_and_stringify_subvalues,
+    workflows_from_dict,
+)
+from ert.config.forward_model_step import ForwardModelStep, ForwardModelStepPlugin
+from ert.config.model_config import ModelConfig
+from ert.config.queue_config import QueueConfig
 from ert.ensemble_evaluator import EnsembleSnapshot, EvaluatorServerConfig
+from ert.plugins.plugin_manager import ErtPluginManager
 from ert.runpaths import Runpaths
 from ert.storage import open_storage
 from everest.config import ControlConfig, ControlVariableGuessListConfig, EverestConfig
+from everest.config.control_variable_config import ControlVariableConfig
 from everest.optimizer.everest2ropt import everest2ropt
-from everest.simulator.everest_to_ert import everest_to_ert_config
-from everest.strings import EVEREST
+from everest.simulator.everest_to_ert import (
+    everest_to_ert_config,
+    everest_to_ert_config_dict,
+)
+from everest.strings import EVEREST, STORAGE_DIR
 
 from ..run_arg import RunArg, create_run_arguments
 from .base_run_model import BaseRunModel, StatusEvents
@@ -104,6 +124,9 @@ def __init__(
         simulation_callback: SimulationCallback | None,
         optimization_callback: OptimizerCallback | None,
     ):
+        assert everest_config.log_dir is not None
+        assert everest_config.optimization_output_dir is not None
+
         Path(everest_config.log_dir).mkdir(parents=True, exist_ok=True)
         Path(everest_config.optimization_output_dir).mkdir(parents=True, exist_ok=True)
 
@@ -135,39 +158,122 @@ def __init__(
         self._batch_id: int = 0
         self._status: SimulationStatus | None = None
 
-        storage = open_storage(config.ens_path, mode="w")
+        ens_path = os.path.join(everest_config.output_dir, STORAGE_DIR)
+        storage = open_storage(ens_path, mode="w")
         status_queue: queue.SimpleQueue[StatusEvents] = queue.SimpleQueue()
 
+        config_dict = everest_to_ert_config_dict(everest_config)
+
+        runpath_file: Path = Path(
+            os.path.join(everest_config.output_dir, ".res_runpath_list")
+        )
+
+        assert everest_config.config_file is not None
+        config_file: Path = Path(everest_config.config_file)
+
+        model_config = ModelConfig.from_dict(config_dict)
+        queue_config = QueueConfig.from_dict(config_dict)
+
+        ensemble_config = EnsembleConfig.from_dict(config_dict)
+
+        def _get_variables(
+            variables: list[ControlVariableConfig]
+            | list[ControlVariableGuessListConfig],
+        ) -> list[str] | dict[str, list[str]]:
+            if (
+                isinstance(variables[0], ControlVariableConfig)
+                and getattr(variables[0], "index", None) is None
+            ):
+                return [var.name for var in variables]
+            result: collections.defaultdict[str, list] = collections.defaultdict(list)
+            for variable in variables:
+                if isinstance(variable, ControlVariableGuessListConfig):
+                    result[variable.name].extend(
+                        str(index + 1) for index, _ in enumerate(variable.initial_guess)
+                    )
+                else:
+                    result[variable.name].append(str(variable.index))  # type: ignore
+            return dict(result)
+
+        # This adds an EXT_PARAM key to the ert_config, which is not a true ERT
+        # configuration key. When initializing an ERT config object, it is ignored.
+        # It is used by the Simulator object to inject ExtParamConfig nodes.
+        for control in everest_config.controls or []:
+            ensemble_config.parameter_configs[control.name] = ExtParamConfig(
+                name=control.name,
+                input_keys=_get_variables(control.variables),
+                output_file=control.name + ".json",
+            )
+
+        substitutions = _substitutions_from_dict(config_dict)
+        substitutions["<RUNPATH_FILE>"] = str(runpath_file)
+        substitutions["<RUNPATH>"] = model_config.runpath_format_string
+        substitutions["<ECL_BASE>"] = model_config.eclbase_format_string
+        substitutions["<ECLBASE>"] = model_config.eclbase_format_string
+        substitutions["<NUM_CPU>"] = str(queue_config.preferred_num_cpu)
+
+        ert_templates = read_templates(config_dict)
+        _, _, hooked_workflows = workflows_from_dict(config_dict, substitutions)
+
+        preinstalled_fm_steps: dict[str, ForwardModelStepPlugin] = {}
+        pm = ErtPluginManager()
+        for fm_step_subclass in pm.forward_model_steps:
+            fm_step = fm_step_subclass()
+            preinstalled_fm_steps[fm_step.name] = fm_step
+
+        installed_forward_model_steps: dict[str, ForwardModelStep] = {}
+        installed_forward_model_steps = copy.deepcopy(preinstalled_fm_steps)  # fix
+        installed_forward_model_steps.update(
+            installed_forward_model_steps_from_dict(config_dict)
+        )
+
+        env_pr_fm_step = uppercase_subkeys_and_stringify_subvalues(
+            pm.get_forward_model_configuration()
+        )
+
+        forward_model_steps = create_list_of_forward_model_steps_to_run(
+            installed_forward_model_steps,
+            substitutions,
+            config_dict,
+            installed_forward_model_steps,
+            env_pr_fm_step,
+        )
+
+        env_vars = {}
+        for key, val in config_dict.get("SETENV", []):
+            env_vars[key] = val
+
+        self.support_restart = False
+        self._parameter_configuration = ensemble_config.parameter_configuration
+        self._parameter_configs = ensemble_config.parameter_configs
+        self._response_configuration = ensemble_config.response_configuration
+
         super().__init__(
             storage,
-            config.runpath_file,
-            Path(config.user_config_file),
-            config.env_vars,
-            config.env_pr_fm_step,
-            config.model_config,
-            config.queue_config,
-            config.forward_model_steps,
+            runpath_file,
+            config_file,
+            env_vars,
+            env_pr_fm_step,
+            model_config,
+            queue_config,
+            forward_model_steps,
             status_queue,
-            config.substitutions,
-            config.ert_templates,
-            config.hooked_workflows,
+            substitutions,
+            ert_templates,
+            hooked_workflows,
             active_realizations=[],  # Set dynamically in run_forward_model()
         )
-        self.support_restart = False
-        self._parameter_configuration = config.ensemble_config.parameter_configuration
-        self._parameter_configs = config.ensemble_config.parameter_configs
-        self._response_configuration = config.ensemble_config.response_configuration
 
     @classmethod
     def create(
         cls,
-        ever_config: EverestConfig,
+        everest_config: EverestConfig,
         simulation_callback: SimulationCallback | None = None,
         optimization_callback: OptimizerCallback | None = None,
     ) -> EverestRunModel:
         return cls(
-            config=everest_to_ert_config(ever_config),
-            everest_config=ever_config,
+            config=everest_to_ert_config(everest_config),
+            everest_config=everest_config,
             simulation_callback=simulation_callback,
             optimization_callback=optimization_callback,
         )
@@ -291,6 +397,7 @@ def _create_optimizer(self) -> BasicOptimizer:
         # simplifying code that reads them as fixed width tables. `maximize` is
         # set because ropt reports minimization results, while everest wants
         # maximization results, necessitating a conversion step.
+        assert self._everest_config.optimization_output_dir is not None
         ropt_output_folder = Path(self._everest_config.optimization_output_dir)
         optimizer = (
             BasicOptimizer(

src/everest/simulator/everest_to_ert.py

@@ -489,6 +489,14 @@ def _everest_to_ert_config_dict(
     return ert_config
 
 
+def everest_to_ert_config_dict(everest_config: EverestConfig) -> ConfigDict:
+    with ErtPluginContext():
+        config_dict = _everest_to_ert_config_dict(
+            everest_config, site_config=ErtConfig.read_site_config()
+        )
+    return config_dict
+
+
 def everest_to_ert_config(ever_config: EverestConfig) -> ErtConfig:
     with ErtPluginContext():
         config_dict = _everest_to_ert_config_dict(

test-data/everest/math_func/everest_output/.internal_data/wells.json

@@ -0,0 +1 @@
+[]

test-data/everest/math_func/everest_output/.jobs/_recovery_factor

@@ -0,0 +1 @@
+EXECUTABLE /Users/FAAR/code/ert/src/everest/jobs/scripts/recovery_factor

test-data/everest/math_func/everest_output/.jobs/_render

@@ -0,0 +1 @@
+EXECUTABLE /Users/FAAR/code/ert/src/everest/jobs/scripts/render

test-data/everest/math_func/everest_output/.jobs/_wdcompl

@@ -0,0 +1 @@
+EXECUTABLE /Users/FAAR/code/ert/src/everest/jobs/scripts/wdcompl

test-data/everest/math_func/everest_output/.jobs/_wddatefilter

@@ -0,0 +1 @@
+EXECUTABLE /Users/FAAR/code/ert/src/everest/jobs/scripts/wddatefilter

test-data/everest/math_func/everest_output/.jobs/_wdfilter

@@ -0,0 +1 @@
+EXECUTABLE /Users/FAAR/code/ert/src/everest/jobs/scripts/wdfilter

test-data/everest/math_func/everest_output/.jobs/_wdreorder

@@ -0,0 +1 @@
+EXECUTABLE /Users/FAAR/code/ert/src/everest/jobs/scripts/wdreorder

test-data/everest/math_func/everest_output/.jobs/_wdset

@@ -0,0 +1 @@
+EXECUTABLE /Users/FAAR/code/ert/src/everest/jobs/scripts/wdset

test-data/everest/math_func/everest_output/.jobs/_wdupdate

@@ -0,0 +1 @@
+EXECUTABLE /Users/FAAR/code/ert/src/everest/jobs/scripts/wdupdate

test-data/everest/math_func/everest_output/simulation_results/.fs_version

@@ -0,0 +1 @@
+index.json

test-data/everest/math_func/everest_output/simulation_results/index.json

@@ -0,0 +1,4 @@
+{
+    "version": 9,
+    "migrations": []
+}