diff --git a/.mypy.ini b/.mypy.ini deleted file mode 100644 index 3bffd3df154..00000000000 --- a/.mypy.ini +++ /dev/null @@ -1,82 +0,0 @@ -[mypy] -plugins = pydantic.mypy - -warn_unused_configs = True -disallow_any_generics = True -disallow_subclassing_any = true -disallow_untyped_calls = True -disallow_untyped_defs = True -disallow_incomplete_defs = True -check_untyped_defs = True -disallow_untyped_decorators = True -warn_unused_ignores = True -warn_redundant_casts = True -strict_equality = True -extra_checks = True - - -exclude = src/ert/shared/share - -[mypy-scipy.*] -ignore_missing_imports = True - -[mypy-cloudpickle.*] -ignore_missing_imports = True - -[mypy-numpy.*] -ignore_missing_imports = True - -[mypy-pandas.*] -ignore_missing_imports = True - -[mypy-xtgeo.*] -ignore_missing_imports = True - -[mypy-ert._clib.*] -ignore_missing_imports = True - -[mypy-ert.gui.*] -ignore_missing_imports = True -ignore_errors = True - -[mypy-cwrap.*] -ignore_missing_imports = True - -[mypy-resdata.*] -ignore_missing_imports = True - -[mypy-resfo.*] -ignore_missing_imports = True - -[mypy-sortedcontainers.*] -ignore_missing_imports = True - -[mypy-deprecation.*] -ignore_missing_imports = True - -[mypy-iterative_ensemble_smoother.*] -ignore_missing_imports = True - -[mypy-colors.*] -ignore_missing_imports = True - -[mypy-cloudevents.*] -ignore_missing_imports = True - -[mypy-async_generator.*] -ignore_missing_imports = True - -[mypy-uvicorn.*] -ignore_missing_imports = True - -[mypy-SALib.*] -ignore_missing_imports = True - -[mypy-pluggy.*] -ignore_missing_imports = True - -[mypy-ruamel] -ignore_missing_imports = True - -[mypy-ert.callbacks] -ignore_errors = True diff --git a/src/ert/dark_storage/common.py b/src/ert/dark_storage/common.py index 0cf4bfee031..7060ee68d05 100644 --- a/src/ert/dark_storage/common.py +++ b/src/ert/dark_storage/common.py @@ -30,7 +30,6 @@ def get_response_names(ensemble: EnsembleReader) -> List[str]: def data_for_key( ensemble: EnsembleReader, key: str, - realization_index: Optional[int] = None, ) -> pd.DataFrame: """Returns a pandas DataFrame with the datapoints for a given key for a given case. The row index is the realization number, and the columns are an @@ -39,10 +38,10 @@ def data_for_key( if key.startswith("LOG10_"): key = key[6:] if key in ensemble.get_summary_keyset(): - data = ensemble.load_all_summary_data([key], realization_index) + data = ensemble.load_summary(key) data = data[key].unstack(level="Date") elif key in ensemble.get_gen_kw_keyset(): - data = ensemble.load_all_gen_kw_data(key.split(":")[0], realization_index) + data = ensemble.load_all_gen_kw_data(key.split(":")[0]) if data.empty: return pd.DataFrame() data = data[key].to_frame().dropna() @@ -56,7 +55,6 @@ def data_for_key( data = ensemble.load_gen_data( key, report_step, - realization_index, ).T except (ValueError, KeyError): return pd.DataFrame() diff --git a/src/ert/dark_storage/endpoints/ensembles.py b/src/ert/dark_storage/endpoints/ensembles.py index 8d0a53fc02c..67a020011dc 100644 --- a/src/ert/dark_storage/endpoints/ensembles.py +++ b/src/ert/dark_storage/endpoints/ensembles.py @@ -1,4 +1,3 @@ -from typing import Any, Mapping from uuid import UUID from fastapi import APIRouter, Body, Depends @@ -13,15 +12,6 @@ DEFAULT_BODY = Body(...) -@router.post("/experiments/{experiment_id}/ensembles", response_model=js.EnsembleOut) -def post_ensemble( - *, - ens_in: js.EnsembleIn, - experiment_id: UUID, -) -> js.EnsembleOut: - raise NotImplementedError - - @router.get("/ensembles/{ensemble_id}", response_model=js.EnsembleOut) def get_ensemble( *, @@ -42,27 +32,21 @@ def get_ensemble( ) -@router.put("/ensembles/{ensemble_id}/userdata") -async def replace_ensemble_userdata( - *, - ensemble_id: UUID, - body: Any = DEFAULT_BODY, -) -> None: - raise NotImplementedError - - -@router.patch("/ensembles/{ensemble_id}/userdata") -async def patch_ensemble_userdata( - *, - ensemble_id: UUID, - body: Any = DEFAULT_BODY, -) -> None: - raise NotImplementedError - - -@router.get("/ensembles/{ensemble_id}/userdata", response_model=Mapping[str, Any]) -async def get_ensemble_userdata( +@router.get("/ensembles/{ensemble_id}/small", response_model=js.EnsembleOut) +def get_ensemble_small( *, + storage: StorageAccessor = DEFAULT_STORAGE, ensemble_id: UUID, -) -> Mapping[str, Any]: - raise NotImplementedError +) -> js.EnsembleOut: + ensemble = storage.get_ensemble(ensemble_id) + return js.EnsembleOut( + id=ensemble_id, + children=[], + parent=None, + experiment_id=ensemble.experiment_id, + userdata={"name": ensemble.name}, + size=ensemble.ensemble_size, + parameter_names=[], + response_names=[], + child_ensemble_ids=[], + ) diff --git a/src/ert/dark_storage/endpoints/records.py b/src/ert/dark_storage/endpoints/records.py index 5de1cf58247..a800f7de8d1 100644 --- a/src/ert/dark_storage/endpoints/records.py +++ b/src/ert/dark_storage/endpoints/records.py @@ -1,10 +1,9 @@ import io from itertools import chain -from typing import Any, Dict, List, Mapping, Optional, Union +from typing import Any, Dict, List, Mapping, Union from uuid import UUID, uuid4 -import pandas as pd -from fastapi import APIRouter, Body, Depends, File, Header, Request, UploadFile, status +from fastapi import APIRouter, Body, Depends, File, Header, status from fastapi.responses import Response from typing_extensions import Annotated @@ -27,117 +26,11 @@ DEFAULT_HEADER = Header("application/json") -@router.post("/ensembles/{ensemble_id}/records/{name}/file") -async def post_ensemble_record_file( - *, - name: str, - ensemble_id: UUID, - realization_index: Optional[int] = None, - file: UploadFile = DEFAULT_FILE, -) -> None: - raise NotImplementedError - - -@router.put("/ensembles/{ensemble_id}/records/{name}/blob") -async def add_block( - *, - name: str, - ensemble_id: UUID, - block_index: int, - realization_index: Optional[int] = None, - request: Request, -) -> None: - raise NotImplementedError - - -@router.post("/ensembles/{ensemble_id}/records/{name}/blob") -async def create_blob( - *, - name: str, - ensemble_id: UUID, - realization_index: Optional[int] = None, -) -> None: - raise NotImplementedError - - -@router.patch("/ensembles/{ensemble_id}/records/{name}/blob") -async def finalize_blob( - *, - name: str, - ensemble_id: UUID, - realization_index: Optional[int] = None, -) -> None: - raise NotImplementedError - - -@router.post( - "/ensembles/{ensemble_id}/records/{name}/matrix", response_model=js.RecordOut -) -async def post_ensemble_record_matrix( - *, - ensemble_id: UUID, - name: str, - prior: Optional[str] = None, - realization_index: Optional[int] = None, - content_type: str = DEFAULT_HEADER, - request: Request, -) -> js.RecordOut: - raise NotImplementedError - - -@router.put("/ensembles/{ensemble_id}/records/{name}/userdata") -async def replace_record_userdata( - *, - ensemble_id: UUID, - name: str, - realization_index: Optional[int] = None, - body: Any = DEFAULT_BODY, -) -> None: - raise NotImplementedError - - -@router.patch("/ensembles/{ensemble_id}/records/{name}/userdata") -async def patch_record_userdata( - *, - ensemble_id: UUID, - name: str, - realization_index: Optional[int] = None, - body: Any = DEFAULT_BODY, -) -> None: - raise NotImplementedError - - -@router.get( - "/ensembles/{ensemble_id}/records/{name}/userdata", response_model=Mapping[str, Any] -) -async def get_record_userdata( - *, - ensemble_id: UUID, - name: str, - realization_index: Optional[int] = None, -) -> Mapping[str, Any]: - raise NotImplementedError - - -@router.post("/ensembles/{ensemble_id}/records/{name}/observations") -async def post_record_observations( - *, - ensemble_id: UUID, - name: str, - realization_index: Optional[int] = None, - observation_ids: List[UUID] = DEFAULT_BODY, -) -> None: - raise NotImplementedError - - @router.get("/ensembles/{ensemble_id}/records/{name}/observations") async def get_record_observations( *, res: LibresFacade = DEFAULT_LIBRESFACADE, - db: StorageReader = DEFAULT_STORAGE, - ensemble_id: UUID, name: str, - realization_index: Optional[int] = None, ) -> List[js.ObservationOut]: obs_keys = res.observation_keys(name) obss = observations_for_obs_keys(res, obs_keys) @@ -174,15 +67,8 @@ async def get_ensemble_record( name: str, ensemble_id: UUID, accept: Annotated[Union[str, None], Header()] = None, - realization_index: Optional[int] = None, - label: Optional[str] = None, ) -> Any: - dataframe = data_for_key(db.get_ensemble(ensemble_id), name, realization_index) - if realization_index is not None: - # dataframe.loc returns a Series, and when we reconstruct a DataFrame - # from a Series, it defaults to be oriented the wrong way, so we must - # transpose it - dataframe = pd.DataFrame(dataframe.loc[realization_index]).T + dataframe = data_for_key(db.get_ensemble(ensemble_id), name) media_type = accept if accept is not None else "text/csv" if media_type == "application/x-parquet": @@ -202,15 +88,6 @@ async def get_ensemble_record( ) -@router.get("/ensembles/{ensemble_id}/records/{name}/labels", response_model=List[str]) -async def get_record_labels( - *, - ensemble_id: UUID, - name: str, -) -> List[str]: - return [] - - @router.get("/ensembles/{ensemble_id}/parameters", response_model=List[Dict[str, Any]]) async def get_ensemble_parameters( *, storage: StorageReader = DEFAULT_STORAGE, ensemble_id: UUID @@ -218,27 +95,6 @@ async def get_ensemble_parameters( return ensemble_parameters(storage, ensemble_id) -@router.get( - "/ensembles/{ensemble_id}/records", response_model=Mapping[str, js.RecordOut] -) -async def get_ensemble_records(*, ensemble_id: UUID) -> Mapping[str, js.RecordOut]: - raise NotImplementedError - - -@router.get("/records/{record_id}", response_model=js.RecordOut) -async def get_record(*, record_id: UUID) -> js.RecordOut: - raise NotImplementedError - - -@router.get("/records/{record_id}/data") -async def get_record_data( - *, - record_id: UUID, - accept: Optional[str] = DEFAULT_HEADER, -) -> Any: - raise NotImplementedError - - @router.get( "/ensembles/{ensemble_id}/responses", response_model=Mapping[str, js.RecordOut] ) @@ -249,15 +105,18 @@ def get_ensemble_responses( ensemble_id: UUID, ) -> Mapping[str, js.RecordOut]: response_map: Dict[str, js.RecordOut] = {} - ens = db.get_ensemble(ensemble_id) + name_dict = {} + + for obs in res.get_observations(): + name_dict[obs.observation_key] = obs.observation_type + for name in ens.get_summary_keyset(): - obs_keys = res.observation_keys(name) response_map[str(name)] = js.RecordOut( id=UUID(int=0), name=name, userdata={"data_origin": "Summary"}, - has_observations=len(obs_keys) != 0, + has_observations=name in name_dict, ) for name in res.get_gen_data_keys(): diff --git a/src/ert/gui/tools/plot/plot_api.py b/src/ert/gui/tools/plot/plot_api.py index 20e9ddbabe2..063cec04ed2 100644 --- a/src/ert/gui/tools/plot/plot_api.py +++ b/src/ert/gui/tools/plot/plot_api.py @@ -43,7 +43,7 @@ def _get_all_cases(self) -> List[dict]: for experiment in experiments: for ensemble_id in experiment["ensemble_ids"]: response = client.get( - f"/ensembles/{ensemble_id}", timeout=self._timeout + f"/ensembles/{ensemble_id}/small", timeout=self._timeout ) self._check_response(response) response_json = response.json() @@ -68,23 +68,6 @@ def _check_response(response: requests.Response): f"{response.text} from url: {response.url}." ) - def _get_experiments(self) -> dict: - with StorageService.session() as client: - response: requests.Response = client.get( - "/experiments", timeout=self._timeout - ) - self._check_response(response) - return response.json() - - def _get_ensembles(self, experiement_id) -> List: - with StorageService.session() as client: - response: requests.Response = client.get( - f"/experiments/{experiement_id}/ensembles", timeout=self._timeout - ) - self._check_response(response) - response_json = response.json() - return response_json - def all_data_type_keys(self) -> List: """Returns a list of all the keys except observation keys. @@ -94,9 +77,20 @@ def all_data_type_keys(self) -> List: the key""" all_keys = {} + with StorageService.session() as client: - for experiment in self._get_experiments(): - for ensemble in self._get_ensembles(experiment["id"]): + response: requests.Response = client.get( + "/experiments", timeout=self._timeout + ) + self._check_response(response) + + for experiment in response.json(): + response: requests.Response = client.get( + f"/experiments/{experiment['id']}/ensembles", timeout=self._timeout + ) + self._check_response(response) + + for ensemble in response.json(): response: requests.Response = client.get( f"/ensembles/{ensemble['id']}/responses", timeout=self._timeout ) @@ -133,7 +127,7 @@ def get_all_cases_not_running(self) -> List: info about the case is returned""" # Currently, the ensemble information from the storage API does not contain any # hint if a case is running or not for now we return all the cases, running or - # not + # no return self._get_all_cases() def data_for_key(self, case_name, key) -> pd.DataFrame: diff --git a/src/ert/gui/tools/plot/plot_window.py b/src/ert/gui/tools/plot/plot_window.py index 1fcd777edb5..6cbd5de34d3 100644 --- a/src/ert/gui/tools/plot/plot_window.py +++ b/src/ert/gui/tools/plot/plot_window.py @@ -1,4 +1,5 @@ import logging +import time from typing import List from httpx import RequestError @@ -42,12 +43,10 @@ class PlotWindow(QMainWindow): def __init__(self, config_file, parent): QMainWindow.__init__(self, parent) - + t = time.perf_counter() logger.info("PlotWindow __init__") - self.setMinimumWidth(850) self.setMinimumHeight(650) - self.setWindowTitle(f"Plotting - {config_file}") self.activateWindow() @@ -109,6 +108,8 @@ def __init__(self, config_file, parent): self._data_type_keys_widget.selectDefault() self._updateCustomizer(current_plot_widget) + logger.info(f"PlotWindow __init__ done. time={time.perf_counter() -t}") + def currentPlotChanged(self): key_def = self.getSelectedKey() if key_def is None: diff --git a/src/ert/storage/local_ensemble.py b/src/ert/storage/local_ensemble.py index 5176b70f946..9f5fda0ce56 100644 --- a/src/ert/storage/local_ensemble.py +++ b/src/ert/storage/local_ensemble.py @@ -105,8 +105,10 @@ def get_realization_mask_without_parent_failure(self) -> npt.NDArray[np.bool_]: def get_realization_mask_with_parameters(self) -> npt.NDArray[np.bool_]: return np.array([self._get_parameter(i) for i in range(self.ensemble_size)]) - def get_realization_mask_with_responses(self) -> npt.NDArray[np.bool_]: - return np.array([self._get_response(i) for i in range(self.ensemble_size)]) + def get_realization_mask_with_responses( + self, key: Optional[str] = None + ) -> npt.NDArray[np.bool_]: + return np.array([self._get_response(i, key) for i in range(self.ensemble_size)]) def _get_parameter(self, realization: int) -> bool: if not self.experiment.parameter_configuration: @@ -117,10 +119,14 @@ def _get_parameter(self, realization: int) -> bool: for parameter in self.experiment.parameter_configuration ) - def _get_response(self, realization: int) -> bool: + def _get_response(self, realization: int, key: Optional[str] = None) -> bool: if not self.experiment.response_configuration: return False path = self.mount_point / f"realization-{realization}" + + if key: + return (path / f"{key}.nc").exists() + return all( (path / f"{response}.nc").exists() for response in self._filter_response_configuration() @@ -180,9 +186,13 @@ def realizations_initialized(self, realizations: List[int]) -> bool: return all((responses[real] or parameters[real]) for real in realizations) - def get_realization_list_with_responses(self) -> List[int]: + def get_realization_list_with_responses( + self, key: Optional[str] = None + ) -> List[int]: return [ - idx for idx, b in enumerate(self.get_realization_mask_with_responses()) if b + idx + for idx, b in enumerate(self.get_realization_mask_with_responses(key)) + if b ] def set_failure( @@ -253,20 +263,14 @@ def _get_gen_data_config(self, key: str) -> GenDataConfig: @deprecated("Check the experiment for registered responses") def get_gen_data_keyset(self) -> List[str]: - keylist = [ - k - for k, v in self.experiment.response_info.items() - if "_ert_kind" in v and v["_ert_kind"] == "GenDataConfig" - ] - gen_data_list = [] - for key in keylist: - gen_data_config = self._get_gen_data_config(key) - if gen_data_config.report_steps is None: - gen_data_list.append(f"{key}@0") - else: - for report_step in gen_data_config.report_steps: - gen_data_list.append(f"{key}@{report_step}") + for k, v in self.experiment.response_configuration.items(): + if isinstance(v, GenDataConfig): + if v.report_steps is None: + gen_data_list.append(f"{k}@0") + else: + for report_step in v.report_steps: + gen_data_list.append(f"{k}@{report_step}") return sorted(gen_data_list, key=lambda k: k.lower()) @deprecated("Check the experiment for registered parameters") @@ -293,7 +297,7 @@ def load_gen_data( report_step: int, realization_index: Optional[int] = None, ) -> pd.DataFrame: - realizations = self.get_realization_list_with_responses() + realizations = self.get_realization_list_with_responses(key) if realization_index is not None: if realization_index not in realizations: raise IndexError(f"No such realization {realization_index}") @@ -368,6 +372,31 @@ def load_responses( assert isinstance(response, xr.Dataset) return response + def load_responses_summary(self, key: str) -> xr.Dataset: + loaded = [] + for realization in [i for i in range(self.ensemble_size)]: + input_path = self.mount_point / f"realization-{realization}" / "summary.nc" + if input_path.exists(): + ds = xr.open_dataset(input_path, engine="scipy") + ds = ds.query(name=f'name=="{key}"') + loaded.append(ds) + response = xr.combine_nested(loaded, concat_dim="realization") + assert isinstance(response, xr.Dataset) + return response + + def load_summary(self, key: str) -> pd.DataFrame: + try: + df = self.load_responses_summary(key).to_dataframe() + except (ValueError, KeyError): + return pd.DataFrame() + + df = df.unstack(level="name") + df.columns = [col[1] for col in df.columns.values] + df.index = df.index.rename( + {"time": "Date", "realization": "Realization"} + ).reorder_levels(["Realization", "Date"]) + return df + @deprecated("Use load_responses") def load_all_summary_data( self, @@ -386,6 +415,7 @@ def load_all_summary_data( df = self.load_responses("summary", tuple(realizations)).to_dataframe() except (ValueError, KeyError): return pd.DataFrame() + # df= df.query(f'name == "{key}"') df = df.unstack(level="name") df.columns = [col[1] for col in df.columns.values] df.index = df.index.rename( diff --git a/tests/performance_tests/performance_utils.py b/tests/performance_tests/performance_utils.py index 772a84c204a..b30b4968549 100644 --- a/tests/performance_tests/performance_utils.py +++ b/tests/performance_tests/performance_utils.py @@ -152,9 +152,9 @@ def dark_storage_app(monkeypatch): folder = py.path.local(tempfile.mkdtemp()) make_poly_example( folder, - "../../test-data/poly_template", - gen_data_count=34, - gen_data_entries=15, + "test-data/poly_template", + gen_data_count=3400, + gen_data_entries=150, summary_data_entries=100, reals=200, summary_data_count=4000, @@ -163,7 +163,7 @@ def dark_storage_app(monkeypatch): sum_obs_every=10, gen_obs_every=1, parameter_entries=10, - parameter_count=8, + parameter_count=10, update_steps=1, ) print(folder) diff --git a/tests/unit_tests/dark_storage/test_http_endpoints.py b/tests/unit_tests/dark_storage/test_http_endpoints.py index d005fed6f53..909a679db36 100644 --- a/tests/unit_tests/dark_storage/test_http_endpoints.py +++ b/tests/unit_tests/dark_storage/test_http_endpoints.py @@ -134,14 +134,6 @@ def test_get_response(poly_example_tmp_dir, dark_storage_client): assert len(record_df1.columns) == 10 assert len(record_df1.index) == 3 - resp: Response = dark_storage_client.get( - f"/ensembles/{ensemble_id1}/records/POLY_RES@0?realization_index=2" - ) - stream = io.BytesIO(resp.content) - record_df1_indexed = pd.read_csv(stream, index_col=0, float_precision="round_trip") - assert len(record_df1_indexed.columns) == 10 - assert len(record_df1_indexed.index) == 1 - def test_get_ensemble_parameters(poly_example_tmp_dir, dark_storage_client): resp: Response = dark_storage_client.get("/experiments") @@ -220,18 +212,6 @@ def test_misfit_endpoint(poly_example_tmp_dir, dark_storage_client): assert misfit.shape == (3, 5) -def test_get_record_labels(poly_example_tmp_dir, dark_storage_client): - resp: Response = dark_storage_client.get("/experiments") - answer_json = resp.json() - ensemble_id = answer_json[0]["ensemble_ids"][0] - resp: Response = dark_storage_client.get( - f"/ensembles/{ensemble_id}/records/POLY_RES@0/labels" - ) - labels = resp.json() - - assert labels == [] - - @pytest.mark.parametrize( "coeffs", [ diff --git a/tests/unit_tests/gui/tools/plot/conftest.py b/tests/unit_tests/gui/tools/plot/conftest.py index 9b9d043c5d2..ef1a9ba1a1b 100644 --- a/tests/unit_tests/gui/tools/plot/conftest.py +++ b/tests/unit_tests/gui/tools/plot/conftest.py @@ -87,10 +87,10 @@ def mocked_requests_get(*args, **kwargs): history_parquet_data = history_stream.getvalue() ensemble = { - "/ensembles/ens_id_1": {"name": "ensemble_1"}, - "/ensembles/ens_id_2": {"name": ".ensemble_2"}, - "/ensembles/ens_id_3": {"name": "default_0"}, - "/ensembles/ens_id_4": {"name": "default_1"}, + "/ensembles/ens_id_1/small": {"name": "ensemble_1"}, + "/ensembles/ens_id_2/small": {"name": ".ensemble_2"}, + "/ensembles/ens_id_3/small": {"name": "default_0"}, + "/ensembles/ens_id_4/small": {"name": "default_1"}, } observations = { "/ensembles/ens_id_3/records/WOPR:OP1/observations": { diff --git a/tests/unit_tests/gui/tools/plot/test_plot_api.py b/tests/unit_tests/gui/tools/plot/test_plot_api.py index 0448d85a1e5..60163aa3ec0 100644 --- a/tests/unit_tests/gui/tools/plot/test_plot_api.py +++ b/tests/unit_tests/gui/tools/plot/test_plot_api.py @@ -3,6 +3,8 @@ import pytest from pandas.testing import assert_frame_equal +from tests.unit_tests.gui.tools.plot.conftest import MockResponse + def test_key_def_structure(api): key_defs = api.all_data_type_keys() @@ -102,15 +104,18 @@ def test_load_history_data(api): ) -def test_plot_api_request_errors(api, mocker): +def test_plot_api_request_errors_all_data_type_keys(api, mocker): # Mock the experiment name to be something unexpected mocker.patch( - "ert.gui.tools.plot.plot_api.PlotApi._get_experiments", - return_value=[{"id": "mocked"}], + "tests.unit_tests.gui.tools.plot.conftest.mocked_requests_get", + return_value=MockResponse(None, 404, text="error"), ) + with pytest.raises(httpx.RequestError): api.all_data_type_keys() + +def test_plot_api_request_errors(api): case_name = "default_0" with pytest.raises(httpx.RequestError): api.observations_for_key(case_name, "should_not_be_there")