Improve plotting performance

src/ert/dark_storage/common.py

@@ -30,7 +30,6 @@ def get_response_names(ensemble: EnsembleReader) -> List[str]:
 def data_for_key(
     ensemble: EnsembleReader,
     key: str,
-    realization_index: Optional[int] = None,
 ) -> pd.DataFrame:
     """Returns a pandas DataFrame with the datapoints for a given key for a
     given case. The row index is the realization number, and the columns are an
@@ -39,10 +38,10 @@ def data_for_key(
     if key.startswith("LOG10_"):
         key = key[6:]
     if key in ensemble.get_summary_keyset():
-        data = ensemble.load_all_summary_data([key], realization_index)
+        data = ensemble.load_summary(key)
         data = data[key].unstack(level="Date")
     elif key in ensemble.get_gen_kw_keyset():
-        data = ensemble.load_all_gen_kw_data(key.split(":")[0], realization_index)
+        data = ensemble.load_all_gen_kw_data(key.split(":")[0])
         if data.empty:
             return pd.DataFrame()
         data = data[key].to_frame().dropna()
@@ -56,7 +55,6 @@ def data_for_key(
             data = ensemble.load_gen_data(
                 key,
                 report_step,
-                realization_index,
             ).T
         except (ValueError, KeyError):
             return pd.DataFrame()

src/ert/dark_storage/endpoints/ensembles.py

@@ -1,4 +1,3 @@
-from typing import Any, Mapping
 from uuid import UUID
 
 from fastapi import APIRouter, Body, Depends
@@ -13,15 +12,6 @@
 DEFAULT_BODY = Body(...)
 
 
-@router.post("/experiments/{experiment_id}/ensembles", response_model=js.EnsembleOut)
-def post_ensemble(
-    *,
-    ens_in: js.EnsembleIn,
-    experiment_id: UUID,
-) -> js.EnsembleOut:
-    raise NotImplementedError
-
-
 @router.get("/ensembles/{ensemble_id}", response_model=js.EnsembleOut)
 def get_ensemble(
     *,
@@ -42,27 +32,21 @@ def get_ensemble(
     )
 
 
-@router.put("/ensembles/{ensemble_id}/userdata")
-async def replace_ensemble_userdata(
-    *,
-    ensemble_id: UUID,
-    body: Any = DEFAULT_BODY,
-) -> None:
-    raise NotImplementedError
-
-
-@router.patch("/ensembles/{ensemble_id}/userdata")
-async def patch_ensemble_userdata(
-    *,
-    ensemble_id: UUID,
-    body: Any = DEFAULT_BODY,
-) -> None:
-    raise NotImplementedError
-
-
-@router.get("/ensembles/{ensemble_id}/userdata", response_model=Mapping[str, Any])
-async def get_ensemble_userdata(
+@router.get("/ensembles/{ensemble_id}/small", response_model=js.EnsembleOut)
+def get_ensemble_small(
     *,
+    storage: StorageAccessor = DEFAULT_STORAGE,
     ensemble_id: UUID,
-) -> Mapping[str, Any]:
-    raise NotImplementedError
+) -> js.EnsembleOut:
+    ensemble = storage.get_ensemble(ensemble_id)
+    return js.EnsembleOut(
+        id=ensemble_id,
+        children=[],
+        parent=None,
+        experiment_id=ensemble.experiment_id,
+        userdata={"name": ensemble.name},
+        size=ensemble.ensemble_size,
+        parameter_names=[],
+        response_names=[],
+        child_ensemble_ids=[],
+    )

src/ert/dark_storage/endpoints/records.py

@@ -1,10 +1,9 @@
 import io
 from itertools import chain
-from typing import Any, Dict, List, Mapping, Optional, Union
+from typing import Any, Dict, List, Mapping, Union
 from uuid import UUID, uuid4
 
-import pandas as pd
-from fastapi import APIRouter, Body, Depends, File, Header, Request, UploadFile, status
+from fastapi import APIRouter, Body, Depends, File, Header, status
 from fastapi.responses import Response
 from typing_extensions import Annotated
 
@@ -27,117 +26,11 @@
 DEFAULT_HEADER = Header("application/json")
 
 
-@router.post("/ensembles/{ensemble_id}/records/{name}/file")
-async def post_ensemble_record_file(
-    *,
-    name: str,
-    ensemble_id: UUID,
-    realization_index: Optional[int] = None,
-    file: UploadFile = DEFAULT_FILE,
-) -> None:
-    raise NotImplementedError
-
-
-@router.put("/ensembles/{ensemble_id}/records/{name}/blob")
-async def add_block(
-    *,
-    name: str,
-    ensemble_id: UUID,
-    block_index: int,
-    realization_index: Optional[int] = None,
-    request: Request,
-) -> None:
-    raise NotImplementedError
-
-
-@router.post("/ensembles/{ensemble_id}/records/{name}/blob")
-async def create_blob(
-    *,
-    name: str,
-    ensemble_id: UUID,
-    realization_index: Optional[int] = None,
-) -> None:
-    raise NotImplementedError
-
-
-@router.patch("/ensembles/{ensemble_id}/records/{name}/blob")
-async def finalize_blob(
-    *,
-    name: str,
-    ensemble_id: UUID,
-    realization_index: Optional[int] = None,
-) -> None:
-    raise NotImplementedError
-
-
-@router.post(
-    "/ensembles/{ensemble_id}/records/{name}/matrix", response_model=js.RecordOut
-)
-async def post_ensemble_record_matrix(
-    *,
-    ensemble_id: UUID,
-    name: str,
-    prior: Optional[str] = None,
-    realization_index: Optional[int] = None,
-    content_type: str = DEFAULT_HEADER,
-    request: Request,
-) -> js.RecordOut:
-    raise NotImplementedError
-
-
-@router.put("/ensembles/{ensemble_id}/records/{name}/userdata")
-async def replace_record_userdata(
-    *,
-    ensemble_id: UUID,
-    name: str,
-    realization_index: Optional[int] = None,
-    body: Any = DEFAULT_BODY,
-) -> None:
-    raise NotImplementedError
-
-
-@router.patch("/ensembles/{ensemble_id}/records/{name}/userdata")
-async def patch_record_userdata(
-    *,
-    ensemble_id: UUID,
-    name: str,
-    realization_index: Optional[int] = None,
-    body: Any = DEFAULT_BODY,
-) -> None:
-    raise NotImplementedError
-
-
-@router.get(
-    "/ensembles/{ensemble_id}/records/{name}/userdata", response_model=Mapping[str, Any]
-)
-async def get_record_userdata(
-    *,
-    ensemble_id: UUID,
-    name: str,
-    realization_index: Optional[int] = None,
-) -> Mapping[str, Any]:
-    raise NotImplementedError
-
-
-@router.post("/ensembles/{ensemble_id}/records/{name}/observations")
-async def post_record_observations(
-    *,
-    ensemble_id: UUID,
-    name: str,
-    realization_index: Optional[int] = None,
-    observation_ids: List[UUID] = DEFAULT_BODY,
-) -> None:
-    raise NotImplementedError
-
-
 @router.get("/ensembles/{ensemble_id}/records/{name}/observations")
 async def get_record_observations(
     *,
     res: LibresFacade = DEFAULT_LIBRESFACADE,
-    db: StorageReader = DEFAULT_STORAGE,
-    ensemble_id: UUID,
     name: str,
-    realization_index: Optional[int] = None,
 ) -> List[js.ObservationOut]:
     obs_keys = res.observation_keys(name)
     obss = observations_for_obs_keys(res, obs_keys)
@@ -174,15 +67,8 @@ async def get_ensemble_record(
     name: str,
     ensemble_id: UUID,
     accept: Annotated[Union[str, None], Header()] = None,
-    realization_index: Optional[int] = None,
-    label: Optional[str] = None,
 ) -> Any:
-    dataframe = data_for_key(db.get_ensemble(ensemble_id), name, realization_index)
-    if realization_index is not None:
-        # dataframe.loc returns a Series, and when we reconstruct a DataFrame
-        # from a Series, it defaults to be oriented the wrong way, so we must
-        # transpose it
-        dataframe = pd.DataFrame(dataframe.loc[realization_index]).T
+    dataframe = data_for_key(db.get_ensemble(ensemble_id), name)
 
     media_type = accept if accept is not None else "text/csv"
     if media_type == "application/x-parquet":
@@ -202,43 +88,13 @@ async def get_ensemble_record(
         )
 
 
-@router.get("/ensembles/{ensemble_id}/records/{name}/labels", response_model=List[str])
-async def get_record_labels(
-    *,
-    ensemble_id: UUID,
-    name: str,
-) -> List[str]:
-    return []
-
-
 @router.get("/ensembles/{ensemble_id}/parameters", response_model=List[Dict[str, Any]])
 async def get_ensemble_parameters(
     *, storage: StorageReader = DEFAULT_STORAGE, ensemble_id: UUID
 ) -> List[Dict[str, Any]]:
     return ensemble_parameters(storage, ensemble_id)
 
 
-@router.get(
-    "/ensembles/{ensemble_id}/records", response_model=Mapping[str, js.RecordOut]
-)
-async def get_ensemble_records(*, ensemble_id: UUID) -> Mapping[str, js.RecordOut]:
-    raise NotImplementedError
-
-
-@router.get("/records/{record_id}", response_model=js.RecordOut)
-async def get_record(*, record_id: UUID) -> js.RecordOut:
-    raise NotImplementedError
-
-
-@router.get("/records/{record_id}/data")
-async def get_record_data(
-    *,
-    record_id: UUID,
-    accept: Optional[str] = DEFAULT_HEADER,
-) -> Any:
-    raise NotImplementedError
-
-
 @router.get(
     "/ensembles/{ensemble_id}/responses", response_model=Mapping[str, js.RecordOut]
 )
@@ -249,15 +105,18 @@ def get_ensemble_responses(
     ensemble_id: UUID,
 ) -> Mapping[str, js.RecordOut]:
     response_map: Dict[str, js.RecordOut] = {}
-
     ens = db.get_ensemble(ensemble_id)
+    name_dict = {}
+
+    for obs in res.get_observations():
+        name_dict[obs.observation_key] = obs.observation_type
+
     for name in ens.get_summary_keyset():
-        obs_keys = res.observation_keys(name)
         response_map[str(name)] = js.RecordOut(
             id=UUID(int=0),
             name=name,
             userdata={"data_origin": "Summary"},
-            has_observations=len(obs_keys) != 0,
+            has_observations=name in name_dict,
         )
 
     for name in res.get_gen_data_keys():

src/ert/gui/tools/plot/plot_api.py

@@ -43,7 +43,7 @@ def _get_all_cases(self) -> List[dict]:
                 for experiment in experiments:
                     for ensemble_id in experiment["ensemble_ids"]:
                         response = client.get(
-                            f"/ensembles/{ensemble_id}", timeout=self._timeout
+                            f"/ensembles/{ensemble_id}/small", timeout=self._timeout
                         )
                         self._check_response(response)
                         response_json = response.json()
@@ -68,23 +68,6 @@ def _check_response(response: requests.Response):
                 f"{response.text} from url: {response.url}."
             )
 
-    def _get_experiments(self) -> dict:
-        with StorageService.session() as client:
-            response: requests.Response = client.get(
-                "/experiments", timeout=self._timeout
-            )
-            self._check_response(response)
-            return response.json()
-
-    def _get_ensembles(self, experiement_id) -> List:
-        with StorageService.session() as client:
-            response: requests.Response = client.get(
-                f"/experiments/{experiement_id}/ensembles", timeout=self._timeout
-            )
-            self._check_response(response)
-            response_json = response.json()
-            return response_json
-
     def all_data_type_keys(self) -> List:
         """Returns a list of all the keys except observation keys.
 
@@ -94,9 +77,20 @@ def all_data_type_keys(self) -> List:
         the key"""
 
         all_keys = {}
+
         with StorageService.session() as client:
-            for experiment in self._get_experiments():
-                for ensemble in self._get_ensembles(experiment["id"]):
+            response: requests.Response = client.get(
+                "/experiments", timeout=self._timeout
+            )
+            self._check_response(response)
+
+            for experiment in response.json():
+                response: requests.Response = client.get(
+                    f"/experiments/{experiment['id']}/ensembles", timeout=self._timeout
+                )
+                self._check_response(response)
+
+                for ensemble in response.json():
                     response: requests.Response = client.get(
                         f"/ensembles/{ensemble['id']}/responses", timeout=self._timeout
                     )
@@ -133,7 +127,7 @@ def get_all_cases_not_running(self) -> List:
         info about the case is returned"""
         # Currently, the ensemble information from the storage API does not contain any
         # hint if a case is running or not for now we return all the cases, running or
-        # not
+        # no
         return self._get_all_cases()
 
     def data_for_key(self, case_name, key) -> pd.DataFrame: