diff --git a/src/database.py b/src/database.py index 51be1c6c..7ef3b37a 100644 --- a/src/database.py +++ b/src/database.py @@ -40,7 +40,7 @@ convert_forecasts_to_many_datetime_many_generation, convert_location_sql_to_many_datetime_many_generation, ) -from utils import floor_30_minutes_dt, get_start_datetime +from utils import filter_forecast_values, floor_30_minutes_dt, get_start_datetime logger = structlog.stdlib.get_logger() @@ -122,14 +122,17 @@ def get_forecasts_from_database( else: # To speed up read time we only look at the last 12 hours of results, and take floor 30 mins - yesterday_start_datetime = floor_30_minutes_dt( - datetime.now(tz=timezone.utc) - timedelta(hours=12) - ) + if start_datetime_utc is None: + start_datetime_utc = floor_30_minutes_dt( + datetime.now(tz=timezone.utc) - timedelta(hours=12) + ) + + start_created_utc = floor_30_minutes_dt(datetime.now(tz=timezone.utc) - timedelta(hours=12)) forecasts = get_all_gsp_ids_latest_forecast( session=session, - start_created_utc=yesterday_start_datetime, - start_target_time=yesterday_start_datetime, + start_created_utc=start_created_utc, + start_target_time=start_datetime_utc, preload_children=True, model_name="blend", end_target_time=end_datetime_utc, @@ -137,7 +140,10 @@ def get_forecasts_from_database( if compact: return convert_forecasts_to_many_datetime_many_generation( - forecasts=forecasts, historic=historic + forecasts=forecasts, + historic=historic, + start_datetime_utc=start_datetime_utc, + end_datetime_utc=end_datetime_utc, ) else: @@ -147,6 +153,12 @@ def get_forecasts_from_database( else: forecasts = [Forecast.from_orm(forecast) for forecast in forecasts] + forecasts = filter_forecast_values( + end_datetime_utc=end_datetime_utc, + forecasts=forecasts, + start_datetime_utc=start_datetime_utc, + ) + # return as many forecasts return ManyForecasts(forecasts=forecasts) diff --git a/src/gsp.py b/src/gsp.py index 3c99dd29..8e734677 100644 --- a/src/gsp.py +++ b/src/gsp.py @@ -1,5 +1,4 @@ """Get GSP boundary data from eso """ -from datetime import datetime from typing import List, Optional, Union import structlog @@ -24,6 +23,7 @@ LocationWithGSPYields, OneDatetimeManyForecastValues, ) +from utils import format_datetime GSP_TOTAL = 317 @@ -47,8 +47,8 @@ def get_all_available_forecasts( historic: Optional[bool] = True, session: Session = Depends(get_session), user: Auth0User = Security(get_user()), - start_datetime_utc: Optional[datetime] = None, - end_datetime_utc: Optional[datetime] = None, + start_datetime_utc: Optional[str] = None, + end_datetime_utc: Optional[str] = None, compact: Optional[bool] = False, ) -> Union[ManyForecasts, List[OneDatetimeManyForecastValues]]: """### Get all forecasts for all GSPs @@ -65,12 +65,15 @@ def get_all_available_forecasts( #### Parameters - **historic**: boolean that defaults to `true`, returning yesterday's and today's forecasts for all GSPs - - **start_datetime_utc**: optional start datetime for the query. - - **end_datetime_utc**: optional end datetime for the query. + - **start_datetime_utc**: optional start datetime for the query. e.g '2023-08-12 10:00:00+00:00' + - **end_datetime_utc**: optional end datetime for the query. e.g '2023-08-12 14:00:00+00:00' """ logger.info(f"Get forecasts for all gsps. The option is {historic=} for user {user}") + start_datetime_utc = format_datetime(start_datetime_utc) + end_datetime_utc = format_datetime(end_datetime_utc) + forecasts = get_forecasts_from_database( session=session, historic=historic, @@ -128,8 +131,8 @@ def get_forecasts_for_a_specific_gsp( session: Session = Depends(get_session), forecast_horizon_minutes: Optional[int] = None, user: Auth0User = Security(get_user()), - start_datetime_utc: Optional[datetime] = None, - end_datetime_utc: Optional[datetime] = None, + start_datetime_utc: Optional[str] = None, + end_datetime_utc: Optional[str] = None, ) -> Union[Forecast, List[ForecastValue]]: """### Get recent forecast values for a specific GSP @@ -155,6 +158,9 @@ def get_forecasts_for_a_specific_gsp( logger.info(f"Get forecasts for gsp id {gsp_id} forecast of forecast with only values.") logger.info(f"This is for user {user}") + start_datetime_utc = format_datetime(start_datetime_utc) + end_datetime_utc = format_datetime(end_datetime_utc) + if gsp_id > GSP_TOTAL: return Response(None, status.HTTP_204_NO_CONTENT) @@ -183,8 +189,8 @@ def get_truths_for_all_gsps( regime: Optional[str] = None, session: Session = Depends(get_session), user: Auth0User = Security(get_user()), - start_datetime_utc: Optional[datetime] = None, - end_datetime_utc: Optional[datetime] = None, + start_datetime_utc: Optional[str] = None, + end_datetime_utc: Optional[str] = None, compact: Optional[bool] = False, ) -> Union[List[LocationWithGSPYields], List[GSPYieldGroupByDatetime]]: """### Get PV_Live values for all GSPs for yesterday and today @@ -207,6 +213,9 @@ def get_truths_for_all_gsps( """ logger.info(f"Get PV Live estimates values for all gsp id and regime {regime} for user {user}") + start_datetime_utc = format_datetime(start_datetime_utc) + end_datetime_utc = format_datetime(end_datetime_utc) + return get_truth_values_for_all_gsps_from_database( session=session, regime=regime, @@ -253,8 +262,8 @@ def get_truths_for_a_specific_gsp( request: Request, gsp_id: int, regime: Optional[str] = None, - start_datetime_utc: Optional[datetime] = None, - end_datetime_utc: Optional[datetime] = None, + start_datetime_utc: Optional[str] = None, + end_datetime_utc: Optional[str] = None, session: Session = Depends(get_session), user: Auth0User = Security(get_user()), ) -> List[GSPYield]: @@ -280,6 +289,9 @@ def get_truths_for_a_specific_gsp( f"Get PV Live estimates values for gsp id {gsp_id} " f"and regime {regime} for user {user}" ) + start_datetime_utc = format_datetime(start_datetime_utc) + end_datetime_utc = format_datetime(end_datetime_utc) + if gsp_id > GSP_TOTAL: return Response(None, status.HTTP_204_NO_CONTENT) diff --git a/src/pydantic_models.py b/src/pydantic_models.py index cbbc6d11..e74e3fa1 100644 --- a/src/pydantic_models.py +++ b/src/pydantic_models.py @@ -109,7 +109,10 @@ def convert_location_sql_to_many_datetime_many_generation( def convert_forecasts_to_many_datetime_many_generation( - forecasts: List[ForecastSQL], historic: bool = True + forecasts: List[ForecastSQL], + historic: bool = True, + start_datetime_utc: Optional[datetime] = None, + end_datetime_utc: Optional[datetime] = None, ) -> List[OneDatetimeManyForecastValues]: """Change forecasts to list of OneDatetimeManyForecastValues @@ -134,6 +137,10 @@ def convert_forecasts_to_many_datetime_many_generation( for forecast_value in forecast_values: datetime_utc = forecast_value.target_time + if start_datetime_utc is not None and datetime_utc < start_datetime_utc: + continue + if end_datetime_utc is not None and datetime_utc > end_datetime_utc: + continue forecast_mw = round(forecast_value.expected_power_generation_megawatts, 2) # if the datetime object is not in the dictionary, add it diff --git a/src/tests/test_gsp.py b/src/tests/test_gsp.py index 9a924c27..c01a8460 100644 --- a/src/tests/test_gsp.py +++ b/src/tests/test_gsp.py @@ -60,7 +60,7 @@ def test_read_latest_all_gsp(db_session, api_client): r = ManyForecasts(**response.json()) assert len(r.forecasts) == 10 - assert len(r.forecasts[0].forecast_values) == 112 + assert len(r.forecasts[0].forecast_values) == 40 def test_read_latest_gsp_id_greater_than_total(db_session, api_client): diff --git a/src/utils.py b/src/utils.py index a2837ca2..fc0d60c5 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,11 +1,11 @@ """ Utils functions for main.py """ import os from datetime import datetime, timedelta -from typing import Optional, Union +from typing import List, Optional, Union import numpy as np import structlog -from nowcasting_datamodel.models import ForecastValue +from nowcasting_datamodel.models import Forecast, ForecastValue from pydantic import Field from pytz import timezone @@ -64,6 +64,24 @@ def floor_6_hours_dt(dt: datetime): return dt +def format_datetime(datetime_str: str = None): + """ + Format datetime string to datetime object + + If None return None, if not timezone, add UTC + :param datetime_str: + :return: + """ + if datetime_str is None: + return None + + else: + datetime_output = datetime.fromisoformat(datetime_str) + if datetime_output.tzinfo is None: + datetime_output = utc.localize(datetime_output) + return datetime_output + + def get_start_datetime( n_history_days: Optional[Union[str, int]] = None, start_datetime: Optional[datetime] = None ) -> datetime: @@ -80,11 +98,9 @@ def get_start_datetime( :return: start datetime """ - if ( - start_datetime is None - or start_datetime >= datetime.now(tz=timezone.utc) - or datetime.now(tz=timezone.utc) - start_datetime > timedelta(days=3) - ): + now = datetime.now(tz=utc) + + if start_datetime is None or now - start_datetime > timedelta(days=3): if n_history_days is None: n_history_days = os.getenv("N_HISTORY_DAYS", "yesterday") @@ -163,3 +179,40 @@ def format_plevels(national_forecast_value: NationalForecastValue): national_forecast_value.plevels["plevel_90"] = ( national_forecast_value.expected_power_generation_megawatts * 1.2 ) + + +def filter_forecast_values( + forecasts: List[Forecast], + end_datetime_utc: Optional[datetime] = None, + start_datetime_utc: Optional[datetime] = None, +) -> List[Forecast]: + """ + Filter forecast values by start and end datetime + + :param forecasts: list of forecasts + :param end_datetime_utc: start datetime + :param start_datetime_utc: e + :return: + """ + if start_datetime_utc is not None or end_datetime_utc is not None: + logger.info(f"Filtering forecasts from {start_datetime_utc} to {end_datetime_utc}") + forecasts_filtered = [] + for forecast in forecasts: + forecast_values = forecast.forecast_values + if start_datetime_utc is not None: + forecast_values = [ + forecast_value + for forecast_value in forecast_values + if forecast_value.target_time >= start_datetime_utc + ] + if end_datetime_utc is not None: + forecast_values = [ + forecast_value + for forecast_value in forecast_values + if forecast_value.target_time <= end_datetime_utc + ] + forecast.forecast_values = forecast_values + + forecasts_filtered.append(forecast) + forecasts = forecasts_filtered + return forecasts