From e125c2ad78a8da49be61319cc31e4dd5e9585bf6 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Wed, 17 Jul 2024 15:23:02 +0530 Subject: [PATCH 01/26] added import statements of elexonpy --- src/national.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/national.py b/src/national.py index 09457c6..2396b38 100644 --- a/src/national.py +++ b/src/national.py @@ -1,9 +1,11 @@ """National API routes""" import os +from datetime import datetime from typing import List, Optional, Union import structlog +import pandas as pd from fastapi import APIRouter, Depends, HTTPException, Request, Security from fastapi_auth0 import Auth0User from nowcasting_datamodel.read.read import get_latest_forecast_for_gsps @@ -16,12 +18,13 @@ get_session, get_truth_values_for_a_specific_gsp_from_database, ) +from elexonpy.api_client import ApiClient +from elexonpy.api.generation_forecast_api import GenerationForecastApi from pydantic_models import NationalForecast, NationalForecastValue, NationalYield from utils import N_CALLS_PER_HOUR, filter_forecast_values, format_datetime, format_plevels, limiter logger = structlog.stdlib.get_logger() - adjust_limit = float(os.getenv("ADJUST_MW_LIMIT", 0.0)) get_plevels = bool(os.getenv("GET_PLEVELS", True)) @@ -29,6 +32,9 @@ tags=["National"], ) +# Initialize Elexon API client +api_client = ApiClient() +forecast_api = GenerationForecastApi(api_client) @router.get( "/forecast", @@ -188,4 +194,4 @@ def get_national_pvlive( return get_truth_values_for_a_specific_gsp_from_database( session=session, gsp_id=0, regime=regime - ) + ) \ No newline at end of file From 1ab383722c5f23c76b3fabca877924a88aaa1912 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 17 Jul 2024 09:55:33 +0000 Subject: [PATCH 02/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/national.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/national.py b/src/national.py index 2396b38..74f67d0 100644 --- a/src/national.py +++ b/src/national.py @@ -36,6 +36,7 @@ api_client = ApiClient() forecast_api = GenerationForecastApi(api_client) + @router.get( "/forecast", response_model=Union[NationalForecast, List[NationalForecastValue]], @@ -194,4 +195,4 @@ def get_national_pvlive( return get_truth_values_for_a_specific_gsp_from_database( session=session, gsp_id=0, regime=regime - ) \ No newline at end of file + ) From bae31a90a5a6dd8d7c58a074f4f373b02db22b37 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Thu, 18 Jul 2024 23:52:38 +0530 Subject: [PATCH 03/26] added API for solar forecast --- src/main.py | 3 +-- src/national.py | 42 ++++++++++++++++++------------------------ 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/src/main.py b/src/main.py index d401d35..a6fac5f 100644 --- a/src/main.py +++ b/src/main.py @@ -19,7 +19,6 @@ from status import router as status_router from system import router as system_router from utils import limiter, traces_sampler - # flake8: noqa E501 structlog.configure( @@ -208,7 +207,7 @@ async def add_process_time_header(request: Request, call_next): # Dependency v0_route_solar = "/v0/solar/GB" v0_route_system = "/v0/system/GB" - +app.include_router(national_router, prefix="/v0/solar/national") app.include_router(national_router, prefix=f"{v0_route_solar}/national") app.include_router(gsp_router, prefix=f"{v0_route_solar}/gsp") app.include_router(status_router, prefix=f"{v0_route_solar}") diff --git a/src/national.py b/src/national.py index 34a561f..98df5a7 100644 --- a/src/national.py +++ b/src/national.py @@ -1,12 +1,12 @@ """National API routes""" import os -from datetime import datetime +from datetime import datetime, timedelta, timezone from typing import List, Optional, Union import structlog import pandas as pd -from fastapi import APIRouter, Depends, HTTPException, Request, Security +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Security from fastapi_auth0 import Auth0User from nowcasting_datamodel.read.read import get_latest_forecast_for_gsps from sqlalchemy.orm.session import Session @@ -18,10 +18,11 @@ get_session, get_truth_values_for_a_specific_gsp_from_database, ) -from elexonpy.api_client import ApiClient -from elexonpy.api.generation_forecast_api import GenerationForecastApi + from pydantic_models import NationalForecast, NationalForecastValue, NationalYield from utils import N_CALLS_PER_HOUR, filter_forecast_values, format_datetime, format_plevels, limiter +from elexonpy.api_client import ApiClient +from elexonpy.api.generation_forecast_api import GenerationForecastApi logger = structlog.stdlib.get_logger() @@ -198,26 +199,19 @@ def get_national_pvlive( ) -@router.get( - "/bmrs", - response_model=dict, - # dependencies=[Depends(get_auth_implicit_scheme())], - summary="Get BMRS Forecast", -) -# @cache_response -@limiter.limit(f"{N_CALLS_PER_HOUR}/hour") +@router.get("/bmrs", summary="Get BMRS Forecast") def get_bmrs_forecast( - request: Request, - # session: Session = Depends(get_session), - # user: Auth0User = Security(get_user()), -) -> dict: - """ - - This route returns the most recent BMRS forecast for each _target_time_. - - #### Parameters + start_datetime_utc: datetime = Query(default=datetime.utcnow() - timedelta(days=3), description="Start date and time in UTC"), + end_datetime_utc: datetime = Query(default=datetime.utcnow() + timedelta(days=3), description="End date and time in UTC"), + process_type: str = Query("Day Ahead", description="Process type") +): + response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( + _from=start_datetime_utc, + to=end_datetime_utc, + process_type=process_type, + format='json' + ) - """ - logger.debug("Get bmrs forecast") + df = pd.DataFrame([item.to_dict() for item in response.data]) - return {"message": "This route is not yet implemented. Please check back later."} + return {"data": df.to_dict(orient="records")} From 91588007730273711a5738c486a10195496094cf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 18:22:57 +0000 Subject: [PATCH 04/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/main.py | 1 + src/national.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main.py b/src/main.py index a6fac5f..9cdcf66 100644 --- a/src/main.py +++ b/src/main.py @@ -19,6 +19,7 @@ from status import router as status_router from system import router as system_router from utils import limiter, traces_sampler + # flake8: noqa E501 structlog.configure( diff --git a/src/national.py b/src/national.py index 98df5a7..f813d8e 100644 --- a/src/national.py +++ b/src/national.py @@ -201,15 +201,16 @@ def get_national_pvlive( @router.get("/bmrs", summary="Get BMRS Forecast") def get_bmrs_forecast( - start_datetime_utc: datetime = Query(default=datetime.utcnow() - timedelta(days=3), description="Start date and time in UTC"), - end_datetime_utc: datetime = Query(default=datetime.utcnow() + timedelta(days=3), description="End date and time in UTC"), - process_type: str = Query("Day Ahead", description="Process type") + start_datetime_utc: datetime = Query( + default=datetime.utcnow() - timedelta(days=3), description="Start date and time in UTC" + ), + end_datetime_utc: datetime = Query( + default=datetime.utcnow() + timedelta(days=3), description="End date and time in UTC" + ), + process_type: str = Query("Day Ahead", description="Process type"), ): response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( - _from=start_datetime_utc, - to=end_datetime_utc, - process_type=process_type, - format='json' + _from=start_datetime_utc, to=end_datetime_utc, process_type=process_type, format="json" ) df = pd.DataFrame([item.to_dict() for item in response.data]) From 81af7130c5fe42c5edd20c7413c1f5bab0c0e8c3 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Fri, 19 Jul 2024 16:52:04 +0530 Subject: [PATCH 05/26] minor fix --- src/national.py | 91 ++++++++++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/src/national.py b/src/national.py index f813d8e..cfa6f8d 100644 --- a/src/national.py +++ b/src/national.py @@ -1,28 +1,27 @@ -"""National API routes""" +"""Fetch BMRS forecast data.""" import os -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from typing import List, Optional, Union -import structlog import pandas as pd -from fastapi import APIRouter, Depends, HTTPException, Query, Request, Security -from fastapi_auth0 import Auth0User -from nowcasting_datamodel.read.read import get_latest_forecast_for_gsps from sqlalchemy.orm.session import Session +import structlog from auth_utils import get_auth_implicit_scheme, get_user from cache import cache_response from database import ( - get_latest_forecast_values_for_a_specific_gsp_from_database, - get_session, - get_truth_values_for_a_specific_gsp_from_database, -) - -from pydantic_models import NationalForecast, NationalForecastValue, NationalYield -from utils import N_CALLS_PER_HOUR, filter_forecast_values, format_datetime, format_plevels, limiter -from elexonpy.api_client import ApiClient + get_latest_forecast_values_for_a_specific_gsp_from_database, get_session, + get_truth_values_for_a_specific_gsp_from_database) from elexonpy.api.generation_forecast_api import GenerationForecastApi +from elexonpy.api_client import ApiClient +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Security +from fastapi_auth0 import Auth0User +from nowcasting_datamodel.read.read import get_latest_forecast_for_gsps +from pydantic_models import (NationalForecast, NationalForecastValue, + NationalYield) +from utils import (N_CALLS_PER_HOUR, filter_forecast_values, format_datetime, + format_plevels, limiter) logger = structlog.stdlib.get_logger() @@ -38,6 +37,40 @@ forecast_api = GenerationForecastApi(api_client) +@router.get("/bmrs", summary="Get BMRS Forecast") +def get_bmrs_forecast( + start_datetime_utc: datetime = Query( + default=datetime.utcnow() - timedelta(days=3), description="Start date and time in UTC" + ), + end_datetime_utc: datetime = Query( + default=datetime.utcnow() + timedelta(days=3), description="End date and time in UTC" + ), + process_type: str = Query("Day Ahead", description="Process type"), +): + """ + Fetch BMRS solar forecasts from the Elexon API. + + Parameters: + - start_datetime_utc (datetime): Start date and time in UTC. + - end_datetime_utc (datetime): End date and time in UTC. + - process_type (str): Process type for the forecast. + + Returns: + - dict: Dictionary containing the fetched BMRS forecast data. + + """ + # Fetch data using the forecast API + response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( + _from=start_datetime_utc, to=end_datetime_utc, process_type=process_type, format="json" + ) + + # Convert response to DataFrame + df = pd.DataFrame([item.to_dict() for item in response.data]) + + # Return as JSON + return {"data": df.to_dict(orient="records")} + + @router.get( "/forecast", response_model=Union[NationalForecast, List[NationalForecastValue]], @@ -55,7 +88,9 @@ def get_national_forecast( end_datetime_utc: Optional[str] = None, creation_limit_utc: Optional[str] = None, ) -> Union[NationalForecast, List[NationalForecastValue]]: - """Get the National Forecast + """ + + Fetch national forecasts. This route returns the most recent forecast for each _target_time_. @@ -75,6 +110,9 @@ def get_national_forecast( - **creation_utc_limit**: optional, only return forecasts made before this datetime. Note you can only go 7 days back at the moment + Returns: + dict: The national forecast data. + """ logger.debug("Get national forecasts") @@ -175,7 +213,7 @@ def get_national_pvlive( session: Session = Depends(get_session), user: Auth0User = Security(get_user()), ) -> List[NationalYield]: - """### Get national PV_Live values for yesterday and/or today + """### Get national PV_Live values. Returns a series of real-time solar energy generation readings from PV_Live for all of Great Britain. @@ -190,29 +228,10 @@ def get_national_pvlive( #### Parameters - **regime**: can choose __in-day__ or __day-after__ - """ + """ logger.info(f"Get national PV Live estimates values " f"for regime {regime} for {user}") return get_truth_values_for_a_specific_gsp_from_database( session=session, gsp_id=0, regime=regime ) - - -@router.get("/bmrs", summary="Get BMRS Forecast") -def get_bmrs_forecast( - start_datetime_utc: datetime = Query( - default=datetime.utcnow() - timedelta(days=3), description="Start date and time in UTC" - ), - end_datetime_utc: datetime = Query( - default=datetime.utcnow() + timedelta(days=3), description="End date and time in UTC" - ), - process_type: str = Query("Day Ahead", description="Process type"), -): - response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( - _from=start_datetime_utc, to=end_datetime_utc, process_type=process_type, format="json" - ) - - df = pd.DataFrame([item.to_dict() for item in response.data]) - - return {"data": df.to_dict(orient="records")} From 9ca6cf0a512e511bc70e1d910ead6e215973c129 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:22:34 +0000 Subject: [PATCH 06/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/national.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/national.py b/src/national.py index cfa6f8d..5f35e2b 100644 --- a/src/national.py +++ b/src/national.py @@ -11,17 +11,17 @@ from auth_utils import get_auth_implicit_scheme, get_user from cache import cache_response from database import ( - get_latest_forecast_values_for_a_specific_gsp_from_database, get_session, - get_truth_values_for_a_specific_gsp_from_database) + get_latest_forecast_values_for_a_specific_gsp_from_database, + get_session, + get_truth_values_for_a_specific_gsp_from_database, +) from elexonpy.api.generation_forecast_api import GenerationForecastApi from elexonpy.api_client import ApiClient from fastapi import APIRouter, Depends, HTTPException, Query, Request, Security from fastapi_auth0 import Auth0User from nowcasting_datamodel.read.read import get_latest_forecast_for_gsps -from pydantic_models import (NationalForecast, NationalForecastValue, - NationalYield) -from utils import (N_CALLS_PER_HOUR, filter_forecast_values, format_datetime, - format_plevels, limiter) +from pydantic_models import NationalForecast, NationalForecastValue, NationalYield +from utils import N_CALLS_PER_HOUR, filter_forecast_values, format_datetime, format_plevels, limiter logger = structlog.stdlib.get_logger() From 746d979df215ff2a6d26901372288c25a0f2181d Mon Sep 17 00:00:00 2001 From: 14Richa Date: Fri, 19 Jul 2024 17:23:47 +0530 Subject: [PATCH 07/26] minor fix --- src/national.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/national.py b/src/national.py index cfa6f8d..0cea8de 100644 --- a/src/national.py +++ b/src/national.py @@ -38,7 +38,10 @@ @router.get("/bmrs", summary="Get BMRS Forecast") + +@limiter.limit(f"{N_CALLS_PER_HOUR}/hour") def get_bmrs_forecast( + request: Request, start_datetime_utc: datetime = Query( default=datetime.utcnow() - timedelta(days=3), description="Start date and time in UTC" ), From 8b988ab4dac82911114a9e8d37cd0d3f42ec68bf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:54:24 +0000 Subject: [PATCH 08/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/national.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/national.py b/src/national.py index 3c29005..1f33dfc 100644 --- a/src/national.py +++ b/src/national.py @@ -38,7 +38,6 @@ @router.get("/bmrs", summary="Get BMRS Forecast") - @limiter.limit(f"{N_CALLS_PER_HOUR}/hour") def get_bmrs_forecast( request: Request, From 299497d7c015fd731c9b0e2bd6c3924628b5eb5c Mon Sep 17 00:00:00 2001 From: 14Richa Date: Fri, 19 Jul 2024 17:26:22 +0530 Subject: [PATCH 09/26] changed function name --- src/national.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/national.py b/src/national.py index 3c29005..5ac0145 100644 --- a/src/national.py +++ b/src/national.py @@ -40,7 +40,7 @@ @router.get("/bmrs", summary="Get BMRS Forecast") @limiter.limit(f"{N_CALLS_PER_HOUR}/hour") -def get_bmrs_forecast( +def get_elexon_forecast( request: Request, start_datetime_utc: datetime = Query( default=datetime.utcnow() - timedelta(days=3), description="Start date and time in UTC" From 0193ad0d49c74295e9ddaef1115ab99b81805d30 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Fri, 19 Jul 2024 17:35:19 +0530 Subject: [PATCH 10/26] added filter functionality --- src/national.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/national.py b/src/national.py index 48b74f1..2098abc 100644 --- a/src/national.py +++ b/src/national.py @@ -37,7 +37,7 @@ forecast_api = GenerationForecastApi(api_client) -@router.get("/bmrs", summary="Get BMRS Forecast") +@router.get("/bmrs", summary="Get BMRS Solar Forecast") @limiter.limit(f"{N_CALLS_PER_HOUR}/hour") def get_elexon_forecast( request: Request, @@ -58,19 +58,31 @@ def get_elexon_forecast( - process_type (str): Process type for the forecast. Returns: - - dict: Dictionary containing the fetched BMRS forecast data. - + - dict: Dictionary containing the fetched BMRS solar forecast data. """ - # Fetch data using the forecast API - response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( - _from=start_datetime_utc, to=end_datetime_utc, process_type=process_type, format="json" - ) + try: + # Fetch data using the forecast API + response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( + _from=start_datetime_utc.isoformat(), + to=end_datetime_utc.isoformat(), + process_type=process_type, + format="json" + ) + + if not response.data: + return {"data": []} + + # Convert response to DataFrame + df = pd.DataFrame([item.to_dict() for item in response.data]) + + # Filter to include only solar forecasts + solar_df = df[df['business_type'] == 'Solar generation'] + result = {"data": solar_df.to_dict(orient="records")} - # Convert response to DataFrame - df = pd.DataFrame([item.to_dict() for item in response.data]) + return result - # Return as JSON - return {"data": df.to_dict(orient="records")} + except Exception as e: + return {"error": str(e)} @router.get( From e4b19e4fecabfb84bd1c0f1eab2217c73c850d9a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:05:35 +0000 Subject: [PATCH 11/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/national.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/national.py b/src/national.py index 2098abc..08004f1 100644 --- a/src/national.py +++ b/src/national.py @@ -66,7 +66,7 @@ def get_elexon_forecast( _from=start_datetime_utc.isoformat(), to=end_datetime_utc.isoformat(), process_type=process_type, - format="json" + format="json", ) if not response.data: @@ -76,7 +76,7 @@ def get_elexon_forecast( df = pd.DataFrame([item.to_dict() for item in response.data]) # Filter to include only solar forecasts - solar_df = df[df['business_type'] == 'Solar generation'] + solar_df = df[df["business_type"] == "Solar generation"] result = {"data": solar_df.to_dict(orient="records")} return result From bc5cece3e7a40cbd997dfa39bdfbad2b9b34952c Mon Sep 17 00:00:00 2001 From: 14Richa Date: Fri, 19 Jul 2024 19:05:28 +0530 Subject: [PATCH 12/26] added API link --- src/national.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/national.py b/src/national.py index 2098abc..e3b7254 100644 --- a/src/national.py +++ b/src/national.py @@ -59,6 +59,9 @@ def get_elexon_forecast( Returns: - dict: Dictionary containing the fetched BMRS solar forecast data. + + - External API Link: [Elexon API Documentation](https://bmrs.elexon.co.uk/api-documentation/endpoint/forecast/generation/wind-and-solar/day-ahead + """ try: # Fetch data using the forecast API From f86c36bd9370cd5c21c1e2cb906064a289d320fe Mon Sep 17 00:00:00 2001 From: 14Richa Date: Sat, 20 Jul 2024 15:30:50 +0530 Subject: [PATCH 13/26] minor fix --- src/national.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/national.py b/src/national.py index b204cfb..b3b6b6d 100644 --- a/src/national.py +++ b/src/national.py @@ -1,4 +1,4 @@ -"""Fetch BMRS forecast data.""" +"""National API routes""" import os from datetime import datetime, timedelta From 25a345bcf4e118d13a4eb4ded53f41ccebf8ab20 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Sat, 20 Jul 2024 15:34:40 +0530 Subject: [PATCH 14/26] remove try except block --- src/national.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/national.py b/src/national.py index b3b6b6d..9fecb33 100644 --- a/src/national.py +++ b/src/national.py @@ -60,32 +60,26 @@ def get_elexon_forecast( Returns: - dict: Dictionary containing the fetched BMRS solar forecast data. - - External API Link: [Elexon API Documentation](https://bmrs.elexon.co.uk/api-documentation/endpoint/forecast/generation/wind-and-solar/day-ahead + - External API Link: [Elexon API Documentation](https://bmrs.elexon.co.uk/api-documentation/endpoint/forecast/generation/wind-and-solar/day-ahead) """ - try: - # Fetch data using the forecast API - response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( - _from=start_datetime_utc.isoformat(), - to=end_datetime_utc.isoformat(), - process_type=process_type, - format="json", - ) - - if not response.data: - return {"data": []} + response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( + _from=start_datetime_utc.isoformat(), + to=end_datetime_utc.isoformat(), + process_type=process_type, + format="json", + ) - # Convert response to DataFrame - df = pd.DataFrame([item.to_dict() for item in response.data]) + if not response.data: + return {"data": []} - # Filter to include only solar forecasts - solar_df = df[df["business_type"] == "Solar generation"] - result = {"data": solar_df.to_dict(orient="records")} + df = pd.DataFrame([item.to_dict() for item in response.data]) - return result + # Filter to include only solar forecasts + solar_df = df[df["business_type"] == "Solar generation"] + result = {"data": solar_df.to_dict(orient="records")} - except Exception as e: - return {"error": str(e)} + return result @router.get( From f25421681d0f0857114892664140b725f66c567f Mon Sep 17 00:00:00 2001 From: 14Richa Date: Sat, 20 Jul 2024 15:46:51 +0530 Subject: [PATCH 15/26] changed bmrs to elexon --- src/national.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/national.py b/src/national.py index 9fecb33..eeccf55 100644 --- a/src/national.py +++ b/src/national.py @@ -37,7 +37,7 @@ forecast_api = GenerationForecastApi(api_client) -@router.get("/bmrs", summary="Get BMRS Solar Forecast") +@router.get("/elexon", summary="Get BMRS Solar Forecast") @limiter.limit(f"{N_CALLS_PER_HOUR}/hour") def get_elexon_forecast( request: Request, From 67a229325a32dd4492a66bd3b67fc8cb45561d39 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 23 Jul 2024 21:12:55 +0530 Subject: [PATCH 16/26] minor fix --- src/national.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/national.py b/src/national.py index eeccf55..b6cab90 100644 --- a/src/national.py +++ b/src/national.py @@ -224,7 +224,7 @@ def get_national_pvlive( session: Session = Depends(get_session), user: Auth0User = Security(get_user()), ) -> List[NationalYield]: - """### Get national PV_Live values. + """### Get national PV_Live values for yesterday and/or today Returns a series of real-time solar energy generation readings from PV_Live for all of Great Britain. From e01ea24ee9e1b062d4d7672fbcb0ffae1428ec84 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Tue, 23 Jul 2024 21:41:12 +0530 Subject: [PATCH 17/26] resolving pre hook --- src/national.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/national.py b/src/national.py index b6cab90..c70b384 100644 --- a/src/national.py +++ b/src/national.py @@ -5,9 +5,14 @@ from typing import List, Optional, Union import pandas as pd +import structlog +from elexonpy.api.generation_forecast_api import GenerationForecastApi +from elexonpy.api_client import ApiClient +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Security +from fastapi_auth0 import Auth0User +from nowcasting_datamodel.read.read import get_latest_forecast_for_gsps from sqlalchemy.orm.session import Session -import structlog from auth_utils import get_auth_implicit_scheme, get_user from cache import cache_response from database import ( @@ -15,11 +20,6 @@ get_session, get_truth_values_for_a_specific_gsp_from_database, ) -from elexonpy.api.generation_forecast_api import GenerationForecastApi -from elexonpy.api_client import ApiClient -from fastapi import APIRouter, Depends, HTTPException, Query, Request, Security -from fastapi_auth0 import Auth0User -from nowcasting_datamodel.read.read import get_latest_forecast_for_gsps from pydantic_models import NationalForecast, NationalForecastValue, NationalYield from utils import N_CALLS_PER_HOUR, filter_forecast_values, format_datetime, format_plevels, limiter @@ -52,16 +52,18 @@ def get_elexon_forecast( """ Fetch BMRS solar forecasts from the Elexon API. - Parameters: - - start_datetime_utc (datetime): Start date and time in UTC. - - end_datetime_utc (datetime): End date and time in UTC. - - process_type (str): Process type for the forecast. + Args: + request (Request): The request object containing metadata about the HTTP request. + start_datetime_utc (datetime): The start date and time in UTC. + end_datetime_utc (datetime): The end date and time in UTC. + process_type (str): The type of process (e.g., 'Day Ahead'). Returns: - - dict: Dictionary containing the fetched BMRS solar forecast data. - - - External API Link: [Elexon API Documentation](https://bmrs.elexon.co.uk/api-documentation/endpoint/forecast/generation/wind-and-solar/day-ahead) + dict: Dictionary containing the fetched BMRS solar forecast data. + External API: + [Elexon API Documentation] + (https://bmrs.elexon.co.uk/api-documentation/endpoint/forecast/generation/wind-and-solar/day-ahead) """ response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( _from=start_datetime_utc.isoformat(), From b12e8c21de4832d69cad8ba4710b2085bd62c7bb Mon Sep 17 00:00:00 2001 From: 14Richa Date: Wed, 24 Jul 2024 14:26:05 +0530 Subject: [PATCH 18/26] minor fix in docstring --- src/national.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/national.py b/src/national.py index c70b384..3e69d1e 100644 --- a/src/national.py +++ b/src/national.py @@ -50,7 +50,7 @@ def get_elexon_forecast( process_type: str = Query("Day Ahead", description="Process type"), ): """ - Fetch BMRS solar forecasts from the Elexon API. + Fetch BMRS Solar and wind(?) forecasts from the Elexon API. Args: request (Request): The request object containing metadata about the HTTP request. From c40b4ec348ccd9683f251401df41535ce1875122 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Wed, 24 Jul 2024 19:26:19 +0530 Subject: [PATCH 19/26] removed router --- src/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main.py b/src/main.py index 9cdcf66..08ed289 100644 --- a/src/main.py +++ b/src/main.py @@ -208,7 +208,6 @@ async def add_process_time_header(request: Request, call_next): # Dependency v0_route_solar = "/v0/solar/GB" v0_route_system = "/v0/system/GB" -app.include_router(national_router, prefix="/v0/solar/national") app.include_router(national_router, prefix=f"{v0_route_solar}/national") app.include_router(gsp_router, prefix=f"{v0_route_solar}/gsp") app.include_router(status_router, prefix=f"{v0_route_solar}") From 7d21b1380798d47dbc90b24ce16b03a92b5f15dc Mon Sep 17 00:00:00 2001 From: 14Richa Date: Wed, 24 Jul 2024 19:29:13 +0530 Subject: [PATCH 20/26] minor fix in naming --- src/national.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/national.py b/src/national.py index 3e69d1e..0648f90 100644 --- a/src/national.py +++ b/src/national.py @@ -37,7 +37,7 @@ forecast_api = GenerationForecastApi(api_client) -@router.get("/elexon", summary="Get BMRS Solar Forecast") +@router.get("/elexon", summary="Get elexon Solar Forecast") @limiter.limit(f"{N_CALLS_PER_HOUR}/hour") def get_elexon_forecast( request: Request, @@ -50,7 +50,7 @@ def get_elexon_forecast( process_type: str = Query("Day Ahead", description="Process type"), ): """ - Fetch BMRS Solar and wind(?) forecasts from the Elexon API. + Fetch elexon Solar and wind(?) forecasts from the Elexon API. Args: request (Request): The request object containing metadata about the HTTP request. @@ -59,11 +59,11 @@ def get_elexon_forecast( process_type (str): The type of process (e.g., 'Day Ahead'). Returns: - dict: Dictionary containing the fetched BMRS solar forecast data. + dict: Dictionary containing the fetched elexon solar forecast data. External API: [Elexon API Documentation] - (https://bmrs.elexon.co.uk/api-documentation/endpoint/forecast/generation/wind-and-solar/day-ahead) + (https://elexon.elexon.co.uk/api-documentation/endpoint/forecast/generation/wind-and-solar/day-ahead) """ response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( _from=start_datetime_utc.isoformat(), From 841554a6ba59562277f1519ab5ebee214d99feff Mon Sep 17 00:00:00 2001 From: 14Richa Date: Wed, 24 Jul 2024 19:30:32 +0530 Subject: [PATCH 21/26] added elexonpy in requirement.txt file --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b5c23b3..979d07d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ slowapi pathy==0.10.3 fsspec s3fs +elexonpy From 448bd814f4aa0c84cc64487ccf2488b687db4c5f Mon Sep 17 00:00:00 2001 From: 14Richa Date: Thu, 25 Jul 2024 17:28:21 +0530 Subject: [PATCH 22/26] resolve hook error --- src/tests/test_elexon_forecast.py | 109 ++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 src/tests/test_elexon_forecast.py diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py new file mode 100644 index 0000000..94441da --- /dev/null +++ b/src/tests/test_elexon_forecast.py @@ -0,0 +1,109 @@ +from unittest.mock import MagicMock, patch + +from fastapi.testclient import TestClient + +from main import app + +client = TestClient(app) + + +class MockForecastItem: + def __init__(self, **entries): + self.__dict__.update(entries) + + def to_dict(self): + return self.__dict__ + + +mock_data = [ + MockForecastItem( + publish_time="2024-07-24T16:45:09+00:00", + process_type="Day ahead", + business_type="Solar generation", + psr_type="Solar", + start_time="2024-07-25T23:30:00+00:00", + settlement_date="2024-07-26", + settlement_period=2, + quantity=0, + ), + MockForecastItem( + publish_time="2024-07-24T16:45:09+00:00", + process_type="Day ahead", + business_type="Solar generation", + psr_type="Solar", + start_time="2024-07-25T23:00:00+00:00", + settlement_date="2024-07-26", + settlement_period=1, + quantity=0, + ), +] + +# Define the full path for the patch target +PATCH_TARGET = ( + "elexonpy.api.generation_forecast_api." + "GenerationForecastApi.forecast_generation_wind_and_solar_day_ahead_get" +) + + +def setup_mock_forecast(mock_forecast_api, data): + mock_response = MagicMock() + mock_response.data = data + mock_forecast_api.return_value = mock_response + + +@patch(PATCH_TARGET) +def test_get_elexon_forecast_with_data(mock_forecast_api): + setup_mock_forecast(mock_forecast_api, mock_data) + endpoint = "/v0/solar/GB/national/elexon" + params = { + "start_datetime_utc": "2024-07-22T10:56:59.194610", + "end_datetime_utc": "2024-07-28T10:56:59.194680", + "process_type": "Day Ahead", + } + + response = client.get(endpoint, params=params) + + expected_response = { + "data": [ + { + "publish_time": "2024-07-24T16:45:09+00:00", + "process_type": "Day ahead", + "business_type": "Solar generation", + "psr_type": "Solar", + "start_time": "2024-07-25T23:30:00+00:00", + "settlement_date": "2024-07-26", + "settlement_period": 2, + "quantity": 0, + }, + { + "publish_time": "2024-07-24T16:45:09+00:00", + "process_type": "Day ahead", + "business_type": "Solar generation", + "psr_type": "Solar", + "start_time": "2024-07-25T23:00:00+00:00", + "settlement_date": "2024-07-26", + "settlement_period": 1, + "quantity": 0, + }, + ] + } + + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/json" + assert response.json() == expected_response + + +@patch(PATCH_TARGET) +def test_get_elexon_forecast_no_data(mock_forecast_api): + setup_mock_forecast(mock_forecast_api, []) + + endpoint = "/v0/solar/GB/national/elexon" + params = { + "start_datetime_utc": "2024-07-22T10:56:59.194610", + "end_datetime_utc": "2024-07-28T10:56:59.194680", + "process_type": "Day Ahead", + } + response = client.get(endpoint, params=params) + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/json" + assert response.json() == {"data": []} From 8e8167aa98ad4491b8f4e709627bf58520f63f3c Mon Sep 17 00:00:00 2001 From: 14Richa Date: Thu, 25 Jul 2024 21:48:38 +0530 Subject: [PATCH 23/26] minor fix --- src/tests/test_elexon_forecast.py | 126 +++++++++++------------------- 1 file changed, 45 insertions(+), 81 deletions(-) diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py index 94441da..364340c 100644 --- a/src/tests/test_elexon_forecast.py +++ b/src/tests/test_elexon_forecast.py @@ -1,69 +1,13 @@ -from unittest.mock import MagicMock, patch +import pytest +import requests +import requests_mock -from fastapi.testclient import TestClient +API_URL = "/v0/solar/GB/national/elexon" -from main import app -client = TestClient(app) - - -class MockForecastItem: - def __init__(self, **entries): - self.__dict__.update(entries) - - def to_dict(self): - return self.__dict__ - - -mock_data = [ - MockForecastItem( - publish_time="2024-07-24T16:45:09+00:00", - process_type="Day ahead", - business_type="Solar generation", - psr_type="Solar", - start_time="2024-07-25T23:30:00+00:00", - settlement_date="2024-07-26", - settlement_period=2, - quantity=0, - ), - MockForecastItem( - publish_time="2024-07-24T16:45:09+00:00", - process_type="Day ahead", - business_type="Solar generation", - psr_type="Solar", - start_time="2024-07-25T23:00:00+00:00", - settlement_date="2024-07-26", - settlement_period=1, - quantity=0, - ), -] - -# Define the full path for the patch target -PATCH_TARGET = ( - "elexonpy.api.generation_forecast_api." - "GenerationForecastApi.forecast_generation_wind_and_solar_day_ahead_get" -) - - -def setup_mock_forecast(mock_forecast_api, data): - mock_response = MagicMock() - mock_response.data = data - mock_forecast_api.return_value = mock_response - - -@patch(PATCH_TARGET) -def test_get_elexon_forecast_with_data(mock_forecast_api): - setup_mock_forecast(mock_forecast_api, mock_data) - endpoint = "/v0/solar/GB/national/elexon" - params = { - "start_datetime_utc": "2024-07-22T10:56:59.194610", - "end_datetime_utc": "2024-07-28T10:56:59.194680", - "process_type": "Day Ahead", - } - - response = client.get(endpoint, params=params) - - expected_response = { +@pytest.fixture +def mock_data(): + return { "data": [ { "publish_time": "2024-07-24T16:45:09+00:00", @@ -88,22 +32,42 @@ def test_get_elexon_forecast_with_data(mock_forecast_api): ] } - assert response.status_code == 200 - assert response.headers["Content-Type"] == "application/json" - assert response.json() == expected_response - -@patch(PATCH_TARGET) -def test_get_elexon_forecast_no_data(mock_forecast_api): - setup_mock_forecast(mock_forecast_api, []) - - endpoint = "/v0/solar/GB/national/elexon" - params = { - "start_datetime_utc": "2024-07-22T10:56:59.194610", - "end_datetime_utc": "2024-07-28T10:56:59.194680", - "process_type": "Day Ahead", - } - response = client.get(endpoint, params=params) - assert response.status_code == 200 - assert response.headers["Content-Type"] == "application/json" - assert response.json() == {"data": []} +def test_get_elexon_forecast_with_data(mock_data): + with requests_mock.Mocker() as m: + url = ( + f"{API_URL}?start_datetime_utc=2024-07-22T10:56:59.194610" + f"&end_datetime_utc=2024-07-28T10:56:59.194680" + f"&process_type=Day Ahead" + ) + m.get(url, json=mock_data, headers={"Content-Type": "application/json"}) + + response = requests.get(url) + print("Response Headers:", response.headers) + # Assertions + assert response.status_code == 200 + assert response.headers.get("Content-Type") == "application/json" + assert response.json() == mock_data + + +@pytest.fixture +def empty_mock_data(): + return {"data": []} + + +def test_get_elexon_forecast_no_data(empty_mock_data): + with requests_mock.Mocker() as m: + url = ( + f"{API_URL}?start_datetime_utc=2024-07-22T10:56:59.194610" + f"&end_datetime_utc=2024-07-28T10:56:59.194680" + f"&process_type=Day Ahead" + ) + + m.get(url, json=empty_mock_data, headers={"Content-Type": "application/json"}) + + response = requests.get(url) + print("Response Headers:", response.headers) + # Assertions + assert response.status_code == 200 + assert response.headers.get("Content-Type") == "application/json" + assert response.json() == empty_mock_data From e734329028cc4ba2942c43a9e1a085bb8e0049ce Mon Sep 17 00:00:00 2001 From: 14Richa Date: Sat, 27 Jul 2024 14:58:20 +0530 Subject: [PATCH 24/26] added response model --- src/national.py | 73 +++++++++++++++++++++++++++++------------- src/pydantic_models.py | 27 +++++++++++++++- 2 files changed, 77 insertions(+), 23 deletions(-) diff --git a/src/national.py b/src/national.py index 0648f90..75156f5 100644 --- a/src/national.py +++ b/src/national.py @@ -20,7 +20,13 @@ get_session, get_truth_values_for_a_specific_gsp_from_database, ) -from pydantic_models import NationalForecast, NationalForecastValue, NationalYield +from pydantic_models import ( + NationalForecast, + NationalForecastValue, + NationalYield, + SolarForecastResponse, + SolarForecastValue, +) from utils import N_CALLS_PER_HOUR, filter_forecast_values, format_datetime, format_plevels, limiter logger = structlog.stdlib.get_logger() @@ -59,29 +65,52 @@ def get_elexon_forecast( process_type (str): The type of process (e.g., 'Day Ahead'). Returns: - dict: Dictionary containing the fetched elexon solar forecast data. - - External API: - [Elexon API Documentation] - (https://elexon.elexon.co.uk/api-documentation/endpoint/forecast/generation/wind-and-solar/day-ahead) + SolarForecastResponse: The forecast data wrapped in a SolarForecastResponse model. """ - response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( - _from=start_datetime_utc.isoformat(), - to=end_datetime_utc.isoformat(), - process_type=process_type, - format="json", - ) - - if not response.data: - return {"data": []} - - df = pd.DataFrame([item.to_dict() for item in response.data]) - - # Filter to include only solar forecasts - solar_df = df[df["business_type"] == "Solar generation"] - result = {"data": solar_df.to_dict(orient="records")} + try: + response = forecast_api.forecast_generation_wind_and_solar_day_ahead_get( + _from=start_datetime_utc.isoformat(), + to=end_datetime_utc.isoformat(), + process_type=process_type, + format="json", + ) - return result + if not response.data: + return SolarForecastResponse(data=[]) + + df = pd.DataFrame([item.to_dict() for item in response.data]) + logger.debug("DataFrame Columns: %s", df.columns) + logger.debug("DataFrame Sample: %s", df.head()) + + # Filter to include only solar forecasts + solar_df = df[df["business_type"] == "Solar generation"] + logger.debug("Filtered Solar DataFrame: %s", solar_df.head()) + + forecast_values = [] + for _, row in solar_df.iterrows(): + try: + forecast_values.append( + SolarForecastValue( + timestamp=pd.to_datetime(row["publish_time"]).to_pydatetime(), + expected_power_generation_megawatts=row.get("quantity"), + plevels=None, + ) + ) + except KeyError as e: + logger.error("KeyError: %s. Data: %s", str(e), row) + raise HTTPException(status_code=500, detail="Internal Server Error") + except Exception as e: + logger.error( + "Error during DataFrame to Model conversion: %s. Data: %s", str(e), row + ) + raise HTTPException(status_code=500, detail="Internal Server Error") + + result = SolarForecastResponse(data=forecast_values) + return result + + except Exception as e: + logger.error("Unhandled exception: %s", str(e)) + raise HTTPException(status_code=500, detail="Internal Server Error") @router.get( diff --git a/src/pydantic_models.py b/src/pydantic_models.py index ca01033..68faca7 100644 --- a/src/pydantic_models.py +++ b/src/pydantic_models.py @@ -7,7 +7,7 @@ from nowcasting_datamodel.models import Forecast, ForecastSQL, ForecastValue, Location, LocationSQL from nowcasting_datamodel.models.utils import EnhancedBaseModel -from pydantic import Field, validator +from pydantic import BaseModel, Field, validator logger = logging.getLogger(__name__) @@ -214,3 +214,28 @@ class NationalForecast(Forecast): """One Forecast of generation at one timestamp""" forecast_values: List[NationalForecastValue] = Field(..., description="List of forecast values") + + +class SolarForecastValue(BaseModel): + """Represents a single solar forecast entry""" + + timestamp: datetime = Field(..., description="Timestamp of the forecast") + expected_power_generation_megawatts: Optional[float] = Field( + None, ge=0, description="Expected power generation in megawatts" + ) + plevels: Optional[str] = Field( + None, description="String representing properties of the forecast" + ) + + @validator("expected_power_generation_megawatts") + def result_check(cls, v): + """Round to 2 decimal places""" + if v is not None: + return round(v, 2) + return v + + +class SolarForecastResponse(BaseModel): + """Wrapper for a list of solar forecast values""" + + data: List[SolarForecastValue] = Field(..., description="List of solar forecast values") From 0d0cc1caa198689876ee326cd08bcf57e520e0bd Mon Sep 17 00:00:00 2001 From: 14Richa Date: Sat, 27 Jul 2024 15:02:24 +0530 Subject: [PATCH 25/26] fixed unit test --- src/tests/test_elexon_forecast.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/tests/test_elexon_forecast.py b/src/tests/test_elexon_forecast.py index 364340c..07f4501 100644 --- a/src/tests/test_elexon_forecast.py +++ b/src/tests/test_elexon_forecast.py @@ -10,24 +10,14 @@ def mock_data(): return { "data": [ { - "publish_time": "2024-07-24T16:45:09+00:00", - "process_type": "Day ahead", - "business_type": "Solar generation", - "psr_type": "Solar", - "start_time": "2024-07-25T23:30:00+00:00", - "settlement_date": "2024-07-26", - "settlement_period": 2, - "quantity": 0, + "timestamp": "2024-07-24T16:45:09+00:00", + "expected_power_generation_megawatts": 0, + "plevels": None, }, { - "publish_time": "2024-07-24T16:45:09+00:00", - "process_type": "Day ahead", - "business_type": "Solar generation", - "psr_type": "Solar", - "start_time": "2024-07-25T23:00:00+00:00", - "settlement_date": "2024-07-26", - "settlement_period": 1, - "quantity": 0, + "timestamp": "2024-07-24T16:45:09+00:00", + "expected_power_generation_megawatts": 0, + "plevels": None, }, ] } From e53d7a840035b77a6436e4da9b21add95b06a2c2 Mon Sep 17 00:00:00 2001 From: 14Richa Date: Sat, 27 Jul 2024 16:29:54 +0530 Subject: [PATCH 26/26] remove plevel --- src/pydantic_models.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/pydantic_models.py b/src/pydantic_models.py index 68faca7..10f9f64 100644 --- a/src/pydantic_models.py +++ b/src/pydantic_models.py @@ -223,9 +223,6 @@ class SolarForecastValue(BaseModel): expected_power_generation_megawatts: Optional[float] = Field( None, ge=0, description="Expected power generation in megawatts" ) - plevels: Optional[str] = Field( - None, description="String representing properties of the forecast" - ) @validator("expected_power_generation_megawatts") def result_check(cls, v):