From 7450fd1b49d06f920ad4f4a132285cb5d96bab93 Mon Sep 17 00:00:00 2001 From: Arturo Reyes Lopez Date: Mon, 9 Dec 2024 14:22:36 -0700 Subject: [PATCH 1/6] Update fuel type for other_uses --- .../versions/2024-12-09-19-31_7ae38a8413ab.py | 95 +++++++++++++++++++ backend/lcfs/web/api/other_uses/repo.py | 90 ++++++++++++++++-- 2 files changed, 179 insertions(+), 6 deletions(-) create mode 100644 backend/lcfs/db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py diff --git a/backend/lcfs/db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py b/backend/lcfs/db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py new file mode 100644 index 000000000..da29c2fc1 --- /dev/null +++ b/backend/lcfs/db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py @@ -0,0 +1,95 @@ +"""Update Fuel Types measured in volume to be other-uses + +Revision ID: 7ae38a8413ab +Revises: 26ab15f8ab18 +Create Date: 2024-12-09 19:31:18.199089 + +""" + +import sqlalchemy as sa +from alembic import op +from datetime import datetime + +# revision identifiers, used by Alembic. +revision = "7ae38a8413ab" +down_revision = "26ab15f8ab18" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + current_time = datetime.now() + + # Update the `other_uses_fossil_derived` field for all specified fuel types + op.execute( + f""" + UPDATE fuel_type + SET other_uses_fossil_derived = true, + update_date = '{current_time}', + update_user = 'no_user' + WHERE fuel_type IN ( + 'Alternative jet fuel', + 'Biodiesel', + 'Ethanol', + 'HDRD', + 'Renewable gasoline', + 'Renewable naphtha' + ) + """ + ) + + # Update the `other_uses_fossil_derived` field for all specified fuel types + op.execute( + f""" + UPDATE fuel_type + SET other_uses_fossil_derived = false, + update_date = '{current_time}', + update_user = 'no_user' + WHERE fuel_type IN ( + 'CNG', + 'Electricity', + 'Hydrogen', + 'LNG', + 'Propane' + ) + """ + ) + + +def downgrade() -> None: + current_time = datetime.now() + + # Revert the `other_uses_fossil_derived` field to false for the first set of fuel types + op.execute( + f""" + UPDATE fuel_type + SET other_uses_fossil_derived = false, + update_date = '{current_time}', + update_user = 'no_user' + WHERE fuel_type IN ( + 'Alternative jet fuel', + 'Biodiesel', + 'Ethanol', + 'HDRD', + 'Renewable gasoline', + 'Renewable naphtha' + ) + """ + ) + + # Revert the `other_uses_fossil_derived` field to true for the second set of fuel types + op.execute( + f""" + UPDATE fuel_type + SET other_uses_fossil_derived = true, + update_date = '{current_time}', + update_user = 'no_user' + WHERE fuel_type IN ( + 'CNG', + 'Electricity', + 'Hydrogen', + 'LNG', + 'Propane' + ) + """ + ) diff --git a/backend/lcfs/web/api/other_uses/repo.py b/backend/lcfs/web/api/other_uses/repo.py index 68a8a434a..8e0f5ca4d 100644 --- a/backend/lcfs/web/api/other_uses/repo.py +++ b/backend/lcfs/web/api/other_uses/repo.py @@ -1,20 +1,21 @@ import structlog -from typing import List, Optional, Tuple, Any +from datetime import date +from typing import List, Optional, Tuple, Dict, Any from fastapi import Depends - from lcfs.db.base import ActionTypeEnum, UserTypeEnum from lcfs.db.dependencies import get_async_db_session -from sqlalchemy import select, delete, func, case, and_ -from sqlalchemy.orm import joinedload +from sqlalchemy import select, delete, func, case, and_, or_ +from sqlalchemy.orm import joinedload, contains_eager from sqlalchemy.ext.asyncio import AsyncSession from lcfs.db.models.compliance import ComplianceReport from lcfs.db.models.compliance.OtherUses import OtherUses from lcfs.db.models.fuel.ProvisionOfTheAct import ProvisionOfTheAct from lcfs.db.models.fuel.FuelCode import FuelCode -from lcfs.db.models.fuel.FuelType import QuantityUnitsEnum +from lcfs.db.models.fuel.FuelType import FuelType, QuantityUnitsEnum +from lcfs.db.models.fuel.FuelInstance import FuelInstance from lcfs.web.api.fuel_code.repo import FuelCodeRepository from lcfs.web.api.other_uses.schema import OtherUsesSchema from lcfs.web.api.base import PaginationRequestSchema @@ -37,7 +38,7 @@ def __init__( async def get_table_options(self) -> dict: """Get all table options""" fuel_categories = await self.fuel_code_repo.get_fuel_categories() - fuel_types = await self.fuel_code_repo.get_formatted_fuel_types() + fuel_types = await self.get_formatted_fuel_types() expected_uses = await self.fuel_code_repo.get_expected_use_types() units_of_measure = [unit.value for unit in QuantityUnitsEnum] provisions_of_the_act = ( @@ -302,3 +303,80 @@ async def get_other_use_version_by_user( result = await self.db.execute(query) return result.scalars().first() + + @repo_handler + async def get_formatted_fuel_types(self) -> List[Dict[str, Any]]: + """Get all fuel type options with their associated fuel categories and fuel codes for other uses""" + # Define the filtering conditions for fuel codes + current_date = date.today() + fuel_code_filters = ( + or_(FuelCode.effective_date == None, FuelCode.effective_date <= current_date) + & or_(FuelCode.expiration_date == None, FuelCode.expiration_date > current_date) + & (FuelType.other_uses_fossil_derived == True) + ) + + # Build the query with filtered fuel_codes + query = ( + select(FuelType) + .outerjoin(FuelType.fuel_instances) + .outerjoin(FuelInstance.fuel_category) + .outerjoin(FuelType.fuel_codes) + .where(fuel_code_filters) + .options( + contains_eager(FuelType.fuel_instances).contains_eager( + FuelInstance.fuel_category + ), + contains_eager(FuelType.fuel_codes), + joinedload(FuelType.provision_1), + joinedload(FuelType.provision_2), + ) + ) + + result = await self.db.execute(query) + fuel_types = result.unique().scalars().all() + + # Prepare the data in the format matching your schema + formatted_fuel_types = [] + for fuel_type in fuel_types: + formatted_fuel_type = { + "fuel_type_id": fuel_type.fuel_type_id, + "fuel_type": fuel_type.fuel_type, + "default_carbon_intensity": fuel_type.default_carbon_intensity, + "units": fuel_type.units if fuel_type.units else None, + "unrecognized": fuel_type.unrecognized, + "fuel_categories": [ + { + "fuel_category_id": fc.fuel_category.fuel_category_id, + "category": fc.fuel_category.category, + } + for fc in fuel_type.fuel_instances + ], + "fuel_codes": [ + { + "fuel_code_id": fc.fuel_code_id, + "fuel_code": fc.fuel_code, + "carbon_intensity": fc.carbon_intensity, + } + for fc in fuel_type.fuel_codes + ], + "provision_of_the_act": [], + } + + if fuel_type.provision_1: + formatted_fuel_type["provision_of_the_act"].append( + { + "provision_of_the_act_id": fuel_type.provision_1_id, + "name": fuel_type.provision_1.name, + } + ) + + if fuel_type.provision_2: + formatted_fuel_type["provision_of_the_act"].append( + { + "provision_of_the_act_id": fuel_type.provision_2_id, + "name": fuel_type.provision_2.name, + } + ) + formatted_fuel_types.append(formatted_fuel_type) + + return formatted_fuel_types \ No newline at end of file From 4573d8f5bfc4760526b7f5db2be9d81410a9ca03 Mon Sep 17 00:00:00 2001 From: Arturo Reyes Lopez Date: Mon, 9 Dec 2024 16:32:10 -0700 Subject: [PATCH 2/6] updating pytest --- .../tests/other_uses/test_other_uses_repo.py | 27 ++++++++++++++----- backend/lcfs/web/api/other_uses/repo.py | 6 ++--- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/backend/lcfs/tests/other_uses/test_other_uses_repo.py b/backend/lcfs/tests/other_uses/test_other_uses_repo.py index 8bd39f562..97b5308f5 100644 --- a/backend/lcfs/tests/other_uses/test_other_uses_repo.py +++ b/backend/lcfs/tests/other_uses/test_other_uses_repo.py @@ -11,14 +11,19 @@ @pytest.fixture -def mock_db_session(): +def mock_query_result(): + # Setup mock for database query result chain + mock_result = AsyncMock() + mock_result.unique = MagicMock(return_value=mock_result) + mock_result.scalars = MagicMock(return_value=mock_result) + mock_result.all = MagicMock(return_value=[MagicMock(spec=OtherUses)]) + return mock_result + + +@pytest.fixture +def mock_db_session(mock_query_result): session = MagicMock(spec=AsyncSession) - execute_result = AsyncMock() - execute_result.unique = MagicMock(return_value=execute_result) - execute_result.scalars = MagicMock(return_value=execute_result) - execute_result.all = MagicMock(return_value=[MagicMock(spec=OtherUses)]) - execute_result.first = MagicMock(return_value=MagicMock(spec=OtherUses)) - session.execute.return_value = execute_result + session.execute = AsyncMock(return_value=mock_query_result) return session @@ -29,6 +34,14 @@ def other_uses_repo(mock_db_session): repo.fuel_code_repo.get_fuel_categories = AsyncMock(return_value=[]) repo.fuel_code_repo.get_fuel_types = AsyncMock(return_value=[]) repo.fuel_code_repo.get_expected_use_types = AsyncMock(return_value=[]) + + # Mock for local get_formatted_fuel_types method + async def mock_get_formatted_fuel_types(): + mock_result = await mock_db_session.execute(AsyncMock()) + return mock_result.unique().scalars().all() + + repo.get_formatted_fuel_types = AsyncMock(side_effect=mock_get_formatted_fuel_types) + return repo diff --git a/backend/lcfs/web/api/other_uses/repo.py b/backend/lcfs/web/api/other_uses/repo.py index 8e0f5ca4d..595de5882 100644 --- a/backend/lcfs/web/api/other_uses/repo.py +++ b/backend/lcfs/web/api/other_uses/repo.py @@ -76,7 +76,7 @@ async def get_latest_other_uses_by_group_uuid( ) result = await self.db.execute(query) - return result.scalars().first() + return await result.unique().scalars().first() @repo_handler async def get_other_uses(self, compliance_report_id: int) -> List[OtherUsesSchema]: @@ -302,7 +302,7 @@ async def get_other_use_version_by_user( ) result = await self.db.execute(query) - return result.scalars().first() + return await result.scalars().first() @repo_handler async def get_formatted_fuel_types(self) -> List[Dict[str, Any]]: @@ -333,7 +333,7 @@ async def get_formatted_fuel_types(self) -> List[Dict[str, Any]]: ) result = await self.db.execute(query) - fuel_types = result.unique().scalars().all() + fuel_types = await result.unique().scalars().all() # Prepare the data in the format matching your schema formatted_fuel_types = [] From 8580d716d28eec96ef590414581998e82b2e9c12 Mon Sep 17 00:00:00 2001 From: Arturo Reyes Lopez Date: Wed, 11 Dec 2024 10:26:34 -0700 Subject: [PATCH 3/6] changes in await --- .../tests/other_uses/test_other_uses_repo.py | 19 +++++++++---------- backend/lcfs/web/api/other_uses/repo.py | 15 ++++++++++----- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/backend/lcfs/tests/other_uses/test_other_uses_repo.py b/backend/lcfs/tests/other_uses/test_other_uses_repo.py index 97b5308f5..8ef79cd70 100644 --- a/backend/lcfs/tests/other_uses/test_other_uses_repo.py +++ b/backend/lcfs/tests/other_uses/test_other_uses_repo.py @@ -207,22 +207,21 @@ async def test_get_latest_other_uses_by_group_uuid(other_uses_repo, mock_db_sess mock_other_use_gov.user_type = UserTypeEnum.GOVERNMENT mock_other_use_gov.version = 2 - mock_other_use_supplier = MagicMock(spec=OtherUses) - mock_other_use_supplier.user_type = UserTypeEnum.SUPPLIER - mock_other_use_supplier.version = 3 - - # Mock response with both government and supplier versions - mock_db_session.execute.return_value.scalars.return_value.first.side_effect = [ - mock_other_use_gov, - mock_other_use_supplier, - ] + # Setup mock result chain + mock_result = AsyncMock() + mock_result.unique = MagicMock(return_value=mock_result) + mock_result.scalars = MagicMock(return_value=mock_result) + mock_result.first = MagicMock(return_value=mock_other_use_gov) + + # Configure mock db session + mock_db_session.execute = AsyncMock(return_value=mock_result) + other_uses_repo.db = mock_db_session result = await other_uses_repo.get_latest_other_uses_by_group_uuid(group_uuid) assert result.user_type == UserTypeEnum.GOVERNMENT assert result.version == 2 - @pytest.mark.anyio async def test_get_other_use_version_by_user(other_uses_repo, mock_db_session): group_uuid = "test-group-uuid" diff --git a/backend/lcfs/web/api/other_uses/repo.py b/backend/lcfs/web/api/other_uses/repo.py index 595de5882..9f49a3d5d 100644 --- a/backend/lcfs/web/api/other_uses/repo.py +++ b/backend/lcfs/web/api/other_uses/repo.py @@ -76,7 +76,7 @@ async def get_latest_other_uses_by_group_uuid( ) result = await self.db.execute(query) - return await result.unique().scalars().first() + return result.unique().scalars().first() @repo_handler async def get_other_uses(self, compliance_report_id: int) -> List[OtherUsesSchema]: @@ -310,8 +310,13 @@ async def get_formatted_fuel_types(self) -> List[Dict[str, Any]]: # Define the filtering conditions for fuel codes current_date = date.today() fuel_code_filters = ( - or_(FuelCode.effective_date == None, FuelCode.effective_date <= current_date) - & or_(FuelCode.expiration_date == None, FuelCode.expiration_date > current_date) + or_( + FuelCode.effective_date == None, FuelCode.effective_date <= current_date + ) + & or_( + FuelCode.expiration_date == None, + FuelCode.expiration_date > current_date, + ) & (FuelType.other_uses_fossil_derived == True) ) @@ -333,7 +338,7 @@ async def get_formatted_fuel_types(self) -> List[Dict[str, Any]]: ) result = await self.db.execute(query) - fuel_types = await result.unique().scalars().all() + fuel_types = result.unique().scalars().all() # Prepare the data in the format matching your schema formatted_fuel_types = [] @@ -379,4 +384,4 @@ async def get_formatted_fuel_types(self) -> List[Dict[str, Any]]: ) formatted_fuel_types.append(formatted_fuel_type) - return formatted_fuel_types \ No newline at end of file + return formatted_fuel_types From d13cd66f3d745944f5959b705b7567daf3154770 Mon Sep 17 00:00:00 2001 From: Arturo Reyes Lopez Date: Wed, 11 Dec 2024 12:24:47 -0700 Subject: [PATCH 4/6] fixes in migrations --- .../db/migrations/versions/2024-12-09-22-33_cd8698fe40e6.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lcfs/db/migrations/versions/2024-12-09-22-33_cd8698fe40e6.py b/backend/lcfs/db/migrations/versions/2024-12-09-22-33_cd8698fe40e6.py index 8ec2b8223..3a512b338 100644 --- a/backend/lcfs/db/migrations/versions/2024-12-09-22-33_cd8698fe40e6.py +++ b/backend/lcfs/db/migrations/versions/2024-12-09-22-33_cd8698fe40e6.py @@ -11,7 +11,7 @@ # revision identifiers, used by Alembic. revision = "cd8698fe40e6" -down_revision = "9206124a098b" +down_revision = "7ae38a8413ab" branch_labels = None depends_on = None From a7e6979af1f6242cc7e72edbcf7f43dc924936d8 Mon Sep 17 00:00:00 2001 From: Arturo Reyes Lopez Date: Wed, 11 Dec 2024 12:26:11 -0700 Subject: [PATCH 5/6] fix migration issue --- .../db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lcfs/db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py b/backend/lcfs/db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py index da29c2fc1..0f422117e 100644 --- a/backend/lcfs/db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py +++ b/backend/lcfs/db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py @@ -12,7 +12,7 @@ # revision identifiers, used by Alembic. revision = "7ae38a8413ab" -down_revision = "26ab15f8ab18" +down_revision = "9206124a098b" branch_labels = None depends_on = None From 0253e2ad65b88b158a87701857355eb44cce4f0b Mon Sep 17 00:00:00 2001 From: Arturo Reyes Lopez Date: Wed, 11 Dec 2024 14:17:21 -0700 Subject: [PATCH 6/6] Updating migration to correct down_revision --- .../versions/2024-12-09-22-33_cd8698fe40e6.py | 2 +- ...e38a8413ab.py => 2024-12-09-23-31_7ae38a8413ab.py} | 2 +- backend/lcfs/tests/other_uses/test_other_uses_repo.py | 11 ++++++++--- backend/lcfs/web/api/other_uses/repo.py | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) rename backend/lcfs/db/migrations/versions/{2024-12-09-19-31_7ae38a8413ab.py => 2024-12-09-23-31_7ae38a8413ab.py} (98%) diff --git a/backend/lcfs/db/migrations/versions/2024-12-09-22-33_cd8698fe40e6.py b/backend/lcfs/db/migrations/versions/2024-12-09-22-33_cd8698fe40e6.py index 3a512b338..8ec2b8223 100644 --- a/backend/lcfs/db/migrations/versions/2024-12-09-22-33_cd8698fe40e6.py +++ b/backend/lcfs/db/migrations/versions/2024-12-09-22-33_cd8698fe40e6.py @@ -11,7 +11,7 @@ # revision identifiers, used by Alembic. revision = "cd8698fe40e6" -down_revision = "7ae38a8413ab" +down_revision = "9206124a098b" branch_labels = None depends_on = None diff --git a/backend/lcfs/db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py b/backend/lcfs/db/migrations/versions/2024-12-09-23-31_7ae38a8413ab.py similarity index 98% rename from backend/lcfs/db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py rename to backend/lcfs/db/migrations/versions/2024-12-09-23-31_7ae38a8413ab.py index 0f422117e..51856f8c4 100644 --- a/backend/lcfs/db/migrations/versions/2024-12-09-19-31_7ae38a8413ab.py +++ b/backend/lcfs/db/migrations/versions/2024-12-09-23-31_7ae38a8413ab.py @@ -12,7 +12,7 @@ # revision identifiers, used by Alembic. revision = "7ae38a8413ab" -down_revision = "9206124a098b" +down_revision = "cd8698fe40e6" branch_labels = None depends_on = None diff --git a/backend/lcfs/tests/other_uses/test_other_uses_repo.py b/backend/lcfs/tests/other_uses/test_other_uses_repo.py index 8ef79cd70..67ea7d1d5 100644 --- a/backend/lcfs/tests/other_uses/test_other_uses_repo.py +++ b/backend/lcfs/tests/other_uses/test_other_uses_repo.py @@ -233,9 +233,14 @@ async def test_get_other_use_version_by_user(other_uses_repo, mock_db_session): mock_other_use.version = version mock_other_use.user_type = user_type - mock_db_session.execute.return_value.scalars.return_value.first.return_value = ( - mock_other_use - ) + # Set up mock result chain + mock_result = AsyncMock() + mock_result.scalars = MagicMock(return_value=mock_result) + mock_result.first = MagicMock(return_value=mock_other_use) + + # Configure mock db session + mock_db_session.execute = AsyncMock(return_value=mock_result) + other_uses_repo.db = mock_db_session result = await other_uses_repo.get_other_use_version_by_user( group_uuid, version, user_type diff --git a/backend/lcfs/web/api/other_uses/repo.py b/backend/lcfs/web/api/other_uses/repo.py index 9f49a3d5d..8e515b484 100644 --- a/backend/lcfs/web/api/other_uses/repo.py +++ b/backend/lcfs/web/api/other_uses/repo.py @@ -302,7 +302,7 @@ async def get_other_use_version_by_user( ) result = await self.db.execute(query) - return await result.scalars().first() + return result.scalars().first() @repo_handler async def get_formatted_fuel_types(self) -> List[Dict[str, Any]]: