diff --git a/.github/workflows/prod-ci.yaml b/.github/workflows/prod-ci.yaml index f96807cc9..3478be8ff 100644 --- a/.github/workflows/prod-ci.yaml +++ b/.github/workflows/prod-ci.yaml @@ -42,16 +42,31 @@ jobs: echo "IMAGE_TAG retrieved from Test is $imagetag" echo "IMAGE_TAG=$imagetag" >> $GITHUB_OUTPUT + get-current-time: + name: Get Current Time + runs-on: ubuntu-latest + needs: get-image-tag + + outputs: + CURRENT_TIME: ${{ steps.get-current-time.outputs.CURRENT_TIME }} + + steps: + - id: get-current-time + run: | + TZ="America/Vancouver" + echo "CURRENT_TIME=$(date '+%Y-%m-%d %H:%M:%S %Z')" >> $GITHUB_OUTPUT + # Deplog the image which is running on test to prod deploy-on-prod: name: Deploy LCFS on Prod runs-on: ubuntu-latest - needs: get-image-tag + needs: [get-image-tag, get-current-time] timeout-minutes: 60 env: IMAGE_TAG: ${{ needs.get-image-tag.outputs.IMAGE_TAG }} + CURRENT_TIME: ${{ needs.get-current-time.outputs.CURRENT_TIME }} steps: @@ -66,9 +81,17 @@ jobs: uses: trstringer/manual-approval@v1.6.0 with: secret: ${{ github.TOKEN }} - approvers: AlexZorkin,kuanfandevops,hamed-valiollahi,airinggov,areyeslo,dhaselhan,Grulin,justin-lepitzki,kevin-hashimoto + approvers: AlexZorkin,kuanfandevops,hamed-valiollahi,airinggov,areyeslo,dhaselhan,Grulin minimum-approvals: 2 - issue-title: "LCFS ${{env.IMAGE_TAG }} Prod Deployment" + issue-title: "LCFS ${{env.IMAGE_TAG }} Prod Deployment at ${{ env.CURRENT_TIME }}." + + - name: Log in to Openshift + uses: redhat-actions/oc-login@v1.3 + with: + openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }} + openshift_token: ${{ secrets.OPENSHIFT_TOKEN }} + insecure_skip_tls_verify: true + namespace: ${{ env.PROD_NAMESPACE }} - name: Tag LCFS images from Test to Prod run: | @@ -88,6 +111,6 @@ jobs: git config --global user.name "GitHub Actions" git add lcfs/charts/lcfs-frontend/values-prod.yaml git add lcfs/charts/lcfs-backend/values-prod.yaml - git commit -m "update the version with pre-release number for prod" + git commit -m "Update image tag ${{env.IMAGE_TAG }} for prod" git push \ No newline at end of file diff --git a/.github/workflows/test-ci.yaml b/.github/workflows/test-ci.yaml index e8ca4820d..1119b9432 100644 --- a/.github/workflows/test-ci.yaml +++ b/.github/workflows/test-ci.yaml @@ -225,7 +225,7 @@ jobs: uses: trstringer/manual-approval@v1.6.0 with: secret: ${{ github.TOKEN }} - approvers: AlexZorkin,kuanfandevops,hamed-valiollahi,airinggov,areyeslo,dhaselhan,Grulin,justin-lepitzki,kevin-hashimoto + approvers: AlexZorkin,kuanfandevops,hamed-valiollahi,airinggov,areyeslo,dhaselhan,Grulin,kevin-hashimoto minimum-approvals: 1 issue-title: "LCFS ${{ env.VERSION }}-${{ env.PRE_RELEASE }} Test Deployment" diff --git a/.gitignore b/.gitignore index 81b02a92e..e437ac444 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ *.py[cod] *$py.class docs/ +.DS_Store # C extensions *.so diff --git a/backend/Dockerfile.openshift b/backend/Dockerfile.openshift index 58fb4ef1d..2b4b4a1f4 100644 --- a/backend/Dockerfile.openshift +++ b/backend/Dockerfile.openshift @@ -1,7 +1,7 @@ # Base stage for common setup -FROM artifacts.developer.gov.bc.ca/docker-remote/python:3.11-slim-bullseye as base +FROM artifacts.developer.gov.bc.ca/docker-remote/python:3.11-bullseye as base -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends procps \ gcc \ && rm -rf /var/lib/apt/lists/* @@ -23,9 +23,9 @@ ENV POETRY_CACHE_DIR=/.cache/pypoetry RUN poetry install --only main # Removing gcc -RUN apt-get purge -y \ - gcc \ - && rm -rf /var/lib/apt/lists/* +# RUN apt-get purge -y \ +# gcc \ +# && rm -rf /var/lib/apt/lists/* # Copying the actual application, wait-for-it script, and prestart script COPY . /app/ diff --git a/backend/lcfs/__main__.py b/backend/lcfs/__main__.py index 397e0a313..146f7cff8 100644 --- a/backend/lcfs/__main__.py +++ b/backend/lcfs/__main__.py @@ -14,6 +14,7 @@ def main() -> None: reload=settings.reload, log_level=settings.log_level.value.lower(), factory=True, + timeout_keep_alive=settings.timeout_keep_alive, ) except Exception as e: print(e) diff --git a/backend/lcfs/conftest.py b/backend/lcfs/conftest.py index 69c244a6e..cea7bae8a 100644 --- a/backend/lcfs/conftest.py +++ b/backend/lcfs/conftest.py @@ -37,7 +37,7 @@ from lcfs.db.models.user.UserRole import UserRole from lcfs.db.seeders.seed_database import seed_database from lcfs.db.utils import create_test_database, drop_test_database -from lcfs.services.redis.dependency import get_redis_pool +from lcfs.services.redis.dependency import get_redis_client from lcfs.settings import settings from lcfs.web.application import get_app @@ -118,19 +118,20 @@ async def dbsession( @pytest.fixture -async def fake_redis_pool() -> AsyncGenerator[ConnectionPool, None]: +async def fake_redis_client() -> AsyncGenerator[aioredis.FakeRedis, None]: """ - Get instance of a fake redis. + Get instance of a fake Redis client. - :yield: FakeRedis instance. + :yield: FakeRedis client instance. """ server = FakeServer() server.connected = True - pool = ConnectionPool(connection_class=FakeConnection, server=server) + redis_client = aioredis.FakeRedis(server=server, decode_responses=True) - yield pool - - await pool.disconnect() + try: + yield redis_client + finally: + await redis_client.close() @pytest.fixture @@ -153,26 +154,24 @@ async def dbsession_factory( @pytest.fixture def fastapi_app( dbsession: AsyncSession, - fake_redis_pool: ConnectionPool, + fake_redis_client: aioredis.FakeRedis, set_mock_user, # Fixture for setting up mock authentication user_roles: List[RoleEnum] = [RoleEnum.ADMINISTRATOR], # Default role ) -> FastAPI: # Create the FastAPI application instance application = get_app() application.dependency_overrides[get_async_db_session] = lambda: dbsession - application.dependency_overrides[get_redis_pool] = lambda: fake_redis_pool + application.dependency_overrides[get_redis_client] = lambda: fake_redis_client # Set up application state for testing - application.state.redis_pool = fake_redis_pool - # application.state.db_session_factory = test_session_factory + application.state.redis_client = fake_redis_client application.state.settings = settings # Set up mock authentication backend with the specified roles set_mock_user(application, user_roles) # Initialize the cache with fake Redis backend - fake_redis = aioredis.FakeRedis(connection_pool=fake_redis_pool) - FastAPICache.init(RedisBackend(fake_redis), prefix="lcfs") + FastAPICache.init(RedisBackend(fake_redis_client), prefix="lcfs") return application diff --git a/backend/lcfs/db/dependencies.py b/backend/lcfs/db/dependencies.py index dfcd2f393..b78c885a2 100644 --- a/backend/lcfs/db/dependencies.py +++ b/backend/lcfs/db/dependencies.py @@ -17,6 +17,7 @@ async_engine = create_async_engine(db_url, future=True) logging.getLogger("sqlalchemy.engine").setLevel(logging.WARN) + async def set_user_context(session: AsyncSession, username: str): """ Set user_id context for the session to be used in auditing. @@ -49,15 +50,3 @@ async def get_async_db_session(request: Request) -> AsyncGenerator[AsyncSession, raise e finally: await session.close() # Always close the session to free up the connection - - -def create_redis(): - return aioredis.ConnectionPool( - host=settings.redis_host, - port=settings.redis_port, - db=settings.redis_db, - decode_responses=True, - ) - - -pool = create_redis() diff --git a/backend/lcfs/db/migrations/versions/2024-12-04-23-00_8491890dd688.py b/backend/lcfs/db/migrations/versions/2024-12-04-23-00_8491890dd688.py new file mode 100644 index 000000000..d12c0a57a --- /dev/null +++ b/backend/lcfs/db/migrations/versions/2024-12-04-23-00_8491890dd688.py @@ -0,0 +1,57 @@ +"""Data Fixes + +Revision ID: 8491890dd688 +Revises: aeaa26f5cdd5 +Create Date: 2024-12-04 23:00:10.708533 + +""" + +from alembic import op +from sqlalchemy import update + +from lcfs.db.models import FuelType, AllocationTransactionType +from lcfs.db.models.fuel.FuelType import QuantityUnitsEnum + +# revision identifiers, used by Alembic. +revision = "8491890dd688" +down_revision = "aeaa26f5cdd5" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.execute( + update(FuelType) + .where(FuelType.fuel_type_id == 6) + .values(units=QuantityUnitsEnum.Kilograms) + ) + + op.execute( + update(FuelType).where(FuelType.fuel_type_id == 20).values(fossil_derived=False) + ) + + # Update 'type' and 'description' in allocation_transaction_type where allocation_transaction_type_id = 2 + op.execute( + update(AllocationTransactionType) + .where(AllocationTransactionType.allocation_transaction_type_id == 2) + .values( + type="Allocated to", + description="Fuel allocated to another supplier under an allocation agreement", + ) + ) + + # Update 'type' and 'description' in allocation_transaction_type where allocation_transaction_type_id = 1 + op.execute( + update(AllocationTransactionType) + .where(AllocationTransactionType.allocation_transaction_type_id == 1) + .values( + type="Allocated from", + description="Fuel allocated from another supplier under an allocation agreement", + ) + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + pass diff --git a/backend/lcfs/db/seeders/common/allocation_agreement_seeder.py b/backend/lcfs/db/seeders/common/allocation_agreement_seeder.py index 8275d1b1a..e48a57227 100644 --- a/backend/lcfs/db/seeders/common/allocation_agreement_seeder.py +++ b/backend/lcfs/db/seeders/common/allocation_agreement_seeder.py @@ -18,15 +18,15 @@ async def seed_allocation_transaction_types(session): allocation_transaction_types_to_seed = [ { "allocation_transaction_type_id": 1, - "type": "Purchased", - "description": "Fuel purchased under an allocation agreement", + "type": "Allocated from", + "description": "Fuel allocated from another supplier under an allocation agreement", "display_order": 1, "effective_date": datetime.strptime("2012-01-01", "%Y-%m-%d").date(), }, { "allocation_transaction_type_id": 2, - "type": "Sold", - "description": "Fuel sold under an allocation agreement", + "type": "Allocated to", + "description": "Fuel allocated to another supplier under an allocation agreement", "display_order": 2, "effective_date": datetime.strptime("2012-01-01", "%Y-%m-%d").date(), }, @@ -43,7 +43,6 @@ async def seed_allocation_transaction_types(session): transaction_type = AllocationTransactionType(**type_data) session.add(transaction_type) - logger.info("Successfully seeded allocation transaction types.") except Exception as e: context = { "function": "seed_allocation_transaction_types", diff --git a/backend/lcfs/db/seeders/common/seed_fuel_data.json b/backend/lcfs/db/seeders/common/seed_fuel_data.json index 828d7576d..bc4bf1d87 100644 --- a/backend/lcfs/db/seeders/common/seed_fuel_data.json +++ b/backend/lcfs/db/seeders/common/seed_fuel_data.json @@ -83,7 +83,7 @@ "provision_1_id": 2, "provision_2_id": 3, "default_carbon_intensity": 123.96, - "units": "kWh", + "units": "kg", "unrecognized": false }, { @@ -97,7 +97,6 @@ "units": "kg", "unrecognized": false }, - { "fuel_type_id": 11, "fuel_type": "Alternative jet fuel", @@ -328,42 +327,52 @@ }, { "end_use_type_id": 14, - "type": "Marine", - "sub_type": "General", + "type": "Aircraft", "intended_use": true }, { "end_use_type_id": 15, - "type": "Marine", - "sub_type": "Operated within 51 to 75% of load range" + "type": "Compression-ignition engine- Marine, general", + "intended_use": true }, { "end_use_type_id": 16, - "type": "Marine", - "sub_type": "Operated within 76 to 100% of load range" + "type": "Compression-ignition engine- Marine, operated within 51 to 75% of load range", + "intended_use": true }, { "end_use_type_id": 17, - "type": "Marine, w/ methane slip reduction kit", - "sub_type": "General" + "type": "Compression-ignition engine- Marine, operated within 76 to 100% of load range", + "intended_use": true }, { "end_use_type_id": 18, - "type": "Marine, w/ methane slip reduction kit", - "sub_type": "Operated within 51 to 75% of load range" + "type": "Compression-ignition engine- Marine, with methane slip reduction kit- General", + "intended_use": true }, { "end_use_type_id": 19, - "type": "Marine, w/ methane slip reduction kit", - "sub_type": "Operated within 76 to 100% of load range" + "type": "Compression-ignition engine- Marine, with methane slip reduction kit- Operated within 51 to 75% of load range", + "intended_use": true }, { "end_use_type_id": 20, - "type": "Unknown" + "type": "Compression-ignition engine- Marine, with methane slip reduction kit- Operated within 76 to 100% of load range", + "intended_use": true }, { "end_use_type_id": 21, - "type": "Aircraft", + "type": "Compression-ignition engine- Marine, unknown whether kit is installed or average operating load range", + "intended_use": true + }, + { + "end_use_type_id": 22, + "type": "Unknown engine type", + "intended_use": true + }, + { + "end_use_type_id": 23, + "type": "Other (i.e. road transportation)", "intended_use": true } ], @@ -399,60 +408,74 @@ "additional_uci_id": 1, "fuel_type_id": 7, "uom_id": 5, - "end_use_type_id": 14, - "intensity": 27.3 + "intensity": 0 }, { "additional_uci_id": 2, + "uom_id": 5, + "intensity": 0 + }, + { + "additional_uci_id": 3, "fuel_type_id": 7, "uom_id": 5, "end_use_type_id": 15, - "intensity": 17.8 + "intensity": 27.3 }, { - "additional_uci_id": 3, + "additional_uci_id": 4, "fuel_type_id": 7, "uom_id": 5, "end_use_type_id": 16, - "intensity": 12.2 + "intensity": 17.8 }, { - "additional_uci_id": 4, + "additional_uci_id": 5, "fuel_type_id": 7, "uom_id": 5, "end_use_type_id": 17, - "intensity": 10.6 + "intensity": 12.2 }, { - "additional_uci_id": 5, + "additional_uci_id": 6, "fuel_type_id": 7, "uom_id": 5, "end_use_type_id": 18, - "intensity": 8.4 + "intensity": 10.6 }, { - "additional_uci_id": 6, + "additional_uci_id": 7, "fuel_type_id": 7, "uom_id": 5, "end_use_type_id": 19, - "intensity": 8.0 + "intensity": 8.4 }, { - "additional_uci_id": 7, + "additional_uci_id": 8, "fuel_type_id": 7, "uom_id": 5, "end_use_type_id": 20, + "intensity": 8.0 + }, + { + "additional_uci_id": 9, + "fuel_type_id": 7, + "uom_id": 5, + "end_use_type_id": 21, "intensity": 27.3 }, { - "additional_uci_id": 8, + "additional_uci_id": 10, "fuel_type_id": 7, "uom_id": 5, - "intensity": 0 + "end_use_type_id": 22, + "intensity": 27.3 }, { - "additional_uci_id": 9, + "additional_uci_id": 11, + "fuel_type_id": 7, "uom_id": 5, + "end_use_type_id": 23, "intensity": 0 } ], @@ -549,75 +572,117 @@ "eer_id": 14, "fuel_category_id": 2, "fuel_type_id": 3, - "end_use_type_id": 14, - "ratio": 2.5 - }, - { - "eer_id": 15, - "fuel_category_id": 2, - "fuel_type_id": 3, "end_use_type_id": 10, "ratio": 2.8 }, { - "eer_id": 16, + "eer_id": 15, "fuel_category_id": 2, "fuel_type_id": 3, "end_use_type_id": 11, "ratio": 2.4 }, { - "eer_id": 17, + "eer_id": 16, "fuel_category_id": 2, "fuel_type_id": 3, "end_use_type_id": 2, "ratio": 1.0 }, { - "eer_id": 18, + "eer_id": 17, "fuel_category_id": 2, "fuel_type_id": 6, "end_use_type_id": 3, "ratio": 1.8 }, { - "eer_id": 19, + "eer_id": 18, "fuel_category_id": 2, "fuel_type_id": 6, "end_use_type_id": 2, "ratio": 0.9 }, + { + "eer_id": 19, + "fuel_category_id": 2, + "fuel_type_id": 13, + "ratio": 0.9 + }, { "eer_id": 20, + "fuel_category_id": 3, + "fuel_type_id": 3, + "ratio": 2.5 + }, + { + "eer_id": 21, + "fuel_category_id": 3, + "fuel_type_id": 11, + "ratio": 1.0 + }, + { + "eer_id": 22, "fuel_category_id": 2, "fuel_type_id": 7, - "end_use_type_id": 12, + "end_use_type_id": 15, "ratio": 1.0 }, { - "eer_id": 21, + "eer_id": 23, "fuel_category_id": 2, "fuel_type_id": 7, - "end_use_type_id": 2, - "ratio": 0.9 + "end_use_type_id": 16, + "ratio": 1.0 }, { - "eer_id": 22, + "eer_id": 24, "fuel_category_id": 2, - "fuel_type_id": 13, - "ratio": 0.9 + "fuel_type_id": 7, + "end_use_type_id": 17, + "ratio": 1.0 }, { - "eer_id": 23, - "fuel_category_id": 3, - "fuel_type_id": 3, - "ratio": 2.5 + "eer_id": 25, + "fuel_category_id": 2, + "fuel_type_id": 7, + "end_use_type_id": 18, + "ratio": 1.0 }, { - "eer_id": 24, - "fuel_category_id": 3, - "fuel_type_id": 11, + "eer_id": 26, + "fuel_category_id": 2, + "fuel_type_id": 7, + "end_use_type_id": 19, "ratio": 1.0 + }, + { + "eer_id": 27, + "fuel_category_id": 2, + "fuel_type_id": 7, + "end_use_type_id": 20, + "ratio": 1.0 + }, + { + "eer_id": 28, + "fuel_category_id": 2, + "fuel_type_id": 7, + "end_use_type_id": 21, + "ratio": 1.0 + }, + { + "eer_id": 29, + "fuel_category_id": 2, + "fuel_type_id": 7, + "end_use_type_id": 22, + "ratio": 0.9 + }, + { + "eer_id": 30, + "fuel_category_id": 2, + "fuel_type_id": 7, + "end_use_type_id": 23, + "ratio": 0.9 } ], "energy_densities": [ @@ -1027,4 +1092,4 @@ "display_order": 4 } ] -} +} \ No newline at end of file diff --git a/backend/lcfs/dependencies/dependencies.py b/backend/lcfs/dependencies/dependencies.py deleted file mode 100644 index 9d160d0dc..000000000 --- a/backend/lcfs/dependencies/dependencies.py +++ /dev/null @@ -1,9 +0,0 @@ -from fastapi import Request -from redis.asyncio import Redis -import boto3 - -async def get_redis_pool(request: Request) -> Redis: - return request.app.state.redis_pool - -async def get_s3_client(request: Request) -> boto3.client: - return request.app.state.s3_client \ No newline at end of file diff --git a/backend/lcfs/services/keycloak/authentication.py b/backend/lcfs/services/keycloak/authentication.py index 86026ba6f..dd6d51f29 100644 --- a/backend/lcfs/services/keycloak/authentication.py +++ b/backend/lcfs/services/keycloak/authentication.py @@ -2,9 +2,8 @@ import httpx import jwt -from fastapi import HTTPException, Depends -from redis import ConnectionPool -from redis.asyncio import Redis +from fastapi import HTTPException +from redis.asyncio import Redis, ConnectionPool from sqlalchemy import func from sqlalchemy.exc import NoResultFound from sqlalchemy.ext.asyncio import async_sessionmaker @@ -27,42 +26,62 @@ class UserAuthentication(AuthenticationBackend): def __init__( self, - redis_pool: Redis, + redis_client: Redis, session_factory: async_sessionmaker, settings: Settings, ): self.session_factory = session_factory self.settings = settings - self.redis_pool = redis_pool + self.redis_client = redis_client self.jwks = None self.jwks_uri = None self.test_keycloak_user = None async def refresh_jwk(self): - # Try to get the JWKS data from Redis cache - jwks_data = await self.redis_pool.get("jwks_data") - - if jwks_data: - jwks_data = json.loads(jwks_data) - self.jwks = jwks_data.get("jwks") - self.jwks_uri = jwks_data.get("jwks_uri") - return - - # If not in cache, retrieve from the well-known endpoint - async with httpx.AsyncClient() as client: - oidc_response = await client.get(self.settings.well_known_endpoint) - jwks_uri = oidc_response.json().get("jwks_uri") - certs_response = await client.get(jwks_uri) - jwks = certs_response.json() - - # Composite object containing both JWKS and JWKS URI - jwks_data = {"jwks": jwks, "jwks_uri": jwks_uri} - - # Cache the composite JWKS data with a TTL of 1 day (86400 seconds) - await self.redis_pool.set("jwks_data", json.dumps(jwks_data), ex=86400) - - self.jwks = jwks - self.jwks_uri = jwks_uri + """ + Refreshes the JSON Web Key (JWK) used for token verification. + This method attempts to retrieve the JWK from Redis cache. + If not found, it fetches it from the well-known endpoint + and stores it in Redis for future use. + """ + try: + # Try to get the JWKS data from Redis cache + jwks_data = await self.redis_client.get("jwks_data") + + if jwks_data: + jwks_data = json.loads(jwks_data) + self.jwks = jwks_data.get("jwks") + self.jwks_uri = jwks_data.get("jwks_uri") + return + + # If not in cache, retrieve from the well-known endpoint + async with httpx.AsyncClient() as client: + oidc_response = await client.get(self.settings.well_known_endpoint) + oidc_response.raise_for_status() + jwks_uri = oidc_response.json().get("jwks_uri") + + if not jwks_uri: + raise ValueError( + "JWKS URI not found in the well-known endpoint response." + ) + + certs_response = await client.get(jwks_uri) + certs_response.raise_for_status() + jwks = certs_response.json() + + # Composite object containing both JWKS and JWKS URI + jwks_data = {"jwks": jwks, "jwks_uri": jwks_uri} + + # Cache the composite JWKS data with a TTL of 1 day (86400 seconds) + await self.redis_client.set("jwks_data", json.dumps(jwks_data), ex=86400) + + self.jwks = jwks + self.jwks_uri = jwks_uri + + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Error refreshing JWK: {str(e)}" + ) async def authenticate(self, request): # Extract the authorization header from the request diff --git a/backend/lcfs/services/rabbitmq/transaction_consumer.py b/backend/lcfs/services/rabbitmq/transaction_consumer.py index 10e5367f4..381142d6c 100644 --- a/backend/lcfs/services/rabbitmq/transaction_consumer.py +++ b/backend/lcfs/services/rabbitmq/transaction_consumer.py @@ -4,7 +4,7 @@ from redis.asyncio import Redis from sqlalchemy.ext.asyncio import AsyncSession -from lcfs.dependencies.dependencies import get_redis_pool +from lcfs.services.redis.dependency import get_redis_client from fastapi import Request from lcfs.db.dependencies import async_engine @@ -50,14 +50,14 @@ async def process_message(self, body: bytes, request: Request): compliance_units = message_content.get("compliance_units_amount") org_id = message_content.get("organization_id") - redis = await get_redis_pool(request) + redis_client = await get_redis_client(request) async with AsyncSession(async_engine) as session: async with session.begin(): repo = OrganizationsRepository(db=session) transaction_repo = TransactionRepository(db=session) redis_balance_service = RedisBalanceService( - transaction_repo=transaction_repo, redis_pool=redis.connection_pool + transaction_repo=transaction_repo, redis_client=redis_client ) org_service = OrganizationsService( repo=repo, diff --git a/backend/lcfs/services/redis/dependency.py b/backend/lcfs/services/redis/dependency.py index 368994ffd..01cc689eb 100644 --- a/backend/lcfs/services/redis/dependency.py +++ b/backend/lcfs/services/redis/dependency.py @@ -1,26 +1,19 @@ -from typing import AsyncGenerator - from redis.asyncio import Redis from starlette.requests import Request -async def get_redis_pool( +# Redis Client Dependency +async def get_redis_client( request: Request, -) -> AsyncGenerator[Redis, None]: # pragma: no cover +) -> Redis: """ - Returns connection pool. - - You can use it like this: - - >>> from redis.asyncio import ConnectionPool, Redis - >>> - >>> async def handler(redis_pool: ConnectionPool = Depends(get_redis_pool)): - >>> async with Redis(connection_pool=redis_pool) as redis: - >>> await redis.get('key') + Returns the Redis client. - I use pools, so you don't acquire connection till the end of the handler. + Usage: + >>> async def handler(redis_client: Redis = Depends(get_redis_client)): + >>> value = await redis_client.get('key') - :param request: current request. - :returns: redis connection pool. + :param request: Current request object. + :returns: Redis client. """ - return request.app.state.redis_pool + return request.app.state.redis_client diff --git a/backend/lcfs/services/redis/lifetime.py b/backend/lcfs/services/redis/lifetime.py index 3959edbff..3b7347ea2 100644 --- a/backend/lcfs/services/redis/lifetime.py +++ b/backend/lcfs/services/redis/lifetime.py @@ -1,45 +1,63 @@ import logging from fastapi import FastAPI -from redis import asyncio as aioredis -from redis.exceptions import RedisError - +from redis.asyncio import Redis +from redis.exceptions import RedisError, TimeoutError +import asyncio from lcfs.settings import settings logger = logging.getLogger(__name__) + async def init_redis(app: FastAPI) -> None: """ - Creates connection pool for redis. + Initializes the Redis client and tests the connection. - :param app: current fastapi application. + :param app: current FastAPI application. """ - try: - app.state.redis_pool = aioredis.from_url( - str(settings.redis_url), - encoding="utf8", - decode_responses=True, - max_connections=200 - ) - await app.state.redis_pool.ping() - logger.info("Redis pool initialized successfully.") - except RedisError as e: - logger.error(f"Redis error during initialization: {e}") - raise - except Exception as e: - logger.error(f"Unexpected error during Redis initialization: {e}") - raise + retries = 5 # Retry logic in case Redis is unavailable initially + for i in range(retries): + try: + # Initialize Redis client + app.state.redis_client = Redis( + host=settings.redis_host, + port=settings.redis_port, + password=settings.redis_pass, + db=settings.redis_base or 0, + decode_responses=True, + max_connections=10, + socket_timeout=5, + socket_connect_timeout=5, + ) + + # Test the connection + await app.state.redis_client.ping() + logger.info("Redis client initialized and connection successful.") + break + except TimeoutError as e: + logger.error(f"Redis timeout during initialization attempt {i + 1}: {e}") + if i == retries - 1: + raise + await asyncio.sleep(2**i) # Exponential backoff + except RedisError as e: + logger.error(f"Redis error during initialization attempt {i + 1}: {e}") + if i == retries - 1: + raise + await asyncio.sleep(2**i) # Exponential backoff + except Exception as e: + logger.error(f"Unexpected error during Redis initialization: {e}") + raise + -async def shutdown_redis(app: FastAPI) -> None: # pragma: no cover +async def shutdown_redis(app: FastAPI) -> None: """ - Closes redis connection pool. + Closes the Redis client during application shutdown. :param app: current FastAPI app. """ try: - if hasattr(app.state, "redis_pool"): - await app.state.redis_pool.close() - await app.state.redis_pool.wait_closed() - logger.info("Redis pool closed successfully.") + if hasattr(app.state, "redis_client") and app.state.redis_client: + await app.state.redis_client.close() + logger.info("Redis client closed successfully.") except RedisError as e: logger.error(f"Redis error during shutdown: {e}") except Exception as e: diff --git a/backend/lcfs/services/s3/client.py b/backend/lcfs/services/s3/client.py index c03b54993..260f8b0ea 100644 --- a/backend/lcfs/services/s3/client.py +++ b/backend/lcfs/services/s3/client.py @@ -1,14 +1,10 @@ import os import uuid - -import boto3 -from fastapi import Depends, Request +from fastapi import Depends from pydantic.v1 import ValidationError from sqlalchemy import select -from sqlalchemy.exc import InvalidRequestError from sqlalchemy.ext.asyncio import AsyncSession -from lcfs.dependencies.dependencies import get_s3_client - +from lcfs.services.s3.dependency import get_s3_client from lcfs.db.dependencies import get_async_db_session from lcfs.db.models.compliance import ComplianceReport from lcfs.db.models.compliance.ComplianceReport import ( @@ -20,7 +16,6 @@ from lcfs.web.core.decorators import repo_handler BUCKET_NAME = settings.s3_bucket - MAX_FILE_SIZE_MB = 50 MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024 # Convert MB to bytes @@ -28,13 +23,13 @@ class DocumentService: def __init__( self, - request: Request, db: AsyncSession = Depends(get_async_db_session), clamav_service: ClamAVService = Depends(), + s3_client=Depends(get_s3_client), ): self.db = db self.clamav_service = clamav_service - self.s3_client = request.app.state.s3_client + self.s3_client = s3_client @repo_handler async def upload_file(self, file, parent_id: str, parent_type="compliance_report"): @@ -45,8 +40,8 @@ async def upload_file(self, file, parent_id: str, parent_type="compliance_report file_size = os.fstat(file.file.fileno()).st_size if file_size > MAX_FILE_SIZE_BYTES: - raise InvalidRequestError( - detail=f"File size exceeds the maximum limit of {MAX_FILE_SIZE_MB} MB.", + raise ValidationError( + f"File size exceeds the maximum limit of {MAX_FILE_SIZE_MB} MB." ) if settings.clamav_enabled: @@ -89,7 +84,6 @@ async def upload_file(self, file, parent_id: str, parent_type="compliance_report return document - # Generate a pre-signed URL for downloading a file from S3 @repo_handler async def generate_presigned_url(self, document_id: int): document = await self.db.get_one(Document, document_id) @@ -104,7 +98,6 @@ async def generate_presigned_url(self, document_id: int): ) return presigned_url - # Delete a file from S3 and remove the entry from the database @repo_handler async def delete_file(self, document_id: int): document = await self.db.get_one(Document, document_id) diff --git a/backend/lcfs/services/s3/dependency.py b/backend/lcfs/services/s3/dependency.py new file mode 100644 index 000000000..fc38ed388 --- /dev/null +++ b/backend/lcfs/services/s3/dependency.py @@ -0,0 +1,32 @@ +import boto3 +from typing import Generator +from lcfs.settings import settings + + +def get_s3_client() -> Generator: + """ + Dependency function to provide a synchronous S3 client using boto3. + + This function creates a new S3 client session for each request that requires it. + The client is properly configured with the necessary AWS credentials and + endpoint settings. + + Usage: + >>> def some_endpoint(s3_client = Depends(get_s3_client)): + >>> # Use the s3_client here + """ + # Initialize the S3 client with the required configurations + client = boto3.client( + "s3", + aws_access_key_id=settings.s3_access_key, # Your AWS access key + aws_secret_access_key=settings.s3_secret_key, # Your AWS secret key + endpoint_url=settings.s3_endpoint, # Custom S3 endpoint (if any) + region_name="us-east-1", # AWS region + ) + + try: + # Yield the S3 client to be used within the request scope + yield client + finally: + # boto3 clients do not require explicit closing, but this ensures cleanup if needed + pass diff --git a/backend/lcfs/services/tfrs/redis_balance.py b/backend/lcfs/services/tfrs/redis_balance.py index 69dc96010..2628a831a 100644 --- a/backend/lcfs/services/tfrs/redis_balance.py +++ b/backend/lcfs/services/tfrs/redis_balance.py @@ -1,12 +1,12 @@ import logging from datetime import datetime -from fastapi import FastAPI, Depends, Request -from redis.asyncio import Redis, ConnectionPool +from fastapi import FastAPI, Depends +from redis.asyncio import Redis from sqlalchemy.ext.asyncio import AsyncSession from lcfs.db.dependencies import async_engine -from lcfs.services.redis.dependency import get_redis_pool +from lcfs.services.redis.dependency import get_redis_client from lcfs.settings import settings from lcfs.web.api.organizations.repo import OrganizationsRepository from lcfs.web.api.transaction.repo import TransactionRepository @@ -17,10 +17,16 @@ async def init_org_balance_cache(app: FastAPI): - redis = await app.state.redis_pool + """ + Initialize the organization balance cache and populate it with data. + + :param app: FastAPI application instance. + """ + # Get the Redis client from app state + redis: Redis = app.state.redis_client + async with AsyncSession(async_engine) as session: async with session.begin(): - organization_repo = OrganizationsRepository(db=session) transaction_repo = TransactionRepository(db=session) @@ -29,21 +35,27 @@ async def init_org_balance_cache(app: FastAPI): # Get the current year current_year = datetime.now().year - logger.info(f"Starting balance cache population {current_year}") + logger.info(f"Starting balance cache population for {current_year}") + # Fetch all organizations all_orgs = await organization_repo.get_organizations() # Loop from the oldest year to the current year for year in range(int(oldest_year), current_year + 1): - # Call the function to process transactions for each year for org in all_orgs: + # Calculate the balance for each organization and year balance = ( await transaction_repo.calculate_available_balance_for_period( org.organization_id, year ) ) + # Set the balance in Redis await set_cache_value(org.organization_id, year, balance, redis) - logger.debug(f"Set balance for {org.name} for {year} to {balance}") + logger.debug( + f"Set balance for organization {org.name} " + f"for {year} to {balance}" + ) + logger.info(f"Cache populated with {len(all_orgs)} organizations") @@ -51,10 +63,10 @@ class RedisBalanceService: def __init__( self, transaction_repo=Depends(TransactionRepository), - redis_pool: ConnectionPool = Depends(get_redis_pool), + redis_client: Redis = Depends(get_redis_client), ): self.transaction_repo = transaction_repo - self.redis_pool = redis_pool + self.redis_client = redis_client @service_handler async def populate_organization_redis_balance( @@ -74,8 +86,7 @@ async def populate_organization_redis_balance( ) ) - async with Redis(connection_pool=self.redis_pool) as redis: - await set_cache_value(organization_id, year, balance, redis) + await set_cache_value(organization_id, year, balance, self.redis_client) logger.debug( f"Set balance for org {organization_id} for {year} to {balance}" ) @@ -84,4 +95,12 @@ async def populate_organization_redis_balance( async def set_cache_value( organization_id: int, period: int, balance: int, redis: Redis ) -> None: + """ + Set a cache value in Redis for a specific organization and period. + + :param organization_id: ID of the organization. + :param period: The year or period for which the balance is being set. + :param balance: The balance value to set in the cache. + :param redis: Redis client instance. + """ await redis.set(name=f"balance_{organization_id}_{period}", value=balance) diff --git a/backend/lcfs/settings.py b/backend/lcfs/settings.py index 5116b5aa6..199ba941d 100644 --- a/backend/lcfs/settings.py +++ b/backend/lcfs/settings.py @@ -30,10 +30,12 @@ class Settings(BaseSettings): host: str = "0.0.0.0" port: int = 8000 - # quantity of workers for uvicorn + # Number of Uvicorn workers workers_count: int = 2 - # Enable uvicorn reloading - reload: bool = True + # Enable Uvicorn reload (True for development, False for production) + reload: bool = False + # App timeout matching OpenShift's ROUTER_DEFAULT_SERVER_TIMEOUT + timeout_keep_alive: int = 30 # Current environment environment: str = "dev" diff --git a/backend/lcfs/tests/compliance_report/test_compliance_report_views.py b/backend/lcfs/tests/compliance_report/test_compliance_report_views.py index 9741e568d..aa5ca7675 100644 --- a/backend/lcfs/tests/compliance_report/test_compliance_report_views.py +++ b/backend/lcfs/tests/compliance_report/test_compliance_report_views.py @@ -12,6 +12,7 @@ from lcfs.web.api.compliance_report.schema import ( ComplianceReportUpdateSchema, ComplianceReportSummaryUpdateSchema, + ChainedComplianceReportSchema, ) from lcfs.services.s3.client import DocumentService @@ -226,7 +227,9 @@ async def test_get_compliance_report_by_id_success( ) as mock_validate_organization_access: set_mock_user(fastapi_app, [RoleEnum.GOVERNMENT]) - mock_compliance_report = compliance_report_base_schema() + mock_compliance_report = ChainedComplianceReportSchema( + report=compliance_report_base_schema(), chain=[] + ) mock_get_compliance_report_by_id.return_value = mock_compliance_report mock_validate_organization_access.return_value = None @@ -240,7 +243,9 @@ async def test_get_compliance_report_by_id_success( expected_response = json.loads(mock_compliance_report.json(by_alias=True)) assert response.json() == expected_response - mock_get_compliance_report_by_id.assert_called_once_with(1, False) + mock_get_compliance_report_by_id.assert_called_once_with( + 1, False, get_chain=True + ) mock_validate_organization_access.assert_called_once_with(1) diff --git a/backend/lcfs/tests/conftest.py b/backend/lcfs/tests/conftest.py index 14f4f3e7d..5cdd52ce1 100644 --- a/backend/lcfs/tests/conftest.py +++ b/backend/lcfs/tests/conftest.py @@ -2,6 +2,7 @@ from lcfs.db.models.user.Role import RoleEnum from lcfs.web.api.base import PaginationRequestSchema, FilterModel, SortOrder +from fakeredis.aioredis import FakeRedis @pytest.fixture @@ -54,3 +55,15 @@ def role_names(self): return self.role_names return MockUserProfile() + + +@pytest.fixture +async def redis_client(): + """ + Fixture to provide a fake Redis client for tests. + """ + client = FakeRedis() + try: + yield client + finally: + await client.close() diff --git a/backend/lcfs/tests/organization/test_organization_views.py b/backend/lcfs/tests/organization/test_organization_views.py index a98a1a306..3f8cacafc 100644 --- a/backend/lcfs/tests/organization/test_organization_views.py +++ b/backend/lcfs/tests/organization/test_organization_views.py @@ -16,6 +16,7 @@ from lcfs.web.api.organization.validation import OrganizationValidation from lcfs.web.api.compliance_report.services import ComplianceReportServices +from lcfs.web.api.compliance_report.schema import ChainedComplianceReportSchema @pytest.mark.anyio @@ -160,7 +161,8 @@ async def test_export_transactions_for_org_success( ): set_mock_user(fastapi_app, [RoleEnum.SUPPLIER]) - mock_transactions_services.export_transactions.return_value = {"streaming": True} + mock_transactions_services.export_transactions.return_value = { + "streaming": True} fastapi_app.dependency_overrides[TransactionsService] = ( lambda: mock_transactions_services @@ -188,7 +190,8 @@ async def test_create_transfer_success( set_mock_user(fastapi_app, [RoleEnum.SUPPLIER]) organization_id = 1 - url = fastapi_app.url_path_for("create_transfer", organization_id=organization_id) + url = fastapi_app.url_path_for( + "create_transfer", organization_id=organization_id) payload = {"from_organization_id": 1, "to_organization_id": 2} @@ -226,7 +229,8 @@ async def test_update_transfer_success( ): set_mock_user(fastapi_app, [RoleEnum.SUPPLIER]) - url = fastapi_app.url_path_for("update_transfer", organization_id=1, transfer_id=1) + url = fastapi_app.url_path_for( + "update_transfer", organization_id=1, transfer_id=1) payload = {"from_organization_id": 1, "to_organization_id": 2} @@ -274,7 +278,8 @@ async def test_create_compliance_report_success( "create_compliance_report", organization_id=organization_id ) - payload = {"compliance_period": "2024", "organization_id": 1, "status": "status"} + payload = {"compliance_period": "2024", + "organization_id": 1, "status": "status"} mock_organization_validation.create_compliance_report.return_value = None mock_compliance_report_services.create_compliance_report.return_value = { @@ -346,7 +351,8 @@ async def test_get_all_org_reported_years_success( ): set_mock_user(fastapi_app, [RoleEnum.SUPPLIER]) - url = fastapi_app.url_path_for("get_all_org_reported_years", organization_id=1) + url = fastapi_app.url_path_for( + "get_all_org_reported_years", organization_id=1) mock_compliance_report_services.get_all_org_reported_years.return_value = [ {"compliance_period_id": 1, "description": "2024"} @@ -379,20 +385,23 @@ async def test_get_compliance_report_by_id_success( ) # Mock the compliance report service's method - mock_compliance_report_services.get_compliance_report_by_id.return_value = { - "compliance_report_id": 1, - "compliance_period_id": 1, - "compliance_period": {"compliance_period_id": 1, "description": "2024"}, - "organization_id": 1, - "organization": {"organization_id": 1, "name": "org1"}, - "current_status_id": 1, - "current_status": {"compliance_report_status_id": 1, "status": "status"}, - "summary": {"summary_id": 1, "is_locked": False}, - "compliance_report_group_uuid": "uuid", - "version": 0, - "supplemental_initiator": SupplementalInitiatorType.SUPPLIER_SUPPLEMENTAL, - "has_supplemental": False, - } + mock_compliance_report_services.get_compliance_report_by_id.return_value = ChainedComplianceReportSchema( + report={ + "compliance_report_id": 1, + "compliance_period_id": 1, + "compliance_period": {"compliance_period_id": 1, "description": "2024"}, + "organization_id": 1, + "organization": {"organization_id": 1, "name": "org1"}, + "current_status_id": 1, + "current_status": {"compliance_report_status_id": 1, "status": "status"}, + "summary": {"summary_id": 1, "is_locked": False}, + "compliance_report_group_uuid": "uuid", + "version": 0, + "supplemental_initiator": SupplementalInitiatorType.SUPPLIER_SUPPLEMENTAL, + "has_supplemental": False, + }, + chain=[] + ) # Create a mock for the validation service mock_compliance_report_validation = AsyncMock() @@ -412,7 +421,7 @@ async def test_get_compliance_report_by_id_success( # Assertions assert response.status_code == 200 mock_compliance_report_services.get_compliance_report_by_id.assert_awaited_once_with( - 1, apply_masking=True + 1, apply_masking=True, get_chain=True ) mock_compliance_report_validation.validate_organization_access.assert_awaited_once_with( 1 diff --git a/backend/lcfs/tests/services/redis/test_redis.py b/backend/lcfs/tests/services/redis/test_redis.py index e10906b8d..af520e7b4 100644 --- a/backend/lcfs/tests/services/redis/test_redis.py +++ b/backend/lcfs/tests/services/redis/test_redis.py @@ -1,63 +1,64 @@ -import uuid - import pytest +from unittest.mock import AsyncMock, patch from fastapi import FastAPI -from httpx import AsyncClient -from redis.asyncio import ConnectionPool, Redis -from starlette import status +from redis.exceptions import RedisError +from lcfs.services.redis.lifetime import init_redis, shutdown_redis @pytest.mark.anyio -async def test_setting_value( - fastapi_app: FastAPI, - fake_redis_pool: ConnectionPool, - client: AsyncClient, -) -> None: +async def test_init_redis_success(): + """ + Test Redis initialization succeeds and pings the client. """ - Tests that you can set value in redis. + app = FastAPI() + mock_redis = AsyncMock() + + with patch("lcfs.services.redis.lifetime.Redis", return_value=mock_redis): + # Mock Redis ping to simulate successful connection + mock_redis.ping.return_value = True + + await init_redis(app) + + assert app.state.redis_client is mock_redis + mock_redis.ping.assert_called_once() + mock_redis.close.assert_not_called() - :param fastapi_app: current application fixture. - :param fake_redis_pool: fake redis pool. - :param client: client fixture. + +@pytest.mark.anyio +async def test_init_redis_failure(): """ - url = fastapi_app.url_path_for("set_redis_value") + Test Redis initialization fails during connection. + """ + app = FastAPI() - test_key = uuid.uuid4().hex - test_val = uuid.uuid4().hex - response = await client.put( - url, - json={ - "key": test_key, - "value": test_val, - }, - ) + with patch( + "lcfs.services.redis.lifetime.Redis", + side_effect=RedisError("Connection failed"), + ): + with pytest.raises(RedisError, match="Connection failed"): + await init_redis(app) - assert response.status_code == status.HTTP_200_OK - async with Redis(connection_pool=fake_redis_pool) as redis: - actual_value = await redis.get(test_key) - assert actual_value.decode() == test_val + assert not hasattr(app.state, "redis_client") @pytest.mark.anyio -async def test_getting_value( - fastapi_app: FastAPI, - fake_redis_pool: ConnectionPool, - client: AsyncClient, -) -> None: - """ - Tests that you can get value from redis by key. - - :param fastapi_app: current application fixture. - :param fake_redis_pool: fake redis pool. - :param client: client fixture. - """ - test_key = uuid.uuid4().hex - test_val = uuid.uuid4().hex - async with Redis(connection_pool=fake_redis_pool) as redis: - await redis.set(test_key, test_val) - url = fastapi_app.url_path_for("get_redis_value") - response = await client.get(url, params={"key": test_key}) - - assert response.status_code == status.HTTP_200_OK - assert response.json()["key"] == test_key - assert response.json()["value"] == test_val +async def test_shutdown_redis_success(): + """ + Test Redis client shutdown succeeds. + """ + app = FastAPI() + mock_redis = AsyncMock() + app.state.redis_client = mock_redis + + await shutdown_redis(app) + + mock_redis.close.assert_called_once() + + +@pytest.mark.anyio +async def test_shutdown_redis_no_client(): + """ + Test Redis shutdown when no client exists. + """ + app = FastAPI() + await shutdown_redis(app) # Should not raise any exceptions diff --git a/backend/lcfs/tests/services/tfrs/test_redis_balance.py b/backend/lcfs/tests/services/tfrs/test_redis_balance.py index 56ce31fb1..39504a5f1 100644 --- a/backend/lcfs/tests/services/tfrs/test_redis_balance.py +++ b/backend/lcfs/tests/services/tfrs/test_redis_balance.py @@ -1,8 +1,8 @@ import pytest -from unittest.mock import AsyncMock, patch, MagicMock +from unittest.mock import AsyncMock, patch, MagicMock, call from datetime import datetime -from redis.asyncio import ConnectionPool, Redis +from redis.asyncio import Redis from lcfs.services.tfrs.redis_balance import ( init_org_balance_cache, @@ -13,61 +13,53 @@ @pytest.mark.anyio async def test_init_org_balance_cache(): - # Mock the session and repositories - mock_session = AsyncMock() - # Mock the Redis client mock_redis = AsyncMock() - mock_redis.set = AsyncMock() # Ensure the `set` method is mocked - - # Mock the settings - mock_settings = MagicMock() - mock_settings.redis_url = "redis://localhost" - - # Create a mock app object - mock_app = MagicMock() + mock_redis.set = AsyncMock() - # Simulate redis_pool as an awaitable returning mock_redis - async def mock_redis_pool(): - return mock_redis + # Patch Redis client creation + with patch("lcfs.services.tfrs.redis_balance.Redis", return_value=mock_redis): + # Mock the app object + mock_app = MagicMock() + mock_app.state.redis_client = mock_redis - mock_app.state.redis_pool = mock_redis_pool() - mock_app.state.settings = mock_settings - - current_year = datetime.now().year - last_year = current_year - 1 + current_year = datetime.now().year + last_year = current_year - 1 - with patch( - "lcfs.web.api.organizations.services.OrganizationsRepository.get_organizations", - return_value=[ - MagicMock(organization_id=1, name="Org1"), - MagicMock(organization_id=2, name="Org2"), - ], - ): + # Mock repository methods with patch( + "lcfs.web.api.organizations.repo.OrganizationsRepository.get_organizations", + return_value=[ + MagicMock(organization_id=1, name="Org1"), + MagicMock(organization_id=2, name="Org2"), + ], + ), patch( "lcfs.web.api.transaction.repo.TransactionRepository.get_transaction_start_year", return_value=last_year, + ), patch( + "lcfs.web.api.transaction.repo.TransactionRepository.calculate_available_balance_for_period", + side_effect=[100, 200, 150, 250], ): - with patch( - "lcfs.web.api.transaction.repo.TransactionRepository.calculate_available_balance_for_period", - side_effect=[100, 200, 150, 250, 300, 350], - ): - # Pass the mock app to the function - await init_org_balance_cache(mock_app) - - # Assert that each cache set operation was called correctly - calls = mock_redis.set.mock_calls - assert len(calls) == 4 - mock_redis.set.assert_any_call(name=f"balance_1_{last_year}", value=100) - mock_redis.set.assert_any_call(name=f"balance_2_{last_year}", value=200) - mock_redis.set.assert_any_call(name=f"balance_1_{current_year}", value=150) - mock_redis.set.assert_any_call(name=f"balance_2_{current_year}", value=250) + # Execute the function with the mocked app + await init_org_balance_cache(mock_app) + + # Define expected calls to Redis `set` + expected_calls = [ + call(name=f"balance_1_{last_year}", value=100), + call(name=f"balance_2_{last_year}", value=200), + call(name=f"balance_1_{current_year}", value=150), + call(name=f"balance_2_{current_year}", value=250), + ] + + # Assert that Redis `set` method was called with the expected arguments + mock_redis.set.assert_has_calls(expected_calls, any_order=True) + + # Ensure the number of calls matches the expected count + assert mock_redis.set.call_count == len(expected_calls) @pytest.mark.anyio -async def test_populate_organization_redis_balance( - fake_redis_pool: ConnectionPool, -): +async def test_populate_organization_redis_balance(redis_client: Redis): # Mock the transaction repository current_year = datetime.now().year last_year = current_year - 1 @@ -82,7 +74,7 @@ async def test_populate_organization_redis_balance( # Create an instance of the service with mocked dependencies service = RedisBalanceService( - transaction_repo=mock_transaction_repo, redis_pool=fake_redis_pool + transaction_repo=mock_transaction_repo, redis_client=redis_client ) await service.populate_organization_redis_balance(organization_id=1) @@ -97,12 +89,8 @@ async def test_populate_organization_redis_balance( ) # Assert that the Redis set method was called with the correct parameters - async with Redis(connection_pool=fake_redis_pool) as redis: - assert int(await redis.get(f"balance_1_{last_year}")) == 100 - assert int(await redis.get(f"balance_1_{current_year}")) == 200 - - -# mock_redis.set.assert_any_call(name=f"balance_1_{current_year}", value=200) + assert int(await redis_client.get(f"balance_1_{last_year}")) == 100 + assert int(await redis_client.get(f"balance_1_{current_year}")) == 200 @pytest.mark.anyio diff --git a/backend/lcfs/tests/test_auth_middleware.py b/backend/lcfs/tests/test_auth_middleware.py index d59076107..146ccaf0c 100644 --- a/backend/lcfs/tests/test_auth_middleware.py +++ b/backend/lcfs/tests/test_auth_middleware.py @@ -1,9 +1,11 @@ from unittest.mock import AsyncMock, patch, MagicMock, Mock import pytest -import asyncio +import json +import redis from starlette.exceptions import HTTPException from starlette.requests import Request +from redis.asyncio import Redis, ConnectionPool from lcfs.db.models import UserProfile from lcfs.services.keycloak.authentication import UserAuthentication @@ -11,7 +13,7 @@ @pytest.fixture -def redis_pool(): +def redis_client(): return AsyncMock() @@ -30,48 +32,88 @@ def settings(): @pytest.fixture -def auth_backend(redis_pool, session_generator, settings): - return UserAuthentication(redis_pool, session_generator[0], settings) +def auth_backend(redis_client, session_generator, settings): + return UserAuthentication(redis_client, session_generator[0], settings) @pytest.mark.anyio async def test_load_jwk_from_redis(auth_backend): - # Mock auth_backend.redis_pool.get to return a JSON string directly - with patch.object(auth_backend.redis_pool, "get", new_callable=AsyncMock) as mock_redis_get: - mock_redis_get.return_value = '{"jwks": "jwks", "jwks_uri": "jwks_uri"}' + # Mock auth_backend.redis_client.get to return a JSON string directly + mock_redis = AsyncMock() + mock_redis.get = AsyncMock( + return_value='{"jwks": "jwks_data", "jwks_uri": "jwks_uri_data"}' + ) + # Patch Redis client in the auth backend + with patch.object(auth_backend, "redis_client", mock_redis): await auth_backend.refresh_jwk() - assert auth_backend.jwks == "jwks" - assert auth_backend.jwks_uri == "jwks_uri" + # Assertions to verify JWKS data was loaded correctly + assert auth_backend.jwks == "jwks_data" + assert auth_backend.jwks_uri == "jwks_uri_data" + + # Verify that Redis `get` was called with the correct key + mock_redis.get.assert_awaited_once_with("jwks_data") @pytest.mark.anyio @patch("httpx.AsyncClient.get") -async def test_refresh_jwk_sets_new_keys_in_redis(mock_get, auth_backend): - # Create a mock response object - mock_response = MagicMock() - - # Set up the json method to return a dictionary with a .get method - mock_json = MagicMock() - mock_json.get.return_value = "{}" - - # Assign the mock_json to the json method of the response - mock_response.json.return_value = mock_json - - mock_response_2 = MagicMock() - mock_response_2.json.return_value = "{}" +async def test_refresh_jwk_sets_new_keys_in_redis(mock_httpx_get, redis_client): + # Mock responses for the well-known endpoint and JWKS URI + mock_oidc_response = MagicMock() + mock_oidc_response.json.return_value = {"jwks_uri": "https://example.com/jwks"} + mock_oidc_response.raise_for_status = MagicMock() + + mock_certs_response = MagicMock() + mock_certs_response.json.return_value = { + "keys": [{"kty": "RSA", "kid": "key2", "use": "sig", "n": "def", "e": "AQAB"}] + } + mock_certs_response.raise_for_status = MagicMock() + + # Configure the mock to return the above responses in order + mock_httpx_get.side_effect = [mock_oidc_response, mock_certs_response] + + # Mock Redis client behavior + redis_client.get = AsyncMock(return_value=None) # JWKS data not in cache + redis_client.set = AsyncMock() + + # Create auth_backend with the mocked Redis client + auth_backend = UserAuthentication( + redis_client=redis_client, + session_factory=AsyncMock(), + settings=MagicMock( + well_known_endpoint="https://example.com/.well-known/openid-configuration" + ), + ) - mock_get.side_effect = [ - mock_response, - mock_response_2, - ] + # Call refresh_jwk + await auth_backend.refresh_jwk() - with patch.object(auth_backend.redis_pool, "get", new_callable=AsyncMock) as mock_redis_get: - mock_redis_get.return_value = None + # Assertions to verify JWKS data was fetched and set correctly + expected_jwks = { + "keys": [{"kty": "RSA", "kid": "key2", "use": "sig", "n": "def", "e": "AQAB"}] + } + assert auth_backend.jwks == expected_jwks + assert auth_backend.jwks_uri == "https://example.com/jwks" - await auth_backend.refresh_jwk() + # Verify that Redis `get` was called with "jwks_data" + redis_client.get.assert_awaited_once_with("jwks_data") + # Verify that the well-known endpoint was called twice + assert mock_httpx_get.call_count == 2 + mock_httpx_get.assert_any_call( + "https://example.com/.well-known/openid-configuration" + ) + mock_httpx_get.assert_any_call("https://example.com/jwks") + + # Verify that Redis `set` was called with the correct parameters + expected_jwks_data = { + "jwks": expected_jwks, + "jwks_uri": "https://example.com/jwks", + } + redis_client.set.assert_awaited_once_with( + "jwks_data", json.dumps(expected_jwks_data), ex=86400 + ) @pytest.mark.anyio diff --git a/backend/lcfs/web/api/allocation_agreement/schema.py b/backend/lcfs/web/api/allocation_agreement/schema.py index f5ffe9179..c2079fe2a 100644 --- a/backend/lcfs/web/api/allocation_agreement/schema.py +++ b/backend/lcfs/web/api/allocation_agreement/schema.py @@ -58,6 +58,7 @@ class AllocationAgreementTableOptionsSchema(BaseSchema): class AllocationAgreementCreateSchema(BaseSchema): compliance_report_id: int allocation_agreement_id: Optional[int] = None + allocation_transaction_type: str transaction_partner: str postal_address: str transaction_partner_email: str @@ -65,11 +66,12 @@ class AllocationAgreementCreateSchema(BaseSchema): fuel_type: str fuel_type_other: Optional[str] = None ci_of_fuel: float - quantity: int + provision_of_the_act: str + quantity: int = Field( + ..., gt=0, description="Quantity must be greater than 0" + ) units: str - allocation_transaction_type: str fuel_category: str - provision_of_the_act: Optional[str] = None fuel_code: Optional[str] = None deleted: Optional[bool] = None diff --git a/backend/lcfs/web/api/compliance_report/repo.py b/backend/lcfs/web/api/compliance_report/repo.py index 71a26cb3d..194afb8d0 100644 --- a/backend/lcfs/web/api/compliance_report/repo.py +++ b/backend/lcfs/web/api/compliance_report/repo.py @@ -29,7 +29,6 @@ ) from lcfs.web.api.compliance_report.schema import ( ComplianceReportBaseSchema, - ComplianceReportSummarySchema, ComplianceReportSummaryUpdateSchema, ) from lcfs.db.models.compliance.ComplianceReportHistory import ComplianceReportHistory @@ -435,34 +434,61 @@ async def get_compliance_report_by_id(self, report_id: int, is_model: bool = Fal """ Retrieve a compliance report from the database by ID """ - result = ( - ( - await self.db.execute( - select(ComplianceReport) - .options( - joinedload(ComplianceReport.organization), - joinedload(ComplianceReport.compliance_period), - joinedload(ComplianceReport.current_status), - joinedload(ComplianceReport.summary), - joinedload(ComplianceReport.history).joinedload( - ComplianceReportHistory.status - ), - joinedload(ComplianceReport.history).joinedload( - ComplianceReportHistory.user_profile - ), - joinedload(ComplianceReport.transaction), - ) - .where(ComplianceReport.compliance_report_id == report_id) - ) + result = await self.db.execute( + select(ComplianceReport) + .options( + joinedload(ComplianceReport.organization), + joinedload(ComplianceReport.compliance_period), + joinedload(ComplianceReport.current_status), + joinedload(ComplianceReport.summary), + joinedload(ComplianceReport.history).joinedload( + ComplianceReportHistory.status + ), + joinedload(ComplianceReport.history).joinedload( + ComplianceReportHistory.user_profile + ), + joinedload(ComplianceReport.transaction), ) - .unique() - .scalars() - .first() + .where(ComplianceReport.compliance_report_id == report_id) ) + + compliance_report = result.scalars().unique().first() + + if not compliance_report: + return None + if is_model: - return result - else: - return ComplianceReportBaseSchema.model_validate(result) + return compliance_report + + return ComplianceReportBaseSchema.model_validate(compliance_report) + + @repo_handler + async def get_compliance_report_chain(self, group_uuid: str): + result = await self.db.execute( + select(ComplianceReport) + .options( + joinedload(ComplianceReport.organization), + joinedload(ComplianceReport.compliance_period), + joinedload(ComplianceReport.current_status), + joinedload(ComplianceReport.summary), + joinedload(ComplianceReport.history).joinedload( + ComplianceReportHistory.status + ), + joinedload(ComplianceReport.history).joinedload( + ComplianceReportHistory.user_profile + ), + joinedload(ComplianceReport.transaction), + ) + .where(ComplianceReport.compliance_report_group_uuid == group_uuid) + .order_by(ComplianceReport.version.desc()) # Ensure ordering by version + ) + + compliance_reports = result.scalars().unique().all() + + return [ + ComplianceReportBaseSchema.model_validate(report) + for report in compliance_reports + ] @repo_handler async def get_fuel_type(self, fuel_type_id: int) -> FuelType: diff --git a/backend/lcfs/web/api/compliance_report/schema.py b/backend/lcfs/web/api/compliance_report/schema.py index d427ee7d5..0f157be8b 100644 --- a/backend/lcfs/web/api/compliance_report/schema.py +++ b/backend/lcfs/web/api/compliance_report/schema.py @@ -160,6 +160,11 @@ class ComplianceReportBaseSchema(BaseSchema): has_supplemental: bool +class ChainedComplianceReportSchema(BaseSchema): + report: ComplianceReportBaseSchema + chain: Optional[List[ComplianceReportBaseSchema]] = [] + + class ComplianceReportCreateSchema(BaseSchema): compliance_period: str organization_id: int diff --git a/backend/lcfs/web/api/compliance_report/services.py b/backend/lcfs/web/api/compliance_report/services.py index b5490755b..31993bc75 100644 --- a/backend/lcfs/web/api/compliance_report/services.py +++ b/backend/lcfs/web/api/compliance_report/services.py @@ -52,7 +52,8 @@ async def create_compliance_report( report_data.status ) if not draft_status: - raise DataNotFoundException(f"Status '{report_data.status}' not found.") + raise DataNotFoundException( + f"Status '{report_data.status}' not found.") # Generate a new group_uuid for the new report series group_uuid = str(uuid.uuid4()) @@ -193,6 +194,7 @@ def _mask_report_status(self, reports: List) -> List: ComplianceReportStatusEnum.Submitted.value ) report.current_status.compliance_report_status_id = None + masked_reports.append(report) else: masked_reports.append(report) @@ -201,22 +203,45 @@ def _mask_report_status(self, reports: List) -> List: @service_handler async def get_compliance_report_by_id( - self, report_id: int, apply_masking: bool = False - ) -> ComplianceReportBaseSchema: + self, report_id: int, apply_masking: bool = False, get_chain: bool = False + ): """Fetches a specific compliance report by ID.""" report = await self.repo.get_compliance_report_by_id(report_id) if report is None: raise DataNotFoundException("Compliance report not found.") + validated_report = ComplianceReportBaseSchema.model_validate(report) masked_report = ( self._mask_report_status([validated_report])[0] if apply_masking else validated_report ) + history_masked_report = self._mask_report_status_for_history( masked_report, apply_masking ) + if get_chain: + compliance_report_chain = await self.repo.get_compliance_report_chain( + report.compliance_report_group_uuid + ) + + if apply_masking: + # Apply masking to each report in the chain + masked_chain = self._mask_report_status( + compliance_report_chain) + # Apply history masking to each report in the chain + masked_chain = [ + self._mask_report_status_for_history(report, apply_masking) + for report in masked_chain + ] + compliance_report_chain = masked_chain + + return { + "report": history_masked_report, + "chain": compliance_report_chain, + } + return history_masked_report def _mask_report_status_for_history( diff --git a/backend/lcfs/web/api/compliance_report/views.py b/backend/lcfs/web/api/compliance_report/views.py index c25a8568f..de43c0c26 100644 --- a/backend/lcfs/web/api/compliance_report/views.py +++ b/backend/lcfs/web/api/compliance_report/views.py @@ -19,7 +19,9 @@ ComplianceReportBaseSchema, ComplianceReportListSchema, ComplianceReportSummarySchema, - ComplianceReportUpdateSchema, ComplianceReportSummaryUpdateSchema, + ChainedComplianceReportSchema, + ComplianceReportUpdateSchema, + ComplianceReportSummaryUpdateSchema, ) from lcfs.web.api.compliance_report.services import ComplianceReportServices from lcfs.web.api.compliance_report.summary_service import ( @@ -66,12 +68,12 @@ async def get_compliance_reports( pagination.filters.append( FilterModel(field="status", filter="Draft", filter_type="text", type="notEqual") ) - return await service.get_compliance_reports_paginated(pagination) + return await service.get_compliance_reports_paginated(pagination) @router.get( "/{report_id}", - response_model=ComplianceReportBaseSchema, + response_model=ChainedComplianceReportSchema, status_code=status.HTTP_200_OK, ) @view_handler([RoleEnum.GOVERNMENT]) @@ -80,12 +82,16 @@ async def get_compliance_report_by_id( report_id: int, service: ComplianceReportServices = Depends(), validate: ComplianceReportValidation = Depends(), -) -> ComplianceReportBaseSchema: +) -> ChainedComplianceReportSchema: await validate.validate_organization_access(report_id) mask_statuses = not user_has_roles(request.user, [RoleEnum.GOVERNMENT]) - return await service.get_compliance_report_by_id(report_id, mask_statuses) + result = await service.get_compliance_report_by_id( + report_id, mask_statuses, get_chain=True + ) + + return result @router.get( @@ -128,6 +134,7 @@ async def update_compliance_report_summary( report_id, summary_data ) + @view_handler(["*"]) @router.put( "/{report_id}", diff --git a/backend/lcfs/web/api/final_supply_equipment/schema.py b/backend/lcfs/web/api/final_supply_equipment/schema.py index 0751354bd..38f80b2fa 100644 --- a/backend/lcfs/web/api/final_supply_equipment/schema.py +++ b/backend/lcfs/web/api/final_supply_equipment/schema.py @@ -40,7 +40,7 @@ class FinalSupplyEquipmentCreateSchema(BaseSchema): compliance_report_id: Optional[int] = None supply_from_date: date supply_to_date: date - kwh_usage: Optional[float] = None + kwh_usage: float serial_nbr: str manufacturer: str model: Optional[str] = None diff --git a/backend/lcfs/web/api/fuel_export/schema.py b/backend/lcfs/web/api/fuel_export/schema.py index bb95ef2a5..80f66af7b 100644 --- a/backend/lcfs/web/api/fuel_export/schema.py +++ b/backend/lcfs/web/api/fuel_export/schema.py @@ -136,6 +136,12 @@ class FuelExportSchema(BaseSchema): compliance_period: Optional[str] = None fuel_type_id: int fuel_type: FuelTypeSchema + fuel_category_id: int + fuel_category: FuelCategoryResponseSchema + end_use_id: Optional[int] = None + end_use_type: Optional[EndUseTypeSchema] = None + provision_of_the_act_id: Optional[int] = None + provision_of_the_act: Optional[ProvisionOfTheActSchema] = None fuel_type_other: Optional[str] = None quantity: int = Field(..., gt=0) units: str @@ -147,14 +153,9 @@ class FuelExportSchema(BaseSchema): energy_density: Optional[float] = None eer: Optional[float] = None energy: Optional[float] = None - fuel_category_id: int - fuel_category: FuelCategoryResponseSchema fuel_code_id: Optional[int] = None fuel_code: Optional[FuelCodeResponseSchema] = None - provision_of_the_act_id: Optional[int] = None - provision_of_the_act: Optional[ProvisionOfTheActSchema] = None - end_use_id: Optional[int] = None - end_use_type: Optional[EndUseTypeSchema] = None + @validator("quantity") def quantity_must_be_positive(cls, v): diff --git a/backend/lcfs/web/api/fuel_supply/schema.py b/backend/lcfs/web/api/fuel_supply/schema.py index 4f6873c70..c83288a54 100644 --- a/backend/lcfs/web/api/fuel_supply/schema.py +++ b/backend/lcfs/web/api/fuel_supply/schema.py @@ -161,6 +161,12 @@ class FuelSupplyResponseSchema(BaseSchema): action_type: str fuel_type_id: int fuel_type: FuelTypeSchema + fuel_category_id: Optional[int] = None + fuel_category: FuelCategoryResponseSchema + end_use_id: Optional[int] = None + end_use_type: Optional[EndUseTypeSchema] = None + provision_of_the_act_id: Optional[int] = None + provision_of_the_act: Optional[ProvisionOfTheActSchema] = None compliance_period: Optional[str] = None quantity: int units: str @@ -171,14 +177,8 @@ class FuelSupplyResponseSchema(BaseSchema): energy_density: Optional[float] = None eer: Optional[float] = None energy: Optional[float] = None - fuel_category_id: Optional[int] = None - fuel_category: FuelCategoryResponseSchema fuel_code_id: Optional[int] = None fuel_code: Optional[FuelCodeResponseSchema] = None - provision_of_the_act_id: Optional[int] = None - provision_of_the_act: Optional[ProvisionOfTheActSchema] = None - end_use_id: Optional[int] = None - end_use_type: Optional[EndUseTypeSchema] = None fuel_type_other: Optional[str] = None diff --git a/backend/lcfs/web/api/organization/views.py b/backend/lcfs/web/api/organization/views.py index eb6244051..e175bf756 100644 --- a/backend/lcfs/web/api/organization/views.py +++ b/backend/lcfs/web/api/organization/views.py @@ -33,6 +33,7 @@ ComplianceReportCreateSchema, ComplianceReportListSchema, CompliancePeriodSchema, + ChainedComplianceReportSchema ) from lcfs.web.api.compliance_report.services import ComplianceReportServices from .services import OrganizationService @@ -55,7 +56,8 @@ async def get_org_users( request: Request, organization_id: int, - status: str = Query(default="Active", description="Active or Inactive users list"), + status: str = Query( + default="Active", description="Active or Inactive users list"), pagination: PaginationRequestSchema = Body(..., embed=False), response: Response = None, org_service: OrganizationService = Depends(), @@ -264,7 +266,7 @@ async def get_compliance_reports( ) -> ComplianceReportListSchema: organization_id = request.user.organization.organization_id return await report_service.get_compliance_reports_paginated( - pagination, organization_id, bceid_user = True + pagination, organization_id, bceid_user=True ) @@ -288,7 +290,7 @@ async def get_all_org_reported_years( @router.get( "/{organization_id}/reports/{report_id}", - response_model=ComplianceReportBaseSchema, + response_model=ChainedComplianceReportSchema, status_code=status.HTTP_200_OK, ) @view_handler([RoleEnum.SUPPLIER]) @@ -299,10 +301,10 @@ async def get_compliance_report_by_id( report_id: int = None, report_service: ComplianceReportServices = Depends(), report_validate: ComplianceReportValidation = Depends(), -) -> ComplianceReportBaseSchema: +) -> ChainedComplianceReportSchema: """ Endpoint to get information of a user by ID This endpoint returns the information of a user by ID, including their roles and organization. """ await report_validate.validate_organization_access(report_id) - return await report_service.get_compliance_report_by_id(report_id, apply_masking=True) + return await report_service.get_compliance_report_by_id(report_id, apply_masking=True, get_chain=True) diff --git a/backend/lcfs/web/api/other_uses/schema.py b/backend/lcfs/web/api/other_uses/schema.py index b13f359e8..7b21dfa1b 100644 --- a/backend/lcfs/web/api/other_uses/schema.py +++ b/backend/lcfs/web/api/other_uses/schema.py @@ -85,13 +85,13 @@ class OtherUsesTableOptionsSchema(BaseSchema): class OtherUsesCreateSchema(BaseSchema): other_uses_id: Optional[int] = None compliance_report_id: int - quantity_supplied: int fuel_type: str fuel_category: str - expected_use: str provision_of_the_act: str - fuel_code: Optional[str] = None + quantity_supplied: int units: str + expected_use: str + fuel_code: Optional[str] = None ci_of_fuel: Optional[float] = None expected_use: str other_uses_id: Optional[int] = None diff --git a/backend/lcfs/web/api/redis/__init__.py b/backend/lcfs/web/api/redis/__init__.py deleted file mode 100644 index 24f6cdcc0..000000000 --- a/backend/lcfs/web/api/redis/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Redis API.""" - -from lcfs.web.api.redis.views import router - -__all__ = ["router"] diff --git a/backend/lcfs/web/api/redis/schema.py b/backend/lcfs/web/api/redis/schema.py deleted file mode 100644 index d4ac74f4a..000000000 --- a/backend/lcfs/web/api/redis/schema.py +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Optional - -from lcfs.web.api.base import BaseSchema - - -class RedisValueDTO(BaseSchema): - """DTO for redis values.""" - - key: str - value: Optional[str] = None # noqa: WPS110 diff --git a/backend/lcfs/web/api/redis/views.py b/backend/lcfs/web/api/redis/views.py deleted file mode 100644 index 5760d75dc..000000000 --- a/backend/lcfs/web/api/redis/views.py +++ /dev/null @@ -1,44 +0,0 @@ -from fastapi import APIRouter -from fastapi.param_functions import Depends -from redis.asyncio import ConnectionPool, Redis - -from lcfs.services.redis.dependency import get_redis_pool -from lcfs.web.api.redis.schema import RedisValueDTO - -router = APIRouter() - - -@router.get("/", response_model=RedisValueDTO) -async def get_redis_value( - key: str, - redis_pool: ConnectionPool = Depends(get_redis_pool), -) -> RedisValueDTO: - """ - Get value from redis. - - :param key: redis key, to get data from. - :param redis_pool: redis connection pool. - :returns: information from redis. - """ - async with Redis(connection_pool=redis_pool) as redis: - redis_value = await redis.get(key) - return RedisValueDTO( - key=key, - value=redis_value, - ) - - -@router.put("/") -async def set_redis_value( - redis_value: RedisValueDTO, - redis_pool: ConnectionPool = Depends(get_redis_pool), -) -> None: - """ - Set value in redis. - - :param redis_value: new value data. - :param redis_pool: redis connection pool. - """ - if redis_value.value is not None: - async with Redis(connection_pool=redis_pool) as redis: - await redis.set(name=redis_value.key, value=redis_value.value) diff --git a/backend/lcfs/web/api/router.py b/backend/lcfs/web/api/router.py index 8ac4ba401..0ae1659cc 100644 --- a/backend/lcfs/web/api/router.py +++ b/backend/lcfs/web/api/router.py @@ -4,7 +4,6 @@ echo, fuel_supply, monitoring, - redis, user, role, notification, @@ -41,7 +40,6 @@ ) api_router.include_router(transfer.router, prefix="/transfers", tags=["transfers"]) api_router.include_router(echo.router, prefix="/echo", tags=["echo"]) -api_router.include_router(redis.router, prefix="/redis", tags=["redis"]) api_router.include_router(user.router, prefix="/users", tags=["users"]) api_router.include_router(role.router, prefix="/roles", tags=["roles"]) api_router.include_router(dashboard.router, prefix="/dashboard", tags=["dashboard"]) diff --git a/backend/lcfs/web/application.py b/backend/lcfs/web/application.py index 4a855937c..6d31484d0 100644 --- a/backend/lcfs/web/application.py +++ b/backend/lcfs/web/application.py @@ -68,13 +68,15 @@ async def authenticate(self, request): return AuthCredentials([]), UnauthenticatedUser() # Lazily retrieve Redis, session, and settings from app state - redis_pool = self.app.state.redis_pool + redis_client = self.app.state.redis_client session_factory = self.app.state.db_session_factory settings = self.app.state.settings # Now that we have the dependencies, we can instantiate the real backend real_backend = UserAuthentication( - redis_pool=redis_pool, session_factory=session_factory, settings=settings + redis_client=redis_client, + session_factory=session_factory, + settings=settings, ) # Call the authenticate method of the real backend diff --git a/backend/lcfs/web/lifetime.py b/backend/lcfs/web/lifetime.py index 186b485cd..5de67c16c 100644 --- a/backend/lcfs/web/lifetime.py +++ b/backend/lcfs/web/lifetime.py @@ -4,7 +4,7 @@ import boto3 from fastapi_cache import FastAPICache from fastapi_cache.backends.redis import RedisBackend -from redis import asyncio as aioredis +from redis.asyncio import Redis from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from lcfs.services.rabbitmq.consumers import start_consumers, stop_consumers @@ -32,33 +32,6 @@ def _setup_db(app: FastAPI) -> None: # pragma: no cover app.state.db_session_factory = session_factory -async def startup_s3(app: FastAPI) -> None: - """ - Initialize the S3 client and store it in the app state. - - :param app: fastAPI application. - """ - app.state.s3_client = boto3.client( - "s3", - aws_access_key_id=settings.s3_access_key, - aws_secret_access_key=settings.s3_secret_key, - endpoint_url=settings.s3_endpoint, - region_name="us-east-1", - ) - print("S3 client initialized.") - - -async def shutdown_s3(app: FastAPI) -> None: - """ - Cleanup the S3 client from the app state. - - :param app: fastAPI application. - """ - if hasattr(app.state, "s3_client"): - del app.state.s3_client - print("S3 client shutdown.") - - def register_startup_event( app: FastAPI, ) -> Callable[[], Awaitable[None]]: # pragma: no cover @@ -83,14 +56,11 @@ async def _startup() -> None: # noqa: WPS430 # Assign settings to app state for global access app.state.settings = settings - # Initialize the cache with Redis backend using app.state.redis_pool - FastAPICache.init(RedisBackend(app.state.redis_pool), prefix="lcfs") + # Initialize FastAPI cache with the Redis client + FastAPICache.init(RedisBackend(app.state.redis_client), prefix="lcfs") await init_org_balance_cache(app) - # Initialize the S3 client - await startup_s3(app) - # Setup RabbitMQ Listeners await start_consumers() @@ -112,7 +82,6 @@ async def _shutdown() -> None: # noqa: WPS430 await app.state.db_engine.dispose() await shutdown_redis(app) - await shutdown_s3(app) await stop_consumers() return _shutdown diff --git a/backend/poetry.lock b/backend/poetry.lock index d55b99abb..dba1d2518 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -274,17 +274,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.35.64" +version = "1.35.36" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.64-py3-none-any.whl", hash = "sha256:cdacf03fc750caa3aa0dbf6158166def9922c9d67b4160999ff8fc350662facc"}, - {file = "boto3-1.35.64.tar.gz", hash = "sha256:bc3fc12b41fa2c91e51ab140f74fb1544408a2b1e00f88a4c2369a66d18ddf20"}, + {file = "boto3-1.35.36-py3-none-any.whl", hash = "sha256:33735b9449cd2ef176531ba2cb2265c904a91244440b0e161a17da9d24a1e6d1"}, + {file = "boto3-1.35.36.tar.gz", hash = "sha256:586524b623e4fbbebe28b604c6205eb12f263cc4746bccb011562d07e217a4cb"}, ] [package.dependencies] -botocore = ">=1.35.64,<1.36.0" +botocore = ">=1.35.36,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -293,13 +293,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.64" +version = "1.35.36" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.64-py3-none-any.whl", hash = "sha256:bbd96bf7f442b1d5e35b36f501076e4a588c83d8d84a1952e9ee1d767e5efb3e"}, - {file = "botocore-1.35.64.tar.gz", hash = "sha256:2f95c83f31c9e38a66995c88810fc638c829790e125032ba00ab081a2cf48cb9"}, + {file = "botocore-1.35.36-py3-none-any.whl", hash = "sha256:64241c778bf2dc863d93abab159e14024d97a926a5715056ef6411418cb9ead3"}, + {file = "botocore-1.35.36.tar.gz", hash = "sha256:354ec1b766f0029b5d6ff0c45d1a0f9e5007b7d2f3ec89bcdd755b208c5bc797"}, ] [package.dependencies] diff --git a/frontend/public/config/config.js b/frontend/public/config/config.js index 01cbed624..a53256ce3 100644 --- a/frontend/public/config/config.js +++ b/frontend/public/config/config.js @@ -7,6 +7,10 @@ export const config = { POST_LOGOUT_URL: 'http://localhost:3000/', SM_LOGOUT_URL: 'https://logontest7.gov.bc.ca/clp-cgi/logoff.cgi?retnow=1&returl=' + }, + feature_flags: { + supplementalReporting: true, + notifications: false } } diff --git a/frontend/src/assets/locales/en/allocationAgreement.json b/frontend/src/assets/locales/en/allocationAgreement.json index cb01bb29a..1da1af078 100644 --- a/frontend/src/assets/locales/en/allocationAgreement.json +++ b/frontend/src/assets/locales/en/allocationAgreement.json @@ -4,7 +4,7 @@ "addAllocationAgreementRowsTitle": "Allocation agreements (e.g., allocating responsibility for fuel)", "allocationAgreementSubtitle": "Enter allocation agreement details below", "allocationAgreementColLabels": { - "transaction": "Transaction", + "transaction": "Responsibility", "transactionPartner": "Legal name of transaction partner", "postalAddress": "Address for service", "transactionPartnerEmail": "Email", diff --git a/frontend/src/assets/locales/en/fuelExport.json b/frontend/src/assets/locales/en/fuelExport.json index 96899ee10..002fba7c1 100644 --- a/frontend/src/assets/locales/en/fuelExport.json +++ b/frontend/src/assets/locales/en/fuelExport.json @@ -12,12 +12,12 @@ "fuelExportColLabels": { "complianceReportId": "Compliance Report ID", "fuelExportId": "Fuel export ID", - "fuelType": "Fuel type", + "fuelTypeId": "Fuel type", "exportDate": "Export date", "fuelTypeOther": "Fuel type other", - "fuelCategory": "Fuel catgory", + "fuelCategoryId": "Fuel catgory", "endUse": "End use", - "provisionOfTheAct": "Determining carbon intensity", + "provisionOfTheActId": "Determining carbon intensity", "fuelCode": "Fuel code", "quantity": "Quantity supplied", "units": "Units", diff --git a/frontend/src/assets/locales/en/fuelSupply.json b/frontend/src/assets/locales/en/fuelSupply.json index 6830b71ef..3e6036080 100644 --- a/frontend/src/assets/locales/en/fuelSupply.json +++ b/frontend/src/assets/locales/en/fuelSupply.json @@ -16,7 +16,7 @@ "fuelTypeOther": "Fuel type other", "fuelCategory": "Fuel category", "endUse": "End use", - "provisionOfTheAct": "Determining carbon intensity", + "provisionOfTheActId": "Determining carbon intensity", "fuelCode": "Fuel code", "quantity": "Quantity supplied", "units": "Units", diff --git a/frontend/src/assets/locales/en/transfer.json b/frontend/src/assets/locales/en/transfer.json index f0e5ffc25..14ba125bd 100644 --- a/frontend/src/assets/locales/en/transfer.json +++ b/frontend/src/assets/locales/en/transfer.json @@ -51,7 +51,7 @@ "loadingText": "Loading transfer...", "processingText": "Processing transfer...", "detailsLabel": "Transfer Details (required)", - "fairMarketText": "The fair market value of any consideration, in CAD", + "fairMarketText": "The fair market value of any consideration, in $CAD", "totalValueText": " per compliance unit for a total value of ", "saLabel": "Signing Authority Declaration", "saConfirmation": "I confirm that records evidencing each matter reported under section 18 of the Low Carbon Fuel (General) Regulation are available on request.", diff --git a/frontend/src/constants/config.js b/frontend/src/constants/config.js index 2763bbcf2..8dfec5e6c 100644 --- a/frontend/src/constants/config.js +++ b/frontend/src/constants/config.js @@ -28,6 +28,15 @@ export function getApiBaseUrl() { return window.lcfs_config.api_base ?? baseUrl } +export const isFeatureEnabled = (featureFlag) => { + return CONFIG.feature_flags[featureFlag] +} + +export const FEATURE_FLAGS = { + SUPPLEMENTAL_REPORTING: 'supplementalReporting', + NOTIFICATIONS: 'notifications' +} + export const CONFIG = { API_BASE: getApiBaseUrl(), KEYCLOAK: { @@ -42,5 +51,10 @@ export const CONFIG = { SM_LOGOUT_URL: window.lcfs_config.keycloak.SM_LOGOUT_URL ?? 'https://logontest7.gov.bc.ca/clp-cgi/logoff.cgi?retnow=1&returl=' + }, + feature_flags: { + supplementalReporting: + window.lcfs_config.feature_flags.supplementalReporting ?? true, + notifications: window.lcfs_config.feature_flags.notifications ?? false } } diff --git a/frontend/src/hooks/useComplianceReports.js b/frontend/src/hooks/useComplianceReports.js index 0a79e4023..c28f04fd4 100644 --- a/frontend/src/hooks/useComplianceReports.js +++ b/frontend/src/hooks/useComplianceReports.js @@ -55,7 +55,7 @@ export const useGetComplianceReport = (orgID, reportID, options) => { return useQuery({ queryKey: ['compliance-report', reportID], queryFn: async () => { - return (await client.get(path)) + return (await client.get(path)).data }, ...options }) @@ -132,22 +132,22 @@ export const useComplianceReportDocuments = (parentID, options) => { } export const useCreateSupplementalReport = (reportID, options) => { - const client = useApiService(); - const queryClient = useQueryClient(); - const path = apiRoutes.createSupplementalReport.replace(':reportID', reportID); + const client = useApiService() + const queryClient = useQueryClient() + const path = apiRoutes.createSupplementalReport.replace(':reportID', reportID) return useMutation({ mutationFn: () => client.post(path), onSuccess: (data) => { - queryClient.invalidateQueries(['compliance-reports']); + queryClient.invalidateQueries(['compliance-reports']) if (options && options.onSuccess) { - options.onSuccess(data); + options.onSuccess(data) } }, onError: (error) => { if (options && options.onError) { - options.onError(error); + options.onError(error) } - }, - }); -}; \ No newline at end of file + } + }) +} diff --git a/frontend/src/utils/__tests__/withFeatureFlag.test.jsx b/frontend/src/utils/__tests__/withFeatureFlag.test.jsx new file mode 100644 index 000000000..b619a1f25 --- /dev/null +++ b/frontend/src/utils/__tests__/withFeatureFlag.test.jsx @@ -0,0 +1,109 @@ +import React from 'react' +import { render, screen } from '@testing-library/react' +import { describe, it, expect, vi, beforeEach } from 'vitest' +import withFeatureFlag from '../withFeatureFlag.jsx' // Adjust the import path as necessary +import { isFeatureEnabled } from '@/constants/config.js' + +// Mock the isFeatureEnabled function +vi.mock('@/constants/config.js', () => ({ + isFeatureEnabled: vi.fn() +})) + +// Mock Navigate component +vi.mock('react-router-dom', () => ({ + ...vi.importActual('react-router-dom'), + Navigate: ({ to }) =>