From 4f51e107b05eb33d53f53eeab0c15c7e2310f2cb Mon Sep 17 00:00:00 2001 From: prv-proton Date: Tue, 10 Dec 2024 03:49:30 -0800 Subject: [PATCH 1/4] Fix: LCFS - BUG AG Grid copy and paste function for Fuel codes table on IDIR side not functioning #1302 --- backend/lcfs/web/api/fuel_code/repo.py | 22 +++---- backend/lcfs/web/api/fuel_code/services.py | 4 ++ backend/lcfs/web/core/decorators.py | 2 +- .../Renderers/ValidationRenderer2.jsx | 2 +- frontend/src/constants/routes/apiRoutes.js | 1 + frontend/src/hooks/useFuelCode.js | 2 +- .../FuelCodes/AddFuelCode/AddEditFuelCode.jsx | 58 ++++++++++++++++++- .../views/FuelCodes/AddFuelCode/_schema.jsx | 9 +-- 8 files changed, 76 insertions(+), 24 deletions(-) diff --git a/backend/lcfs/web/api/fuel_code/repo.py b/backend/lcfs/web/api/fuel_code/repo.py index aa0577466..7e0d0f534 100644 --- a/backend/lcfs/web/api/fuel_code/repo.py +++ b/backend/lcfs/web/api/fuel_code/repo.py @@ -396,19 +396,8 @@ async def create_fuel_code(self, fuel_code: FuelCode) -> FuelCode: """ self.db.add(fuel_code) await self.db.flush() - await self.db.refresh( - fuel_code, - [ - "fuel_code_status", - "fuel_code_prefix", - "fuel_type", - "feedstock_fuel_transport_modes", - "finished_fuel_transport_modes", - ], - ) - # Manually load nested relationships - await self.db.refresh(fuel_code.fuel_type, ["provision_1", "provision_2"]) - return fuel_code + result = await self.get_fuel_code(fuel_code.fuel_code_id) + return result @repo_handler async def get_fuel_code(self, fuel_code_id: int) -> FuelCode: @@ -593,9 +582,14 @@ async def validate_fuel_code(self, suffix: str, prefix_id: int) -> str: result = (await self.db.execute(query)).scalar_one_or_none() if result: fuel_code_main_version = suffix.split(".")[0] - return await self.get_next_available_sub_version_fuel_code_by_prefix( + suffix = await self.get_next_available_sub_version_fuel_code_by_prefix( fuel_code_main_version, prefix_id ) + if int(suffix.split(".")[1]) > 9: + return await self.get_next_available_fuel_code_by_prefix( + result.fuel_code_prefix.prefix + ) + return suffix else: return suffix diff --git a/backend/lcfs/web/api/fuel_code/services.py b/backend/lcfs/web/api/fuel_code/services.py index 039634e6a..d40fde544 100644 --- a/backend/lcfs/web/api/fuel_code/services.py +++ b/backend/lcfs/web/api/fuel_code/services.py @@ -208,6 +208,8 @@ async def convert_to_model( transport_mode_id=matching_transport_mode.transport_mode_id, ) ) + else: + raise ValueError(f"Invalid transport mode: {transport_mode}") for transport_mode in fuel_code_schema.finished_fuel_transport_mode or []: matching_transport_mode = next( @@ -221,6 +223,8 @@ async def convert_to_model( transport_mode_id=matching_transport_mode.transport_mode_id, ) ) + else: + raise ValueError(f"Invalid transport mode: {transport_mode}") return fuel_code diff --git a/backend/lcfs/web/core/decorators.py b/backend/lcfs/web/core/decorators.py index 07dc7c5ab..e67d9afca 100644 --- a/backend/lcfs/web/core/decorators.py +++ b/backend/lcfs/web/core/decorators.py @@ -215,7 +215,7 @@ async def wrapper(*args, **kwargs): return await func(*args, **kwargs) # raise the error to the view layer - except (DatabaseException, HTTPException, DataNotFoundException): + except (DatabaseException, HTTPException, DataNotFoundException, ValueError): raise # all other errors that occur in the service layer will log an error except Exception as e: diff --git a/frontend/src/components/BCDataGrid/components/Renderers/ValidationRenderer2.jsx b/frontend/src/components/BCDataGrid/components/Renderers/ValidationRenderer2.jsx index 043b127b5..5d4931865 100644 --- a/frontend/src/components/BCDataGrid/components/Renderers/ValidationRenderer2.jsx +++ b/frontend/src/components/BCDataGrid/components/Renderers/ValidationRenderer2.jsx @@ -18,7 +18,7 @@ export const ValidationRenderer2 = ({ data }) => { ) case 'error': return ( - + { ...options, mutationFn: async (fuelCodeID) => { return await client.delete( - apiRoutes.updateFuelCode.replace(':fuelCodeId', fuelCodeID) + apiRoutes.deleteFuelCode.replace(':fuelCodeId', fuelCodeID) ) } }) diff --git a/frontend/src/views/FuelCodes/AddFuelCode/AddEditFuelCode.jsx b/frontend/src/views/FuelCodes/AddFuelCode/AddEditFuelCode.jsx index 044c767a7..f4b30e304 100644 --- a/frontend/src/views/FuelCodes/AddFuelCode/AddEditFuelCode.jsx +++ b/frontend/src/views/FuelCodes/AddFuelCode/AddEditFuelCode.jsx @@ -25,6 +25,7 @@ import BCModal from '@/components/BCModal' import BCTypography from '@/components/BCTypography' import { FUEL_CODE_STATUSES } from '@/constants/statuses' import { useCurrentUser } from '@/hooks/useCurrentUser' +import Papa from 'papaparse' const AddEditFuelCodeBase = () => { const { fuelCodeID } = useParams() @@ -197,6 +198,7 @@ const AddEditFuelCodeBase = () => { } else { const res = await createFuelCode(updatedData) updatedData.fuelCodeId = res.data.fuelCodeId + updatedData.fuelSuffix = res.data.fuelSuffix } updatedData = { @@ -210,7 +212,7 @@ const AddEditFuelCodeBase = () => { }) } catch (error) { setErrors({ - [params.node.data.id]: error.response.data.errors[0].fields + [params.node.data.id]: error.response.data?.errors && error.response.data?.errors[0]?.fields }) updatedData = { @@ -229,10 +231,12 @@ const AddEditFuelCodeBase = () => { const errMsg = `Error updating row: ${ fieldLabels.length === 1 ? fieldLabels[0] : '' } ${message}` - + updatedData.validationMsg = errMsg handleError(error, errMsg) } else { - handleError(error, `Error updating row: ${error.message}`) + const errMsg = error.response?.data?.detail || error.message + updatedData.validationMsg = errMsg + handleError(error, `Error updating row: ${errMsg}`) } } @@ -241,6 +245,53 @@ const AddEditFuelCodeBase = () => { [updateFuelCode, t] ) + const handlePaste = useCallback( + (event, { api, columnApi }) => { + const newData = [] + const clipboardData = event.clipboardData || window.clipboardData + const pastedData = clipboardData.getData('text/plain') + const headerRow = api + .getAllDisplayedColumns() + .map((column) => column.colDef.field) + .filter((col) => col) + .join('\t') + const parsedData = Papa.parse(headerRow + '\n' + pastedData, { + delimiter: '\t', + header: true, + transform: (value) => { + const num = Number(value) // Attempt to convert to a number if possible + return isNaN(num) ? value : num // Return the number if valid, otherwise keep as string + }, + skipEmptyLines: true + }) + if (parsedData.data?.length < 0) { + return + } + parsedData.data.forEach((row) => { + const newRow = { ...row } + newRow.id = uuid() + newRow.prefixId = optionsData?.fuelCodePrefixes?.find(o => o.prefix === row.prefix)?.fuelCodePrefixId + newRow.fuelTypeId = optionsData?.fuelTypes?.find(o => o.fuelType === row.fuelType)?.fuelTypeId + newRow.fuelSuffix = newRow.fuelSuffix.toString() + newRow.feedstockFuelTransportMode = row.feedstockFuelTransportMode.split(',').map(item => item.trim()) + newRow.finishedFuelTransportMode = row.finishedFuelTransportMode.split(',').map(item => item.trim()) + newRow.modified = true + newData.push(newRow) + }) + const transactions = api.applyTransaction({ add: newData }) + // Trigger onCellEditingStopped event to update the row in backend. + transactions.add.forEach((node) => { + onCellEditingStopped({ + node, + oldValue: '', + newvalue: undefined, + ...api + }) + }) + }, + [onCellEditingStopped, optionsData] + ) + const duplicateFuelCode = async (params) => { const rowData = { ...params.data, @@ -327,6 +378,7 @@ const AddEditFuelCodeBase = () => { onAction={onAction} showAddRowsButton={!existingFuelCode && hasRoles(roles.analyst)} context={{ errors }} + handlePaste={handlePaste} /> {existingFuelCode?.fuelCodeStatus.status !== FUEL_CODE_STATUSES.APPROVED && ( diff --git a/frontend/src/views/FuelCodes/AddFuelCode/_schema.jsx b/frontend/src/views/FuelCodes/AddFuelCode/_schema.jsx index 3c39a6f4c..96186c584 100644 --- a/frontend/src/views/FuelCodes/AddFuelCode/_schema.jsx +++ b/frontend/src/views/FuelCodes/AddFuelCode/_schema.jsx @@ -97,9 +97,9 @@ export const fuelCodeColDefs = (optionsData, errors, isCreate, canEdit) => [ const selectedPrefix = optionsData?.fuelCodePrefixes?.find( (obj) => obj.prefix === params.newValue ) - params.data.fuelTypeId = selectedPrefix.fuelCodePrefixId + params.data.fuelCodePrefixId = selectedPrefix.fuelCodePrefixId - params.data.fuelCode = optionsData?.fuelCodePrefixes?.find( + params.data.fuelSuffix = optionsData?.fuelCodePrefixes?.find( (obj) => obj.prefix === params.newValue )?.nextFuelCode params.data.company = undefined @@ -327,12 +327,12 @@ export const fuelCodeColDefs = (optionsData, errors, isCreate, canEdit) => [ return selectedOption.fuelType } const selectedOption = optionsData?.fuelTypes?.find( - (obj) => obj.fuelType === params.data.fuel + (obj) => obj.fuelType === params.data.fuelType ) if (selectedOption) { params.data.fuelTypeId = selectedOption.fuelTypeId } - return params.data.fuel + return params.data.fuelType }, valueSetter: (params) => { if (params.newValue) { @@ -341,6 +341,7 @@ export const fuelCodeColDefs = (optionsData, errors, isCreate, canEdit) => [ ) params.data.fuelTypeId = selectedFuelType.fuelTypeId } + return params.data.fuelType }, cellEditorParams: { options: optionsData?.fuelTypes From 28ecdb9feb8a1edb816067b68c1889e1c61fef45 Mon Sep 17 00:00:00 2001 From: prv-proton Date: Tue, 10 Dec 2024 04:16:00 -0800 Subject: [PATCH 2/4] test fixes. --- backend/lcfs/tests/fuel_code/test_fuel_code_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/lcfs/tests/fuel_code/test_fuel_code_service.py b/backend/lcfs/tests/fuel_code/test_fuel_code_service.py index c71608031..5dce49e52 100644 --- a/backend/lcfs/tests/fuel_code/test_fuel_code_service.py +++ b/backend/lcfs/tests/fuel_code/test_fuel_code_service.py @@ -210,7 +210,7 @@ async def test_approve_fuel_code_not_found(): repo_mock.get_fuel_code.return_value = None # Act & Assert - with pytest.raises(ServiceException): + with pytest.raises(ValueError, match="Fuel code not found"): await service.approve_fuel_code(fuel_code_id) repo_mock.get_fuel_code.assert_called_once_with(fuel_code_id) @@ -229,7 +229,7 @@ async def test_approve_fuel_code_invalid_status(): repo_mock.get_fuel_code.return_value = mock_fuel_code # Act & Assert - with pytest.raises(ServiceException): + with pytest.raises(ValueError, match="Fuel code is not in Draft"): await service.approve_fuel_code(fuel_code_id) repo_mock.get_fuel_code.assert_called_once_with(fuel_code_id) From d5ebc64d024b7acf97f1b1922e862a87bb05530b Mon Sep 17 00:00:00 2001 From: prv-proton Date: Tue, 10 Dec 2024 04:17:38 -0800 Subject: [PATCH 3/4] . --- backend/lcfs/tests/other_uses/test_other_uses_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lcfs/tests/other_uses/test_other_uses_services.py b/backend/lcfs/tests/other_uses/test_other_uses_services.py index 3f4705501..14b3bb518 100644 --- a/backend/lcfs/tests/other_uses/test_other_uses_services.py +++ b/backend/lcfs/tests/other_uses/test_other_uses_services.py @@ -189,7 +189,7 @@ async def test_update_other_use_not_found(other_uses_service): mock_repo.get_other_use_version_by_user = AsyncMock(return_value=None) - with pytest.raises(ServiceException): + with pytest.raises(ValueError, match="Other use not found"): await service.update_other_use(other_use_data, UserTypeEnum.SUPPLIER) From 5544e39efb5a65ab02fae0b48021eaaaaeae351e Mon Sep 17 00:00:00 2001 From: prv-proton Date: Tue, 10 Dec 2024 12:21:37 -0800 Subject: [PATCH 4/4] adding support for different date types. --- .../FuelCodes/AddFuelCode/AddEditFuelCode.jsx | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/frontend/src/views/FuelCodes/AddFuelCode/AddEditFuelCode.jsx b/frontend/src/views/FuelCodes/AddFuelCode/AddEditFuelCode.jsx index f4b30e304..6841ea7ac 100644 --- a/frontend/src/views/FuelCodes/AddFuelCode/AddEditFuelCode.jsx +++ b/frontend/src/views/FuelCodes/AddFuelCode/AddEditFuelCode.jsx @@ -212,7 +212,9 @@ const AddEditFuelCodeBase = () => { }) } catch (error) { setErrors({ - [params.node.data.id]: error.response.data?.errors && error.response.data?.errors[0]?.fields + [params.node.data.id]: + error.response.data?.errors && + error.response.data?.errors[0]?.fields }) updatedData = { @@ -258,7 +260,15 @@ const AddEditFuelCodeBase = () => { const parsedData = Papa.parse(headerRow + '\n' + pastedData, { delimiter: '\t', header: true, - transform: (value) => { + transform: (value, field) => { + // Check for date fields and format them + const dateRegex = /^\d{4}-\d{2}-\d{2}$/ // Matches YYYY-MM-DD format + if (field.toLowerCase().includes('date') && !dateRegex.test(value)) { + const parsedDate = new Date(value) + if (!isNaN(parsedDate)) { + return parsedDate.toISOString().split('T')[0] // Format as YYYY-MM-DD + } + } const num = Number(value) // Attempt to convert to a number if possible return isNaN(num) ? value : num // Return the number if valid, otherwise keep as string }, @@ -270,11 +280,19 @@ const AddEditFuelCodeBase = () => { parsedData.data.forEach((row) => { const newRow = { ...row } newRow.id = uuid() - newRow.prefixId = optionsData?.fuelCodePrefixes?.find(o => o.prefix === row.prefix)?.fuelCodePrefixId - newRow.fuelTypeId = optionsData?.fuelTypes?.find(o => o.fuelType === row.fuelType)?.fuelTypeId + newRow.prefixId = optionsData?.fuelCodePrefixes?.find( + (o) => o.prefix === row.prefix + )?.fuelCodePrefixId + newRow.fuelTypeId = optionsData?.fuelTypes?.find( + (o) => o.fuelType === row.fuelType + )?.fuelTypeId newRow.fuelSuffix = newRow.fuelSuffix.toString() - newRow.feedstockFuelTransportMode = row.feedstockFuelTransportMode.split(',').map(item => item.trim()) - newRow.finishedFuelTransportMode = row.finishedFuelTransportMode.split(',').map(item => item.trim()) + newRow.feedstockFuelTransportMode = row.feedstockFuelTransportMode + .split(',') + .map((item) => item.trim()) + newRow.finishedFuelTransportMode = row.finishedFuelTransportMode + .split(',') + .map((item) => item.trim()) newRow.modified = true newData.push(newRow) })