pass through gap data (target/met) values with rounding to 4 decimal digits [MRXN23-601]

api/apps/api/src/modules/scenarios-features/scenario-features-gap-data.service.ts

@@ -17,6 +17,7 @@ import { ScenarioAccessControl } from '@marxan-api/modules/access-control/scenar
 import { assertDefined } from '@marxan/utils';
 import { forbiddenError } from '@marxan-api/modules/access-control';
 import { Either, left, right } from 'fp-ts/lib/Either';
+import { plainToClass } from 'class-transformer';
 
 @Injectable()
 export class ScenarioFeaturesGapDataService extends AppBaseService<
@@ -59,7 +60,19 @@ export class ScenarioFeaturesGapDataService extends AppBaseService<
     ) {
       return left(forbiddenError);
     }
-    return right(await super.findAllPaginated(fetchSpecification, info));
+    /**
+     * @debt Explicitly applying transforms (via `plainToClass()`) here: it
+     * would be best to do this at AppBaseService level, but that would
+     * currently open a rabbit hole due to the use of generics.
+     */
+    const { data, metadata } = await this.findAllPaginated(
+      fetchSpecification,
+      info,
+    );
+    return right({
+      data: plainToClass(ScenarioFeaturesGapData, data),
+      metadata,
+    });
   }
 
   async setFilters(

api/apps/api/src/modules/scenarios-features/scenario-features-output-gap-data.service.ts

@@ -18,6 +18,7 @@ import { forbiddenError } from '@marxan-api/modules/access-control';
 import { assertDefined } from '@marxan/utils';
 import { Either, left, right } from 'fp-ts/lib/Either';
 import { ScenarioAccessControl } from '@marxan-api/modules/access-control/scenarios-acl/scenario-access-control';
+import { plainToClass } from 'class-transformer';
 
 const scenarioFeaturesOutputGapDataFilterKeyNames = ['runId'] as const;
 type ScenarioFeaturesOutputGapDataFilterKeys = keyof Pick<
@@ -73,7 +74,19 @@ export class ScenarioFeaturesOutputGapDataService extends AppBaseService<
     ) {
       return left(forbiddenError);
     }
-    return right(await super.findAllPaginated(fetchSpecification, info));
+    /**
+     * @debt Explicitly applying transforms (via `plainToClass()`) here: it
+     * would be best to do this at AppBaseService level, but that would
+     * currently open a rabbit hole due to the use of generics.
+     */
+    const { data, metadata } = await this.findAllPaginated(
+      fetchSpecification,
+      info,
+    );
+    return right({
+      data: plainToClass(ScenarioFeaturesOutputGapData, data),
+      metadata,
+    });
   }
 
   async setFilters(

api/apps/api/src/modules/scenarios/scenarios.controller.ts

@@ -86,7 +86,6 @@ import {
   GeometryKind,
 } from '@marxan-api/decorators/file-interceptors.decorator';
 import { ProtectedAreaDto } from '@marxan-api/modules/scenarios/dto/protected-area.dto';
-import { UploadShapefileDto } from '@marxan-api/modules/scenarios/dto/upload.shapefile.dto';
 import { ProtectedAreasChangeDto } from '@marxan-api/modules/scenarios/dto/protected-area-change.dto';
 import { StartScenarioBlmCalibrationDto } from '@marxan-api/modules/scenarios/dto/start-scenario-blm-calibration.dto';
 import { BlmCalibrationRunResultDto } from './dto/scenario-blm-calibration-results.dto';
@@ -108,8 +107,6 @@ import { RequestScenarioCloneResponseDto } from './dto/scenario-clone.dto';
 import { ensureShapefileHasRequiredFiles } from '@marxan-api/utils/file-uploads.utils';
 import { WebshotPdfReportConfig } from '@marxan/webshot/webshot.dto';
 import { ClearLockStatusParams } from '@marxan-api/modules/scenarios/dto/clear-lock-status-param.dto';
-import { CostRangeDto } from '@marxan-api/modules/scenarios/dto/cost-range.dto';
-import { plainToClass } from 'class-transformer';
 import { ProjectsService } from '@marxan-api/modules/projects/projects.service';
 import { CostSurfaceService } from '@marxan-api/modules/cost-surface/cost-surface.service';
 

api/apps/geoprocessing/src/migrations/geoprocessing/1710497811000-IncludeDecimalDigitsWhenRoundingTargetAndCurrentDataInGapDataViews.ts

@@ -0,0 +1,147 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class IncludeDecimalDigitsWhenRoundingTargetAndCurrentDataInGapDataViews1710497811000
+  implements MigrationInterface
+{
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+drop view scenario_features_gap_data;
+
+create view scenario_features_gap_data as
+  with gap_data as (
+    select
+      sfd.api_feature_id as feature_id,
+      scenario_id,
+      sum(total_area) total_area,
+      case when sum(current_pa) is not null
+	    then sum(current_pa)
+	    else 0
+	    end as met_area,
+      min(prop) as coverage_target
+    from scenario_features_data sfd
+    group by sfd.api_feature_id, feature_class_id, scenario_id)
+  select
+    scenario_id,
+    feature_id,
+    sum(total_area) as total_area,
+    sum(met_area) as met_area,
+    case
+      when sum(total_area) > 0
+      then round(((sum(met_area)/sum(total_area))*100)::numeric, 4)
+      else 0
+    end as met,
+    sum(total_area) * min(coverage_target) as coverage_target_area,
+    round((min(coverage_target) * 100)::numeric, 4) as coverage_target,
+    sum(met_area) >= (sum(total_area) * min(coverage_target)) as on_target
+  from gap_data
+  group by feature_id, scenario_id;
+    `);
+
+    await queryRunner.query(`
+drop view scenario_features_output_gap_data;
+
+create view scenario_features_output_gap_data as
+  with gap_data as (
+    select
+      amount,
+      occurrences,
+      run_id,
+      sfd.api_feature_id as feature_id,
+      sfd.scenario_id,
+      osfd.total_area,
+      sfd.prop as coverage_target,
+      osfd.target as target_met
+    from output_scenarios_features_data osfd
+    inner join scenario_features_data sfd on osfd.scenario_features_id=sfd.id)
+  select
+    scenario_id,
+    feature_id,
+    sum(total_area) as total_area,
+    sum(amount) as met_area,
+    case
+      when sum(total_area) <> 0 and sum(total_area) is not null then
+        round(((sum(amount)/sum(total_area))*100)::numeric, 4)
+      else
+        0
+    end as met,
+    sum(occurrences) as met_occurrences,
+    sum(total_area) * min(coverage_target) as coverage_target_area,
+    round((min(coverage_target) * 100)::numeric, 4) as coverage_target,
+    bool_and(target_met) as on_target,
+    run_id
+  from gap_data
+  group by run_id, feature_id, scenario_id;
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+drop view scenario_features_gap_data;
+
+create view scenario_features_gap_data as
+  with gap_data as (
+    select
+      sfd.api_feature_id as feature_id,
+      scenario_id,
+      sum(total_area) total_area,
+      case when sum(current_pa) is not null
+	    then sum(current_pa)
+	    else 0
+	    end as met_area,
+      min(prop) as coverage_target
+    from scenario_features_data sfd
+    group by sfd.api_feature_id, feature_class_id, scenario_id)
+  select
+    scenario_id,
+    feature_id,
+    sum(total_area) as total_area,
+    sum(met_area) as met_area,
+    case
+      when sum(total_area) > 0
+      then round((sum(met_area)/sum(total_area))*100)
+      else 0
+    end as met,
+    sum(total_area) * min(coverage_target) as coverage_target_area,
+    round(min(coverage_target) * 100) as coverage_target,
+    sum(met_area) >= (sum(total_area) * min(coverage_target)) as on_target
+  from gap_data
+  group by feature_id, scenario_id;
+    `);
+
+    await queryRunner.query(`
+    drop view scenario_features_output_gap_data;
+
+    create view scenario_features_output_gap_data as
+      with gap_data as (
+        select
+          amount,
+          occurrences,
+          run_id,
+          sfd.api_feature_id as feature_id,
+          sfd.scenario_id,
+          osfd.total_area,
+          sfd.prop as coverage_target,
+          osfd.target as target_met
+        from output_scenarios_features_data osfd
+        inner join scenario_features_data sfd on osfd.scenario_features_id=sfd.id)
+      select
+        scenario_id,
+        feature_id,
+        sum(total_area) as total_area,
+        sum(amount) as met_area,
+        case
+          when sum(total_area) <> 0 and sum(total_area) is not null then
+            round((sum(amount)/sum(total_area))*100)
+          else
+            0
+        end as met,
+        sum(occurrences) as met_occurrences,
+        sum(total_area) * min(coverage_target) as coverage_target_area,
+        round(min(coverage_target) * 100) as coverage_target,
+        bool_and(target_met) as on_target,
+        run_id
+      from gap_data
+      group by run_id, feature_id, scenario_id;
+    `);
+  }
+}

api/libs/features/src/scenario-features-gap-data.geo.entity.ts

@@ -1,4 +1,6 @@
+import { numericStringToFloat } from '@marxan/utils/numeric-string-to-float.utils';
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
+import { Transform } from 'class-transformer';
 import { Column, ViewEntity } from 'typeorm';
 
 @ViewEntity('scenario_features_gap_data')
@@ -20,15 +22,21 @@ export class ScenarioFeaturesGapData {
   metArea!: number;
 
   @ApiProperty()
-  @Column({ name: 'met' })
+  // explicitly set type, otherwise TypeORM (v10, at least) will cast to integer
+  // TypeORM will still represent the value as string though (https://github.com/typeorm/typeorm/issues/873#issuecomment-328912050)
+  @Column({ name: 'met', type: 'double precision' })
+  @Transform(numericStringToFloat)
   met!: number;
 
   @ApiProperty()
   @Column({ name: 'coverage_target_area' })
   coverageTargetArea!: number;
 
   @ApiProperty()
-  @Column({ name: 'coverage_target' })
+  // explicitly set type, otherwise TypeORM (v10, at least) will cast to integer
+  // TypeORM will still represent the value as string though (https://github.com/typeorm/typeorm/issues/873#issuecomment-328912050)
+  @Column({ name: 'coverage_target', type: 'double precision' })
+  @Transform(numericStringToFloat)
   coverageTarget!: number;
 
   @ApiProperty()

api/libs/utils/src/numeric-string-to-float.utils.spec.ts

@@ -0,0 +1,21 @@
+import { numericStringToFloat } from './numeric-string-to-float.utils';
+
+describe('parseOptionalFloat', () => {
+  it('should return undefined if the value is undefined', () => {
+    expect(numericStringToFloat(undefined)).toBeUndefined();
+  });
+
+  it('should throw an exception if the value is not numeric', () => {
+    expect(() => numericStringToFloat('foo')).toThrow('Invalid number: foo');
+  });
+
+  it('should return a numeric representation of the value if the value is numeric', () => {
+    expect(numericStringToFloat('123.456')).toBe(123.456);
+  });
+
+  it('should silently round a float to the maximum precision supported by javascript', () => {
+    expect(numericStringToFloat('123.456789012345678901234567890')).toBe(
+      123.45678901234568,
+    );
+  });
+});

api/libs/utils/src/numeric-string-to-float.utils.ts

@@ -0,0 +1,24 @@
+import { isNil } from 'lodash';
+
+/**
+ * Kind of like parseFloat(), but passing through undefined values, and handling
+ * values that don't cast to a number.
+ *
+ * @debt It silently rounds to the maximum precision supported by Javascript any
+ * input values that are numeric but beyond what can be represented in a
+ * Javascript number (not BigInt). Infinity and -Infinity are also passed
+ * through as corresponding Javascript Infinity numeric values.
+ */
+export function numericStringToFloat(
+  value: string | undefined,
+): number | undefined {
+  // +(null) === 0, so we only cast if input is neither undefined nor null.
+  if (!isNil(value)) {
+    const floatValue = +value;
+    if (!isNaN(floatValue)) {
+      return floatValue;
+    }
+    throw new Error(`Invalid number: ${value}`);
+  }
+  return;
+}

app/components/gap-analysis/item/component.tsx

@@ -41,7 +41,7 @@ export const Item: React.FC<ItemProps> = ({
 }: ItemProps) => {
   const chartRef = useRef<HTMLDivElement>(null);
   const [chartEl, setChartEl] = useState(null);
-  const percentFormatter = useNumberFormatter({ style: 'percent' });
+  const percentFormatter = useNumberFormatter({ style: 'percent', maximumFractionDigits: 4 });
 
   const metStyles = useMemo(() => {
     if (chartEl) {