Endpoint to retrieve filtered EUDR Alerts

api/config/custom-environment-variables.json

@@ -73,9 +73,10 @@
       "sendGridApiKey": "SENDGRID_API_KEY"
     }
   },
-  "carto": {
+  "eudr": {
     "apiKey": "CARTO_API_KEY",
     "baseUrl": "CARTO_BASE_URL",
-    "credentials": "CARTO_CREDENTIALS"
+    "credentials": "EUDR_CREDENTIALS",
+    "dataset": "EUDR_DATASET"
   }
 }

api/config/default.json

@@ -82,9 +82,10 @@
       "sendGridApiKey": null
     }
   },
-  "carto": {
+  "eudr": {
     "apiKey": null,
     "baseUrl": "null",
-    "credentials": null
+    "credentials": null,
+    "dataset": null
   }
 }

api/config/test.json

@@ -46,5 +46,11 @@
     "email": {
       "sendGridApiKey": "SG.forSomeReasonSendGridApiKeysNeedToStartWithSG."
     }
+  },
+  "eudr": {
+    "apiKey": null,
+    "baseUrl": "null",
+    "credentials": null,
+    "dataset": "test_dataset"
   }
 }

api/src/modules/eudr-alerts/alerts-query-builder/big-query-alerts-query.builder.ts

@@ -0,0 +1,118 @@
+import { SelectQueryBuilder } from 'typeorm';
+import { AlertsOutput } from 'modules/eudr-alerts/dto/alerts-output.dto';
+import { GetEUDRAlertsDto } from 'modules/eudr-alerts/dto/get-alerts.dto';
+import { Query } from '@google-cloud/bigquery';
+
+export class BigQueryAlertsQueryBuilder {
+  queryBuilder: SelectQueryBuilder<AlertsOutput>;
+  dto?: GetEUDRAlertsDto;
+
+  constructor(
+    queryBuilder: SelectQueryBuilder<AlertsOutput>,
+    getAlertsDto?: GetEUDRAlertsDto,
+  ) {
+    this.queryBuilder = queryBuilder;
+    this.dto = getAlertsDto;
+  }
+
+  buildQuery(): Query {
+    if (this.dto?.supplierIds) {
+      this.queryBuilder.andWhere('supplierid IN (:...supplierIds)', {
+        supplierIds: this.dto.supplierIds,
+      });
+    }
+    if (this.dto?.geoRegionIds) {
+      this.queryBuilder.andWhere('georegionid IN (:...geoRegionIds)', {
+        geoRegionIds: this.dto.geoRegionIds,
+      });
+    }
+    if (this.dto?.alertConfidence) {
+      this.queryBuilder.andWhere('alertConfidence = :alertConfidence', {
+        alertConfidence: this.dto.alertConfidence,
+      });
+    }
+
+    if (this.dto?.startYear && this.dto?.endYear) {
+      this.addYearRange();
+    } else if (this.dto?.startYear) {
+      this.addYearGreaterThanOrEqual();
+    } else if (this.dto?.endYear) {
+      this.addYearLessThanOrEqual();
+    }
+
+    if (this.dto?.startAlertDate && this.dto?.endAlertDate) {
+      this.addAlertDateRange();
+    } else if (this.dto?.startAlertDate) {
+      this.addAlertDateGreaterThanOrEqual();
+    } else if (this.dto?.endAlertDate) {
+      this.addAlertDateLessThanOrEqual();
+    }
+
+    this.queryBuilder.limit(this.dto?.limit);
+
+    const [query, params] = this.queryBuilder.getQueryAndParameters();
+
+    return this.parseToBigQuery(query, params);
+  }
+
+  addYearRange(): void {
+    this.queryBuilder.andWhere('year BETWEEN :startYear AND :endYear', {
+      startYear: this.dto?.startYear,
+      endYear: this.dto?.endYear,
+    });
+  }
+
+  addYearGreaterThanOrEqual(): void {
+    this.queryBuilder.andWhere('year >= :startYear', {
+      startYear: this.dto?.startYear,
+    });
+  }
+
+  addYearLessThanOrEqual(): void {
+    this.queryBuilder.andWhere('year <= :endYear', {
+      endYear: this.dto?.endYear,
+    });
+  }
+
+  addAlertDateRange(): void {
+    this.queryBuilder.andWhere(
+      'DATE(alertdate) BETWEEN :startAlertDate AND :endAlertDate',
+      {
+        startAlertDate: this.dto?.startAlertDate,
+        endAlertDate: this.dto?.endAlertDate,
+      },
+    );
+  }
+
+  addAlertDateGreaterThanOrEqual(): void {
+    this.queryBuilder.andWhere('DATE(alertdate) >= DATE(:startAlertDate)', {
+      startAlertDate: this.dto?.startAlertDate,
+    });
+  }
+
+  addAlertDateLessThanOrEqual(): void {
+    this.queryBuilder.andWhere('DATE(alertDate) <= :DATE(endAlertDate)', {
+      endAlertDate: this.dto?.endAlertDate,
+    });
+  }
+
+  parseToBigQuery(query: string, params: any[]): Query {
+    return {
+      query: this.removeDoubleQuotesAndReplacePositionalArguments(query),
+      params,
+    };
+  }
+
+  /**
+   * @description: BigQuery does not allow double quotes and the positional argument symbol must be a "?".
+   * So there is a need to replace the way TypeORM handles the positional arguments, with $1, $2, etc.
+   */
+
+  private removeDoubleQuotesAndReplacePositionalArguments(
+    query: string,
+  ): string {
+    return query.replace(/\$\d+|"/g, (match: string) =>
+      match === '"' ? '' : '?',
+    );
+  }
+}

api/src/modules/eudr-alerts/alerts.repository.ts

@@ -1,65 +1,85 @@
-import { BigQuery } from '@google-cloud/bigquery';
-import { Injectable } from '@nestjs/common';
+import {
+  BigQuery,
+  Query,
+  SimpleQueryRowsResponse,
+} from '@google-cloud/bigquery';
+import {
+  Inject,
+  Injectable,
+  Logger,
+  ServiceUnavailableException,
+} from '@nestjs/common';
 import { DataSource, SelectQueryBuilder } from 'typeorm';
-import { AlertsOutput } from './dto/alerts-output.dto';
-import { ResourceStream } from '@google-cloud/paginator';
-import { RowMetadata } from '@google-cloud/bigquery/build/src/table';
-import { IEUDRAlertsRepository } from './eudr.repositoty.interface';
-import { AppConfig } from '../../utils/app.config';
+import { AlertsOutput } from 'modules/eudr-alerts/dto/alerts-output.dto';
+import {
+  EUDRAlertDates,
+  GetEUDRAlertDatesDto,
+  IEUDRAlertsRepository,
+} from 'modules/eudr-alerts/eudr.repositoty.interface';
+import { GetEUDRAlertsDto } from 'modules/eudr-alerts/dto/get-alerts.dto';
+import { BigQueryAlertsQueryBuilder } from 'modules/eudr-alerts/alerts-query-builder/big-query-alerts-query.builder';
 
 const projectId: string = 'carto-dw-ac-zk2uhih6';
 
-const limit: number = 1;
-
 @Injectable()
 export class AlertsRepository implements IEUDRAlertsRepository {
+  logger: Logger = new Logger(AlertsRepository.name);
   bigQueryClient: BigQuery;
-  BASE_DATASET: string = 'cartobq.eudr.dev_mock_data_optimized';
 
-  constructor(private readonly dataSource: DataSource) {
-    const { credentials } = AppConfig.get('carto');
+  constructor(
+    private readonly dataSource: DataSource,
+    @Inject('EUDRCredentials') private credentials: string,
+    @Inject('EUDRDataset') private baseDataset: string,
+  ) {
+    // if (!credentials) {
+    //   this.logger.error('BigQuery credentials are missing');
+    //   throw new ServiceUnavailableException(
+    //     'EUDR Module not available. Tearing down the application',
+    //   );
+    // }
     this.bigQueryClient = new BigQuery({
-      credentials: JSON.parse(credentials),
+      credentials: JSON.parse(this.credentials),
       projectId,
     });
   }
 
-  select(dto?: any): ResourceStream<RowMetadata> {
-    const queryBuilder: SelectQueryBuilder<AlertsOutput> = this.dataSource
-      .createQueryBuilder()
-      .select('georegionid', 'geoRegionId')
-      .addSelect('supplierid', 'supplierId')
-      .addSelect('geometry', 'geometry')
-      .where('alertcount >= :alertCount', { alertCount: 2 })
-      .andWhere('supplierid IN (:...supplierIds)', {
-        supplierIds: [
-          '4132ab95-8b04-4438-b706-a82651f491bd',
-          '4132ab95-8b04-4438-b706-a82651f491bd',
-          '4132ab95-8b04-4438-b706-a82651f491bd',
-        ],
-      });
-    if (limit) {
-      queryBuilder.limit(limit);
+  async getAlerts(dto?: GetEUDRAlertsDto): Promise<AlertsOutput[]> {
+    const queryBuilder: SelectQueryBuilder<AlertsOutput> =
+      this.dataSource.createQueryBuilder();
+    // TODO: Make field selection dynamic
+    queryBuilder.from(this.baseDataset, 'alerts');
+    queryBuilder.select('alertdate', 'alertDate');
+    queryBuilder.addSelect('alertconfidence', 'alertConfidence');
+    queryBuilder.addSelect('year', 'alertYear');
+    queryBuilder.addSelect('alertcount', 'alertCount');
+    try {
+      const response: SimpleQueryRowsResponse = await this.bigQueryClient.query(
+        this.buildQuery(queryBuilder, dto),
+      );
+      if (!response.length || 'error' in response) {
+        this.logger.error('Error in query', response);
+        throw new Error();
+      }
+      return response[0];
+    } catch (e) {
+      this.logger.error('Error in query', e);
+      throw new ServiceUnavailableException(
+        'Unable to retrieve EUDR Data. Please contact your administrator.',
+      );
     }
-    // const [rows] = await this.bigQueryClient.query(
-    //   this.buildQuery(queryBuilder),
-    // );
-    return this.bigQueryClient.createQueryStream(this.buildQuery(queryBuilder));
   }
 
-  private buildQuery(queryBuilder: SelectQueryBuilder<AlertsOutput>): {
-    query: string;
-    params: any[];
-  } {
-    const [query, params] = queryBuilder
-      .from(this.BASE_DATASET, 'alerts')
-      .getQueryAndParameters();
-    const queryOptions = {
-      query: query.replace(/\$\d+|"/g, (match: string) =>
-        match === '"' ? '' : '?',
-      ),
-      params,
-    };
-    return queryOptions;
+  getDates(dto: GetEUDRAlertDatesDto): Promise<EUDRAlertDates[]> {
+    return [] as any;
+  }
+
+  private buildQuery(
+    queryBuilder: SelectQueryBuilder<AlertsOutput>,
+    dto?: GetEUDRAlertsDto,
+  ): Query {
+    const alertsQueryBuilder: BigQueryAlertsQueryBuilder =
+      new BigQueryAlertsQueryBuilder(queryBuilder, dto);
+
+    return alertsQueryBuilder.buildQuery();
   }
 }

api/src/modules/eudr-alerts/dto/alerts-output.dto.ts

@@ -1,11 +1,14 @@
 import { GeoJSON } from 'geojson';
 
-export class AlertsOutput {
-  geoRegionId: string;
-  supplierId: string;
+export type AlertsOutput = {
   alertCount: boolean;
-  geometry: GeoJSON;
   date: Date;
   year: number;
   alertConfidence: 'low' | 'medium' | 'high' | 'very high';
-}
+};
+
+export type AlertGeometry = {
+  geometry: { value: string };
+};
+
+export type AlertsWithGeom = AlertsOutput & AlertGeometry;

api/src/modules/eudr-alerts/dto/get-alerts.dto.ts

@@ -1,6 +1,48 @@
-export class GetEUDRALertsDto {
-  supplierId: string;
-  geoRegionId: string;
-  geom: any;
-  year: number;
+import { Type } from 'class-transformer';
+import {
+  IsArray,
+  IsDate,
+  IsEnum,
+  IsInt,
+  IsNumber,
+  IsOptional,
+  IsUUID,
+} from 'class-validator';
+
+export class GetEUDRAlertsDto {
+  @IsOptional()
+  @IsArray()
+  @IsUUID('4', { each: true })
+  supplierIds: string[];
+
+  @IsOptional()
+  @IsArray()
+  @IsUUID('4', { each: true })
+  geoRegionIds: string[];
+
+  @IsOptional()
+  @IsNumber()
+  @Type(() => Number)
+  startYear: number;
+
+  @IsOptional()
+  @IsNumber()
+  @Type(() => Number)
+  endYear: number;
+
+  alertConfidence: 'high' | 'medium' | 'low';
+
+  @IsOptional()
+  @IsDate()
+  @Type(() => Date)
+  startAlertDate: Date;
+
+  @IsOptional()
+  @IsDate()
+  @Type(() => Date)
+  endAlertDate: Date;
+
+  @IsOptional()
+  @IsInt()
+  limit: number = 1000;
 }

api/src/modules/eudr-alerts/eudr.controller.ts

@@ -32,10 +32,8 @@ import {
 } from 'modules/geo-regions/geo-region.entity';
 import { JSONAPIQueryParams } from 'decorators/json-api-parameters.decorator';
 import { GetEUDRGeoRegions } from 'modules/geo-regions/dto/get-geo-region.dto';
-import { AlertsOutput } from 'modules/eudr-alerts/dto/alerts-output.dto';
 import { EudrService } from 'modules/eudr-alerts/eudr.service';
-import { ResourceStream } from '@google-cloud/paginator';
-import { GetEUDRALertsDto } from './dto/get-alerts.dto';
+import { GetEUDRAlertsDto } from 'modules/eudr-alerts/dto/get-alerts.dto';
 
 @Controller('/api/v1/eudr')
 export class EudrController {
@@ -143,13 +141,8 @@ export class EudrController {
   }
 
   @Get('/alerts')
-  async getAlerts(
-    @Res() response: Response,
-    dto: GetEUDRALertsDto,
-  ): Promise<any> {
-    const stream: ResourceStream<AlertsOutput> =
-      this.eudrAlertsService.getAlerts();
-    this.streamResponse(response, stream);
+  async getAlerts(@Query(ValidationPipe) dto: GetEUDRAlertsDto): Promise<any> {
+    return this.eudrAlertsService.getAlerts(dto);
   }
 
   streamResponse(response: Response, stream: Writable): any {