From e15193e38d4239461cc4c504049d5d9d99446e60 Mon Sep 17 00:00:00 2001 From: Givi Khartishvili Date: Tue, 15 Feb 2022 07:57:20 +0400 Subject: [PATCH] run all SQLs for geoCoding in same transaction as other queries --- .../admin-regions/admin-region.repository.ts | 41 ++++++--- .../admin-regions/admin-regions.service.ts | 34 +++++--- .../modules/geo-coding/geo-coding.service.ts | 22 +++-- .../aggregation-point.geocoding.service.ts | 84 ++++++++++++------- ...country-of-production.geocoding.service.ts | 3 + .../point-of-production.geocoding.service.ts | 36 +++++--- .../unknown-location.geocoding.service.ts | 3 + .../geo-regions/geo-region.repository.ts | 18 +++- .../geo-regions/geo-regions.service.ts | 6 ++ .../sourcing-data-import.service.ts | 1 + 10 files changed, 173 insertions(+), 75 deletions(-) diff --git a/api/src/modules/admin-regions/admin-region.repository.ts b/api/src/modules/admin-regions/admin-region.repository.ts index ae88ff2f7..22704f864 100644 --- a/api/src/modules/admin-regions/admin-region.repository.ts +++ b/api/src/modules/admin-regions/admin-region.repository.ts @@ -1,4 +1,9 @@ -import { EntityRepository, SelectQueryBuilder } from 'typeorm'; +import { + EntityManager, + EntityRepository, + QueryRunner, + SelectQueryBuilder, +} from 'typeorm'; import { AdminRegion } from 'modules/admin-regions/admin-region.entity'; import { ExtendedTreeRepository } from 'utils/tree.repository'; import { CreateAdminRegionDto } from 'modules/admin-regions/dto/create.admin-region.dto'; @@ -13,12 +18,18 @@ export class AdminRegionRepository extends ExtendedTreeRepository< > { logger: Logger = new Logger(AdminRegionRepository.name); - async getAdminRegionAndGeoRegionIdByCoordinatesAndLevel(searchParams: { - lng: number; - lat: number; - level: number; - }): Promise<{ adminRegionId: string; geoRegionId: string }> { - const res: any = await this.query( + async getAdminRegionAndGeoRegionIdByCoordinatesAndLevel( + searchParams: { + lng: number; + lat: number; + level: number; + }, + queryRunner?: QueryRunner, + ): Promise<{ adminRegionId: string; geoRegionId: string }> { + const manager: EntityManager | this = queryRunner + ? queryRunner.manager + : this; + const res: any = await manager.query( ` SELECT a.id AS "adminRegionId", g.id AS "geoRegionId" FROM admin_region a @@ -55,11 +66,17 @@ export class AdminRegionRepository extends ExtendedTreeRepository< // level 1 or 2, and depending on coordinates, level 0. // Check how to properly perform this - async getClosestAdminRegionByCoordinates(coordinates: { - lng: number; - lat: number; - }): Promise { - const res: any = await this.query( + async getClosestAdminRegionByCoordinates( + coordinates: { + lng: number; + lat: number; + }, + queryRunner?: QueryRunner, + ): Promise { + const manager: EntityManager | this = queryRunner + ? queryRunner.manager + : this; + const res: any = await manager.query( `SELECT a.id AS "adminRegionId" , a."name", a."level" , g."name" , g.id AS "geoRegionId" FROM admin_region a RIGHT JOIN geo_region g on a."geoRegionId" = g.id diff --git a/api/src/modules/admin-regions/admin-regions.service.ts b/api/src/modules/admin-regions/admin-regions.service.ts index ced39904d..6c28dcd96 100644 --- a/api/src/modules/admin-regions/admin-regions.service.ts +++ b/api/src/modules/admin-regions/admin-regions.service.ts @@ -15,6 +15,7 @@ import { UpdateAdminRegionDto } from 'modules/admin-regions/dto/update.admin-reg import { FindTreesWithOptionsArgs } from 'utils/tree.repository'; import { SourcingLocationsService } from 'modules/sourcing-locations/sourcing-locations.service'; import { GetAdminRegionTreeWithOptionsDto } from 'modules/admin-regions/dto/get-admin-region-tree-with-options.dto'; +import { QueryBuilder, QueryRunner } from 'typeorm'; @Injectable() export class AdminRegionsService extends AppBaseService< @@ -77,9 +78,12 @@ export class AdminRegionsService extends AppBaseService< // TODO: proper typing after validating this works async getAdminAndGeoRegionIdByCountryIsoAlpha2( countryIsoAlpha2Code: string, + queryRunner?: QueryRunner, ): Promise<{ id: string; geoRegionId: string }> { - const adminAndGeoRegionId: any = await this.adminRegionRepository - .createQueryBuilder('ar') + const adminRegionQueryBuilder: QueryBuilder = queryRunner + ? queryRunner.manager.createQueryBuilder(AdminRegion, 'ar') + : this.adminRegionRepository.createQueryBuilder('ar'); + const adminAndGeoRegionId: any = await adminRegionQueryBuilder .select('id') .addSelect('"geoRegionId"') .where('ar.isoA2 = :countryIsoAlpha2Code', { @@ -106,22 +110,30 @@ export class AdminRegionsService extends AppBaseService< return adminRegion; } - async getAdminRegionIdByCoordinatesAndLevel(searchParams: { - lng: number; - lat: number; - level: number; - }): Promise<{ adminRegionId: string; geoRegionId: string }> { + async getAdminRegionIdByCoordinatesAndLevel( + searchParams: { + lng: number; + lat: number; + level: number; + }, + queryRunner?: QueryRunner, + ): Promise<{ adminRegionId: string; geoRegionId: string }> { return this.adminRegionRepository.getAdminRegionAndGeoRegionIdByCoordinatesAndLevel( searchParams, + queryRunner, ); } - async getClosestAdminRegionByCoordinates(coordinates: { - lng: number; - lat: number; - }): Promise { + async getClosestAdminRegionByCoordinates( + coordinates: { + lng: number; + lat: number; + }, + queryRunner?: QueryRunner, + ): Promise { return this.adminRegionRepository.getClosestAdminRegionByCoordinates( coordinates, + queryRunner, ); } diff --git a/api/src/modules/geo-coding/geo-coding.service.ts b/api/src/modules/geo-coding/geo-coding.service.ts index 7a0c62f68..2c6309df9 100644 --- a/api/src/modules/geo-coding/geo-coding.service.ts +++ b/api/src/modules/geo-coding/geo-coding.service.ts @@ -5,6 +5,7 @@ import { CountryOfProductionService } from 'modules/geo-coding/geocoding-strateg import { UnknownLocationService } from 'modules/geo-coding/geocoding-strategies/unknown-location.geocoding.service'; import { SourcingData } from 'modules/import-data/sourcing-data/dto-processor.service'; import { LOCATION_TYPES } from 'modules/sourcing-locations/sourcing-location.entity'; +import { QueryRunner } from 'typeorm'; @Injectable() export class GeoCodingService { @@ -19,6 +20,7 @@ export class GeoCodingService { async geoCodeLocations( sourcingData: SourcingData[], + queryRunner?: QueryRunner, ): Promise { this.logger.log( `Geocoding locations for ${sourcingData.length} sourcing record elements`, @@ -28,25 +30,25 @@ export class GeoCodingService { sourcingData.map(async (sourcingData: SourcingData) => { if (sourcingData.locationType === LOCATION_TYPES.UNKNOWN) { geoCodedSourcingData.push( - await this.geoCodeUnknownLocationType(sourcingData), + await this.geoCodeUnknownLocationType(sourcingData, queryRunner), ); } if ( sourcingData.locationType === LOCATION_TYPES.COUNTRY_OF_PRODUCTION ) { geoCodedSourcingData.push( - await this.geoCodeCountryOfProduction(sourcingData), + await this.geoCodeCountryOfProduction(sourcingData, queryRunner), ); } if (sourcingData.locationType === LOCATION_TYPES.AGGREGATION_POINT) { geoCodedSourcingData.push( - await this.geoCodeAggregationPoint(sourcingData), + await this.geoCodeAggregationPoint(sourcingData, queryRunner), ); } if (sourcingData.locationType === LOCATION_TYPES.POINT_OF_PRODUCTION) { geoCodedSourcingData.push( - await this.geoCodePointOfProduction(sourcingData), + await this.geoCodePointOfProduction(sourcingData, queryRunner), ); } }), @@ -56,31 +58,41 @@ export class GeoCodingService { async geoCodeAggregationPoint( sourcingData: SourcingData, + queryRunner?: QueryRunner, ): Promise { return this.aggregationPointGeocodingService.geoCodeAggregationPoint( sourcingData, + queryRunner, ); } async geoCodePointOfProduction( sourcingData: SourcingData, + queryRunner?: QueryRunner, ): Promise { return this.pointOfProductionGeocodingService.geoCodePointOfProduction( sourcingData, + queryRunner, ); } async geoCodeCountryOfProduction( sourcingData: SourcingData, + queryRunner?: QueryRunner, ): Promise { return this.countryOfProductionService.geoCodeCountryOfProduction( sourcingData, + queryRunner, ); } async geoCodeUnknownLocationType( sourcingData: SourcingData, + queryRunner?: QueryRunner, ): Promise { - return this.unknownLocationService.geoCodeUnknownLocationType(sourcingData); + return this.unknownLocationService.geoCodeUnknownLocationType( + sourcingData, + queryRunner, + ); } } diff --git a/api/src/modules/geo-coding/geocoding-strategies/aggregation-point.geocoding.service.ts b/api/src/modules/geo-coding/geocoding-strategies/aggregation-point.geocoding.service.ts index 67fc3cc2d..cfd3cf431 100644 --- a/api/src/modules/geo-coding/geocoding-strategies/aggregation-point.geocoding.service.ts +++ b/api/src/modules/geo-coding/geocoding-strategies/aggregation-point.geocoding.service.ts @@ -3,13 +3,17 @@ import { GeoCodingBaseService } from 'modules/geo-coding/geo-coding.base.service import { SourcingData } from 'modules/import-data/sourcing-data/dto-processor.service'; import { GeocodeResponseData } from '@googlemaps/google-maps-services-js/dist/geocode/geocode'; import { GeoRegion } from 'modules/geo-regions/geo-region.entity'; +import { QueryRunner } from 'typeorm'; @Injectable() export class AggregationPointGeocodingService extends GeoCodingBaseService { aggregationPointGeocodingLogger: Logger = new Logger( AggregationPointGeocodingService.name, ); - async geoCodeAggregationPoint(sourcingData: SourcingData): Promise { + async geoCodeAggregationPoint( + sourcingData: SourcingData, + queryRunner?: QueryRunner, + ): Promise { /** * The user must specify a country, and either an address OR coordinates */ @@ -23,22 +27,28 @@ export class AggregationPointGeocodingService extends GeoCodingBaseService { */ if (sourcingData.locationLatitude && sourcingData.locationLongitude) { const geoRegionId: Pick = - await this.geoRegionService.saveGeoRegionAsRadius({ - name: sourcingData.locationCountryInput, - coordinates: { - lng: sourcingData.locationLongitude, - lat: sourcingData.locationLatitude, + await this.geoRegionService.saveGeoRegionAsRadius( + { + name: sourcingData.locationCountryInput, + coordinates: { + lng: sourcingData.locationLongitude, + lat: sourcingData.locationLatitude, + }, }, - }); + queryRunner, + ); /** * Get closest AdminRegion given the same point */ const { adminRegionId } = - await this.adminRegionService.getClosestAdminRegionByCoordinates({ - lng: sourcingData.locationLongitude, - lat: sourcingData.locationLatitude, - }); + await this.adminRegionService.getClosestAdminRegionByCoordinates( + { + lng: sourcingData.locationLongitude, + lat: sourcingData.locationLatitude, + }, + queryRunner, + ); return { ...sourcingData, @@ -65,11 +75,14 @@ export class AggregationPointGeocodingService extends GeoCodingBaseService { */ if (this.isAddressAdminLevel1(geocodedResponseData.results[0].types)) { const { adminRegionId, geoRegionId } = - await this.adminRegionService.getAdminRegionIdByCoordinatesAndLevel({ - lng: geocodedResponseData?.results[0]?.geometry.location.lng, - lat: geocodedResponseData?.results[0]?.geometry.location.lat, - level: 1, - }); + await this.adminRegionService.getAdminRegionIdByCoordinatesAndLevel( + { + lng: geocodedResponseData?.results[0]?.geometry.location.lng, + lat: geocodedResponseData?.results[0]?.geometry.location.lat, + level: 1, + }, + queryRunner, + ); return { ...sourcingData, adminRegionId, @@ -78,11 +91,14 @@ export class AggregationPointGeocodingService extends GeoCodingBaseService { } if (this.isAddressAdminLevel2(geocodedResponseData.results[0].types)) { const { adminRegionId, geoRegionId } = - await this.adminRegionService.getAdminRegionIdByCoordinatesAndLevel({ - lng: geocodedResponseData?.results[0]?.geometry.location.lng, - lat: geocodedResponseData?.results[0]?.geometry.location.lat, - level: 2, - }); + await this.adminRegionService.getAdminRegionIdByCoordinatesAndLevel( + { + lng: geocodedResponseData?.results[0]?.geometry.location.lng, + lat: geocodedResponseData?.results[0]?.geometry.location.lat, + level: 2, + }, + queryRunner, + ); return { ...sourcingData, adminRegionId, @@ -95,21 +111,27 @@ export class AggregationPointGeocodingService extends GeoCodingBaseService { * by it's name */ const geoRegionId: GeoRegion = - await this.geoRegionService.saveGeoRegionAsRadius({ - name: sourcingData.locationCountryInput, - coordinates: { - lat: geocodedResponseData.results[0].geometry.location.lat, - lng: geocodedResponseData.results[0].geometry.location.lng, + await this.geoRegionService.saveGeoRegionAsRadius( + { + name: sourcingData.locationCountryInput, + coordinates: { + lat: geocodedResponseData.results[0].geometry.location.lat, + lng: geocodedResponseData.results[0].geometry.location.lng, + }, }, - }); + queryRunner, + ); /** * Get closest AdminRegion given the same point */ const { adminRegionId } = - await this.adminRegionService.getClosestAdminRegionByCoordinates({ - lng: geocodedResponseData?.results[0]?.geometry.location.lng, - lat: geocodedResponseData?.results[0]?.geometry.location.lat, - }); + await this.adminRegionService.getClosestAdminRegionByCoordinates( + { + lng: geocodedResponseData?.results[0]?.geometry.location.lng, + lat: geocodedResponseData?.results[0]?.geometry.location.lat, + }, + queryRunner, + ); return { ...sourcingData, diff --git a/api/src/modules/geo-coding/geocoding-strategies/country-of-production.geocoding.service.ts b/api/src/modules/geo-coding/geocoding-strategies/country-of-production.geocoding.service.ts index fb3d3a801..3c9948320 100644 --- a/api/src/modules/geo-coding/geocoding-strategies/country-of-production.geocoding.service.ts +++ b/api/src/modules/geo-coding/geocoding-strategies/country-of-production.geocoding.service.ts @@ -3,11 +3,13 @@ import { GeoCodingBaseService } from 'modules/geo-coding/geo-coding.base.service import { SourcingData } from 'modules/import-data/sourcing-data/dto-processor.service'; import { SourcingLocation } from 'modules/sourcing-locations/sourcing-location.entity'; import { GeocodeResponseData } from '@googlemaps/google-maps-services-js/dist/geocode/geocode'; +import { QueryRunner } from 'typeorm'; @Injectable() export class CountryOfProductionService extends GeoCodingBaseService { async geoCodeCountryOfProduction( sourcingData: SourcingData, + queryRunner?: QueryRunner, ): Promise { /** * The user must specify a country, and either address OR coordinates. @@ -31,6 +33,7 @@ export class CountryOfProductionService extends GeoCodingBaseService { const { id: adminRegionId, geoRegionId } = await this.adminRegionService.getAdminAndGeoRegionIdByCountryIsoAlpha2( geoCodedResponse.results[0]?.address_components?.[0]?.short_name, + queryRunner, ); return { ...sourcingData, diff --git a/api/src/modules/geo-coding/geocoding-strategies/point-of-production.geocoding.service.ts b/api/src/modules/geo-coding/geocoding-strategies/point-of-production.geocoding.service.ts index 1660e42e8..412faaaf4 100644 --- a/api/src/modules/geo-coding/geocoding-strategies/point-of-production.geocoding.service.ts +++ b/api/src/modules/geo-coding/geocoding-strategies/point-of-production.geocoding.service.ts @@ -3,10 +3,14 @@ import { GeoCodingBaseService } from 'modules/geo-coding/geo-coding.base.service import { SourcingData } from 'modules/import-data/sourcing-data/dto-processor.service'; import { GeocodeResponseData } from '@googlemaps/google-maps-services-js/dist/geocode/geocode'; import { GeoRegion } from 'modules/geo-regions/geo-region.entity'; +import { QueryRunner } from 'typeorm'; @Injectable() export class PointOfProductionGeocodingService extends GeoCodingBaseService { - async geoCodePointOfProduction(sourcingData: SourcingData): Promise { + async geoCodePointOfProduction( + sourcingData: SourcingData, + queryRunner?: QueryRunner, + ): Promise { if (!sourcingData.locationCountryInput) throw new Error( 'A country must be provided for Point of Production location type', @@ -18,13 +22,16 @@ export class PointOfProductionGeocodingService extends GeoCodingBaseService { if (sourcingData.locationLongitude && sourcingData.locationLatitude) { const geoRegionId: Pick = - await this.geoRegionService.saveGeoRegionAsPoint({ - name: sourcingData.locationCountryInput, - coordinates: { - lat: sourcingData.locationLatitude, - lng: sourcingData.locationLongitude, + await this.geoRegionService.saveGeoRegionAsPoint( + { + name: sourcingData.locationCountryInput, + coordinates: { + lat: sourcingData.locationLatitude, + lng: sourcingData.locationLongitude, + }, }, - }); + queryRunner, + ); const { adminRegionId } = await this.adminRegionService.getClosestAdminRegionByCoordinates({ @@ -43,13 +50,16 @@ export class PointOfProductionGeocodingService extends GeoCodingBaseService { await this.geoCodeByAddress(sourcingData.locationAddressInput); const geoRegionId: Pick = - await this.geoRegionService.saveGeoRegionAsPoint({ - name: sourcingData.locationCountryInput, - coordinates: { - lat: geoCodeResponseData.results[0].geometry.location.lat, - lng: geoCodeResponseData.results[0].geometry.location.lng, + await this.geoRegionService.saveGeoRegionAsPoint( + { + name: sourcingData.locationCountryInput, + coordinates: { + lat: geoCodeResponseData.results[0].geometry.location.lat, + lng: geoCodeResponseData.results[0].geometry.location.lng, + }, }, - }); + queryRunner, + ); const { adminRegionId } = await this.adminRegionService.getClosestAdminRegionByCoordinates({ diff --git a/api/src/modules/geo-coding/geocoding-strategies/unknown-location.geocoding.service.ts b/api/src/modules/geo-coding/geocoding-strategies/unknown-location.geocoding.service.ts index 084768735..5cb840726 100644 --- a/api/src/modules/geo-coding/geocoding-strategies/unknown-location.geocoding.service.ts +++ b/api/src/modules/geo-coding/geocoding-strategies/unknown-location.geocoding.service.ts @@ -3,11 +3,13 @@ import { GeoCodingBaseService } from 'modules/geo-coding/geo-coding.base.service import { SourcingData } from 'modules/import-data/sourcing-data/dto-processor.service'; import { SourcingLocation } from 'modules/sourcing-locations/sourcing-location.entity'; import { GeocodeResponseData } from '@googlemaps/google-maps-services-js/dist/geocode/geocode'; +import { QueryRunner } from 'typeorm'; @Injectable() export class UnknownLocationService extends GeoCodingBaseService { async geoCodeUnknownLocationType( sourcingData: SourcingData, + queryRunner?: QueryRunner, ): Promise { /** * The user must specify a country, but address and coordinates should be empty @@ -30,6 +32,7 @@ export class UnknownLocationService extends GeoCodingBaseService { const { id: adminRegionId, geoRegionId } = await this.adminRegionService.getAdminAndGeoRegionIdByCountryIsoAlpha2( geoCodedResponse.results[0]?.address_components?.[0]?.short_name, + queryRunner, ); return { ...sourcingData, diff --git a/api/src/modules/geo-regions/geo-region.repository.ts b/api/src/modules/geo-regions/geo-region.repository.ts index b08fed095..cfa68034f 100644 --- a/api/src/modules/geo-regions/geo-region.repository.ts +++ b/api/src/modules/geo-regions/geo-region.repository.ts @@ -1,7 +1,10 @@ import { + EntityManager, EntityRepository, getManager, InsertResult, + QueryBuilder, + QueryRunner, Repository, SelectQueryBuilder, } from 'typeorm'; @@ -25,8 +28,12 @@ export class GeoRegionRepository extends Repository { */ async saveGeoRegionAsRadius( newGeoRegionValues: LocationGeoRegionDto, + queryRunner?: QueryRunner, ): Promise { - const selectQuery: SelectQueryBuilder = getManager() + const manager: EntityManager = queryRunner + ? queryRunner.manager + : getManager(); + const selectQuery: SelectQueryBuilder = manager .createQueryBuilder() .select(`hashtext(concat($3::text, points.radius))`) .addSelect(`points.radius`) @@ -37,7 +44,7 @@ export class GeoRegionRepository extends Repository { ) .from('points', 'points'); - const res: any = await this.query( + const res: any = await manager.query( `WITH points AS (SELECT ST_BUFFER(ST_SetSRID(ST_POINT($1,$2),4326)::geometry, 0.5) as radius) INSERT INTO geo_region (name, "theGeom", "h3Compact") @@ -58,12 +65,17 @@ export class GeoRegionRepository extends Repository { /** * Saves a new geo-regions with theGeom as POINT (as it comes) * @param newGeoRegionValues name, coordinates + * @param queryRunner * @returns created geo-regions id */ async saveGeoRegionAsPoint( newGeoRegionValues: LocationGeoRegionDto, + queryRunner?: QueryRunner, ): Promise> { - const res: InsertResult = await this.createQueryBuilder() + const geoRegionQueryBuilder: QueryBuilder = queryRunner + ? queryRunner.manager.createQueryBuilder(GeoRegion, 'geoRegion') + : this.createQueryBuilder(); + const res: InsertResult = await geoRegionQueryBuilder .insert() .values({ name: () => `hashtext(:arg1)`, diff --git a/api/src/modules/geo-regions/geo-regions.service.ts b/api/src/modules/geo-regions/geo-regions.service.ts index 3d9ab7521..7934d90aa 100644 --- a/api/src/modules/geo-regions/geo-regions.service.ts +++ b/api/src/modules/geo-regions/geo-regions.service.ts @@ -13,6 +13,7 @@ import { GeoRegionRepository } from 'modules/geo-regions/geo-region.repository'; import { CreateGeoRegionDto } from 'modules/geo-regions/dto/create.geo-region.dto'; import { UpdateGeoRegionDto } from 'modules/geo-regions/dto/update.geo-region.dto'; import { LocationGeoRegionDto } from 'modules/geo-regions/dto/location.geo-region.dto'; +import { QueryRunner } from 'typeorm'; @Injectable() export class GeoRegionsService extends AppBaseService< @@ -57,22 +58,27 @@ export class GeoRegionsService extends AppBaseService< * Creates a new geo-region row and generates a 50km radius as multipolygon geometry * by given coordinates * + * @param queryRunner * @return generated geo-regions id */ async saveGeoRegionAsRadius( newGeoRegionValues: LocationGeoRegionDto, + queryRunner?: QueryRunner, ): Promise { return await this.geoRegionRepository.saveGeoRegionAsRadius( newGeoRegionValues, + queryRunner, ); } async saveGeoRegionAsPoint( newGeroRegionValues: LocationGeoRegionDto, + queryRunner?: QueryRunner, ): Promise> { return await this.geoRegionRepository.saveGeoRegionAsPoint( newGeroRegionValues, + queryRunner, ); } } diff --git a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts index c2b039807..e22e1bd32 100644 --- a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts +++ b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts @@ -123,6 +123,7 @@ export class SourcingDataImportService { const geoCodedSourcingData: SourcingData[] = await this.geoCodingService.geoCodeLocations( sourcingDataWithOrganizationalEntities, + queryRunner, ); await this.sourcingLocationService.save(