diff --git a/.env.dev b/.env.dev index f009551..3822bac 100644 --- a/.env.dev +++ b/.env.dev @@ -28,7 +28,6 @@ CLICKHOUSE_URL=http://localhost:8123 CLICKHOUSE_USER= CLICKHOUSE_PASSWORD=dev_password_only -# CHAP -CHAP_API_URL=http://localhost -# Chap Configuration + +CHAP_CLI_API_URL=http://localhost:9697 CHAP_URL=http://localhost:8000 diff --git a/src/config/config.ts b/src/config/config.ts index 4ae8817..510df99 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -23,7 +23,7 @@ export const getConfig = () => { trustSelfSigned: process.env.TRUST_SELF_SIGNED === 'false' ? false : true, runningMode: process.env.MODE || '', bodySizeLimit: process.env.BODY_SIZE_LIMIT || '50mb', - chapApiUrl: process.env.CHAP_API_URL, + chapCliApiUrl: process.env.CHAP_CLI_API_URL, minio: { endPoint: process.env.MINIO_ENDPOINT || 'localhost', port: process.env.MINIO_PORT ? parseInt(process.env.MINIO_PORT) : 9000, diff --git a/src/routes/index.ts b/src/routes/index.ts index 9e2fd24..44ddf1e 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -22,6 +22,7 @@ import FormData from 'form-data'; import { createClient } from '@clickhouse/client'; import { ModelPredictionUsingChap } from '../services/ModelPredictionUsingChap'; import { createHistoricalDiseaseTable, insertHistoricDiseaseData } from '../utils/clickhouse'; +import { createOrganizationsTable, insertOrganizationIntoTable } from '../utils/clickhouse'; // Constants const VALID_MIME_TYPES = ['text/csv', 'application/json'] as const; @@ -130,12 +131,11 @@ const handleJsonFile = async ( throw error; } }; +function sanitizeTableName(tableName: string): string { + return tableName.replace(/[^a-zA-Z0-9_-]/g, '_'); +} -const handleJsonPayload = async ( - file: Express.Multer.File, - json: Object, - bucket: string -): Promise => { +const handleJsonPayload = async (file: Express.Multer.File, json: Object, bucket: string): Promise => { try { const uploadResult = await uploadFileBufferToMinio( Buffer.from(JSON.stringify(json)), @@ -144,6 +144,12 @@ const handleJsonPayload = async ( file.mimetype ); + const tableNameOrganizations = sanitizeTableName(file.originalname) + '_organizations_' + (new Date().getMilliseconds()); + + await createOrganizationsTable(tableNameOrganizations); + + await insertOrganizationIntoTable(tableNameOrganizations, file.buffer.toString()); + return uploadResult.success ? createSuccessResponse('UPLOAD_SUCCESS', uploadResult.message) : createErrorResponse('UPLOAD_FAILED', uploadResult.message); @@ -284,7 +290,7 @@ async function getPrediction( bucket: string ) { try { - const { chapApiUrl } = getConfig(); + const { chapCliApiUrl: chapApiUrl } = getConfig(); const { url, password } = getConfig().clickhouse; const client = createClient({ url, @@ -363,6 +369,9 @@ routes.post('/predict', upload.single('file'), async (req, res) => { }, 250); })) as any; + // get organization code + const orgCode = JSON.parse(file.buffer.toString())?.orgUnitsGeoJson.features[0].properties.code; + const bucketName = sanitizeBucketName( `${file.originalname.split('.')[0]}-${Math.round(new Date().getTime() / 1000)}` ); @@ -370,6 +379,7 @@ routes.post('/predict', upload.single('file'), async (req, res) => { const predictionResultsForMinio = predictionResults?.dataValues?.map((d: any) => { return { ...d, + orgCode: orgCode ?? undefined, diseaseId: predictionResults.diseaseId as string, }; }); diff --git a/src/utils/clickhouse.ts b/src/utils/clickhouse.ts index 0004339..4f86248 100644 --- a/src/utils/clickhouse.ts +++ b/src/utils/clickhouse.ts @@ -253,3 +253,140 @@ export async function insertHistoricDiseaseData( } return client.close(); } +function isNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function checkType(feature: any): + {type: 'point', latitude: number, longitude: number} | + {type: 'polygon', coordinates: [[number, number]]} +{ + const type = feature?.geometry?.type?.toLowerCase(); + + if (type == 'point') { + if (Array.isArray(feature?.geometry?.coordinates) && + feature.geometry.coordinates.every(isNumber)) { + return { + type: 'point', + latitude: feature.geometry.coordinates[0], + longitude: feature.geometry.coordinates[1], + }; + } + } + + if (type == 'polygon') { + if (Array.isArray(feature?.geometry?.coordinates?.[0]) && + feature.geometry.coordinates?.[0].every(Array.isArray)) { + return { + type : 'polygon', + coordinates: feature.geometry.coordinates[0] as any, + }; + } + } + + throw new Error('Invalid geometry type. ' + JSON.stringify(feature)); +} + +export async function insertOrganizationIntoTable( + tableName: string, + payload: string, +) { + const client = createClient({ + url, + password, + }); + + const normalizedTableName = tableName.replace(/-/g, '_'); + + logger.info(`Inserting data into ${normalizedTableName}`); + + try { + const json = JSON.parse(payload); + + const values = json.orgUnitsGeoJson.features.map((feature: any) => { + const type = checkType(feature); + + return { + code: feature.properties.code, + name: feature.properties.name, + level: feature.properties.level, + type: type.type, + latitude: type.type == 'point' ? type.latitude : null, + longitude: type.type == 'point' ? type.longitude : null, + coordinates: type.type == 'polygon' ? type.coordinates : null, + }; + }); + + await client.insert({ + table: 'default.' + normalizedTableName, + values, + format: 'JSONEachRow', + }) + + logger.info(`Successfully inserted data into ${normalizedTableName}`); + return true; + } catch (error) { + logger.error('Error inserting data from JSON'); + logger.error(error); + return false; + } finally { + await client.close(); + } +} + +export async function createOrganizationsTable( + tableName: string, +) { + const normalizedTableName = tableName.replace(/-/g, '_'); + + logger.info(`Creating Organizations table from JSON ${normalizedTableName}`); + + const client = createClient({ + url, + password, + }); + + //check if the table exists + try { + const existsResult = await client.query({ + query: `desc ${normalizedTableName}`, + }); + logger.info(`Table ${normalizedTableName} already exists`); + await client.close(); + return false; + } catch (error) { + } + + try { + + const query = ` + CREATE TABLE IF NOT EXISTS \`default\`.${normalizedTableName} + ( code String, + name String, + level String, + type String, + latitude Float32, + longitude Float32, + coordinates Array(Array(Float32)) + ) + ENGINE = MergeTree + ORDER BY code + `; + + logger.info(query); + + const res = await client.query({ query }); + + logger.info(`Successfully created table from JSON ${normalizedTableName}`); + logger.info(res); + + await client.close(); + + return true; + } catch (err) { + logger.error(`Error creating table from JSON ${normalizedTableName}`); + logger.error(err); + return false; + } + +} diff --git a/src/utils/minioClient.ts b/src/utils/minioClient.ts index fd92e55..58ad8be 100644 --- a/src/utils/minioClient.ts +++ b/src/utils/minioClient.ts @@ -4,10 +4,12 @@ import * as Minio from 'minio'; import { getConfig } from '../config/config'; import logger from '../logger'; import { + createOrganizationsTable, createTable, createTableFromJson, insertFromS3, insertFromS3Json, + insertOrganizationIntoTable, } from './clickhouse'; import { getCsvHeaders, validateJsonFile } from './file-validators';