Merge remote-tracking branch 'origin/main' into 86c2d4c60-save-histor…

…ic-disease

.env.dev

@@ -28,7 +28,6 @@ CLICKHOUSE_URL=http://localhost:8123
 CLICKHOUSE_USER=
 CLICKHOUSE_PASSWORD=dev_password_only
 
-# CHAP
-CHAP_API_URL=http://localhost
-# Chap Configuration
+
+CHAP_CLI_API_URL=http://localhost:9697
 CHAP_URL=http://localhost:8000

src/config/config.ts

@@ -23,7 +23,7 @@ export const getConfig = () => {
     trustSelfSigned: process.env.TRUST_SELF_SIGNED === 'false' ? false : true,
     runningMode: process.env.MODE || '',
     bodySizeLimit: process.env.BODY_SIZE_LIMIT || '50mb',
-    chapApiUrl: process.env.CHAP_API_URL,
+    chapCliApiUrl: process.env.CHAP_CLI_API_URL,
     minio: {
       endPoint: process.env.MINIO_ENDPOINT || 'localhost',
       port: process.env.MINIO_PORT ? parseInt(process.env.MINIO_PORT) : 9000,

src/routes/index.ts

@@ -22,6 +22,7 @@ import FormData from 'form-data';
 import { createClient } from '@clickhouse/client';
 import { ModelPredictionUsingChap } from '../services/ModelPredictionUsingChap';
 import { createHistoricalDiseaseTable, insertHistoricDiseaseData } from '../utils/clickhouse';
+import { createOrganizationsTable, insertOrganizationIntoTable } from '../utils/clickhouse';
 
 // Constants
 const VALID_MIME_TYPES = ['text/csv', 'application/json'] as const;
@@ -130,12 +131,11 @@ const handleJsonFile = async (
     throw error;
   }
 };
+function sanitizeTableName(tableName: string): string {
+  return tableName.replace(/[^a-zA-Z0-9_-]/g, '_');
+}
 
-const handleJsonPayload = async (
-  file: Express.Multer.File,
-  json: Object,
-  bucket: string
-): Promise<UploadResponse> => {
+const handleJsonPayload = async (file: Express.Multer.File, json: Object, bucket: string): Promise<UploadResponse> => {
   try {
     const uploadResult = await uploadFileBufferToMinio(
       Buffer.from(JSON.stringify(json)),
@@ -144,6 +144,12 @@ const handleJsonPayload = async (
       file.mimetype
     );
 
+    const tableNameOrganizations = sanitizeTableName(file.originalname) + '_organizations_' + (new Date().getMilliseconds());
+
+    await createOrganizationsTable(tableNameOrganizations);
+
+    await insertOrganizationIntoTable(tableNameOrganizations, file.buffer.toString());
+
     return uploadResult.success
       ? createSuccessResponse('UPLOAD_SUCCESS', uploadResult.message)
       : createErrorResponse('UPLOAD_FAILED', uploadResult.message);
@@ -284,7 +290,7 @@ async function getPrediction(
   bucket: string
 ) {
   try {
-    const { chapApiUrl } = getConfig();
+    const { chapCliApiUrl: chapApiUrl } = getConfig();
     const { url, password } = getConfig().clickhouse;
     const client = createClient({
       url,
@@ -363,13 +369,17 @@ routes.post('/predict', upload.single('file'), async (req, res) => {
         }, 250);
       })) as any;
 
+      // get organization code
+      const orgCode = JSON.parse(file.buffer.toString())?.orgUnitsGeoJson.features[0].properties.code;
+
       const bucketName = sanitizeBucketName(
         `${file.originalname.split('.')[0]}-${Math.round(new Date().getTime() / 1000)}`
       );
 
       const predictionResultsForMinio = predictionResults?.dataValues?.map((d: any) => {
         return {
           ...d,
+          orgCode: orgCode ?? undefined,
           diseaseId: predictionResults.diseaseId as string,
         };
       });

src/utils/clickhouse.ts

@@ -253,3 +253,140 @@ export async function insertHistoricDiseaseData(
   }
   return client.close();
 }
+function isNumber(value: unknown): value is number {
+  return typeof value === "number" && Number.isFinite(value);
+}
+
+function checkType(feature: any):
+  {type: 'point', latitude: number, longitude: number} |
+  {type: 'polygon', coordinates: [[number, number]]}
+{
+  const type = feature?.geometry?.type?.toLowerCase();
+
+  if (type == 'point') {
+    if (Array.isArray(feature?.geometry?.coordinates) &&
+        feature.geometry.coordinates.every(isNumber)) {
+      return {
+        type: 'point',
+        latitude: feature.geometry.coordinates[0],
+        longitude: feature.geometry.coordinates[1],
+      };
+    }
+  }
+
+  if (type == 'polygon') {
+    if (Array.isArray(feature?.geometry?.coordinates?.[0]) &&
+        feature.geometry.coordinates?.[0].every(Array.isArray)) {
+      return {
+        type : 'polygon',
+        coordinates: feature.geometry.coordinates[0] as any,
+      };
+    }
+  }
+
+  throw new Error('Invalid geometry type. ' + JSON.stringify(feature));
+}
+
+export async function insertOrganizationIntoTable(
+  tableName: string,
+  payload: string,
+) {
+  const client = createClient({
+    url,
+    password,
+  });
+
+  const normalizedTableName = tableName.replace(/-/g, '_');
+
+  logger.info(`Inserting data into ${normalizedTableName}`);
+
+  try {
+    const json = JSON.parse(payload);
+
+    const values = json.orgUnitsGeoJson.features.map((feature: any) => {
+      const type = checkType(feature);
+
+      return {
+        code: feature.properties.code,
+        name: feature.properties.name,
+        level: feature.properties.level,
+        type: type.type,
+        latitude: type.type == 'point' ? type.latitude : null,
+        longitude: type.type == 'point' ? type.longitude : null,
+        coordinates: type.type == 'polygon' ? type.coordinates : null,
+      };
+    });
+
+    await client.insert({
+      table: 'default.' + normalizedTableName,
+      values,
+      format: 'JSONEachRow',
+    })
+
+    logger.info(`Successfully inserted data into ${normalizedTableName}`);
+    return true;
+  } catch (error) {
+    logger.error('Error inserting data from JSON');
+    logger.error(error);
+    return false;
+  } finally {
+    await client.close();
+  }
+}
+
+export async function createOrganizationsTable(
+  tableName: string,
+) {
+  const normalizedTableName = tableName.replace(/-/g, '_');
+
+  logger.info(`Creating Organizations table from JSON ${normalizedTableName}`);
+
+  const client = createClient({
+    url,
+    password,
+  });
+
+  //check if the table exists
+  try {
+    const existsResult = await client.query({
+      query: `desc ${normalizedTableName}`,
+    });
+    logger.info(`Table ${normalizedTableName} already exists`);
+    await client.close();
+    return false;
+  } catch (error) {
+  }
+
+  try {
+
+    const query = `
+      CREATE TABLE IF NOT EXISTS \`default\`.${normalizedTableName}
+      ( code String,
+       name String,
+       level String,
+       type String,
+       latitude Float32,
+       longitude Float32,
+       coordinates Array(Array(Float32))
+      )
+      ENGINE = MergeTree
+      ORDER BY code
+    `;
+
+    logger.info(query);
+
+    const res = await client.query({ query });
+
+    logger.info(`Successfully created table from JSON ${normalizedTableName}`);
+    logger.info(res);
+
+    await client.close();
+
+    return true;
+  } catch (err) {
+    logger.error(`Error creating table from JSON ${normalizedTableName}`);
+    logger.error(err);
+    return false;
+  }
+
+}

src/utils/minioClient.ts

@@ -4,10 +4,12 @@ import * as Minio from 'minio';
 import { getConfig } from '../config/config';
 import logger from '../logger';
 import {
+  createOrganizationsTable,
   createTable,
   createTableFromJson,
   insertFromS3,
   insertFromS3Json,
+  insertOrganizationIntoTable,
 } from './clickhouse';
 import { getCsvHeaders, validateJsonFile } from './file-validators';