Merge branch 'develop' of github.com:hashgraph/guardian into develop

.gitignore

@@ -17,13 +17,10 @@ tsconfig.tsbuildinfo
 .pnp.*
 node_modules
 runtime-data/
-<<<<<<< HEAD
-downloads
-test.env.json
-=======
-000_*
->>>>>>> origin/automation-api-tests
 
 # OSX related metadata removal and IDE nonsense
 .DS_store
 .idea
+
+logs/
+**/logs/

Methodology Library/Verra/readme.md

@@ -33,4 +33,4 @@ Methodologies often refer to modules or tools, which include specific methodolog
 | VerraARR | 1707209648.628432394 |[Link](https://github.com/hashgraph/guardian/blob/main/Methodology%20Library/Verra/VerraARR/Verra%20ARR.policy) |
 | PWRM0001 | 1707206885.406272003 |[Link](https://github.com/hashgraph/guardian/blob/main/Methodology%20Library/Verra/PWRM0001/PWRM0001.policy) |
 | PWRM0002 | 1721746888.834099991 |[Link](https://github.com/hashgraph/guardian/blob/main/Methodology%20Library/Verra/PWRM0002/PWRM0002.policy) |
-| VMR0006 | - |[Link](https://github.com/hashgraph/guardian/blob/main/Methodology%20Library/Verra/VMR0006/VMR0006.policy) |
+| VMR0006 | 1721826322.833452372 |[Link](https://github.com/hashgraph/guardian/blob/main/Methodology%20Library/Verra/VMR0006/VMR0006.policy) |

README.md

@@ -266,6 +266,7 @@ If you want to manually build every component with debug information, then build
 * [Yarn](https://classic.yarnpkg.com/lang/en/docs/install/#mac-stable)
 * [Nats 1.12.2](https://nats.io/)
 * [Redict 7.3.0](https://redict.io/)
+* [Seq 2024.3 - optional for logging](https://datalust.co/seq)
 
 ### Build and start each component
 

ai-service/configs/.env.ai-service

@@ -4,4 +4,20 @@ OPENAI_API_KEY=...
 GPT_VERSION=gpt-3.5-turbo
 VECTOR_STORAGE_PATH=./faiss-vector
 DOCS_STORAGE_PATH=./data/generated-data
-MQ_ADDRESS="localhost"
+MQ_ADDRESS="localhost"
+
+#PINO_LOGGER
+TRANSPORTS="CONSOLE, MONGO"
+DB_LOGGER_NAME="logger_db"
+DB_LOGGER_HOST="localhost"
+DB_LOGGER_COLLECTION="log"
+LOG_FILE_PATH="./logs/app.log"
+LOG_LEVEL="info"
+SEQ_SERVER_URL="http://localhost:5341"
+SEQ_UI_URL="http://localhost:5341"
+SEQ_API_KEY=""
+
+#MONGO_INIT
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

ai-service/configs/.env.ai-service.develop

@@ -4,4 +4,20 @@ OPENAI_API_KEY=...
 GPT_VERSION=gpt-3.5-turbo
 VECTOR_STORAGE_PATH=./faiss-vector
 DOCS_STORAGE_PATH=./data/generated-data
-MQ_ADDRESS="localhost"
+MQ_ADDRESS="localhost"
+
+#PINO_LOGGER
+TRANSPORTS="CONSOLE, MONGO"
+DB_LOGGER_NAME="logger_db"
+DB_LOGGER_HOST="localhost"
+DB_LOGGER_COLLECTION="log"
+LOG_FILE_PATH="./logs/app.log"
+LOG_LEVEL="info"
+SEQ_SERVER_URL="http://localhost:5341"
+SEQ_UI_URL="http://localhost:5341"
+SEQ_API_KEY=""
+
+#MONGO_INIT
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

ai-service/configs/.env.ai-service.template

@@ -3,4 +3,20 @@ SERVICE_CHANNEL="ai-service"
 OPENAI_API_KEY=...
 GPT_VERSION=gpt-3.5-turbo
 VECTOR_STORAGE_PATH=./faiss-vector
-DOCS_STORAGE_PATH=./data/generated-data
+DOCS_STORAGE_PATH=./data/generated-data
+
+#PINO_LOGGER
+TRANSPORTS="CONSOLE, MONGO"
+DB_LOGGER_NAME=""
+DB_LOGGER_HOST=""
+DB_LOGGER_COLLECTION=""
+LOG_FILE_PATH=""
+LOG_LEVEL=""
+SEQ_SERVER_URL="http://localhost:5341"
+SEQ_UI_URL="http://localhost:5341"
+SEQ_API_KEY=""
+
+#MONGO_INIT
+MIN_POOL_SIZE=""
+MAX_POOL_SIZE=""
+MAX_IDLE_TIME_MS=""

ai-service/package.json

@@ -21,6 +21,9 @@
     "typescript": "^5.2.2",
     "rxjs": "^7.8.1"
   },
+  "imports": {
+    "#constants": "./dist/constants/index.js"
+  },
   "scripts": {
     "build": "tsc",
     "build:prod": "tsc --project tsconfig.production.json",

ai-service/src/ai-manager.ts

@@ -7,7 +7,7 @@ import { VectorStorage } from './helpers/vector-storage-helper.js';
 import { AISuggestionsDB } from './helpers/ai-suggestions-db.js';
 import { PolicyDescription } from './models/models.js';
 import * as dotenv from 'dotenv';
-import { Logger, Policy, PolicyCategory } from '@guardian/common';
+import { PinoLogger, Policy, PolicyCategory } from '@guardian/common';
 
 dotenv.config();
 
@@ -20,9 +20,9 @@ export class AIManager {
     chain: RetrievalQAChain | null;
     vector: FaissStore | null;
     model: OpenAI;
-    policyDescriptions: PolicyDescription[]
+    policyDescriptions: PolicyDescription[];
 
-    constructor() {
+    constructor(private readonly logger: PinoLogger) {
         this.docPath = process.env.DOCS_STORAGE_PATH || './data/generated-data';
         this.versionGPT = process.env.GPT_VERSION || 'gpt-3.5-turbo';
         this.vectorPath = process.env.VECTOR_STORAGE_PATH || './faiss-vector';
@@ -56,32 +56,32 @@ export class AIManager {
 
     async rebuildVector() {
         try {
-            new Logger().info('rebuild vector', ['AI_SERVICE']);
+            await this.logger.info('rebuild vector', ['AI_SERVICE']);
 
             this.vector = null;
             this.chain = null;
 
             await this.loadDBData();
-            await FilesManager.generateData(this.docPath, this.policies, this.categories, this.policyDescriptions);
-            await VectorStorage.create(this.docPath, this.vectorPath);
+            await FilesManager.generateData(this.docPath, this.policies, this.categories, this.policyDescriptions, this.logger);
+            await VectorStorage.create(this.docPath, this.vectorPath, this.logger);
 
-            new Logger().info('end rebuild vector', ['AI_SERVICE']);
+            await this.logger.info('end rebuild vector', ['AI_SERVICE']);
         } catch (e) {
-            new Logger().error(e.message, ['AI_SERVICE']);
+            await this.logger.error(e.message, ['AI_SERVICE']);
         }
     }
 
     async loadDBData() {
         const dbRequests = new AISuggestionsDB();
 
         this.categories = await dbRequests.getPolicyCategories();
-        new Logger().info('fetched categories', ['AI_SERVICE']);
+        await this.logger.info('fetched categories', ['AI_SERVICE']);
 
         this.policies = await dbRequests.getAllPolicies();
-        new Logger().info('fetched policies', ['AI_SERVICE']);
+        await this.logger.info('fetched policies', ['AI_SERVICE']);
 
         this.policyDescriptions = await dbRequests.getFieldDescriptions(this.policies);
-        new Logger().info('fetched fields descriptions', ['AI_SERVICE']);
+        await this.logger.info('fetched fields descriptions', ['AI_SERVICE']);
 
     }
 }

ai-service/src/api/service/ai-suggestions-service.ts

@@ -1,12 +1,12 @@
 import { ApiResponse } from '../../helpers/api-response.js';
 import { AIManager } from '../../ai-manager.js';
 import { MessageAPI } from '@guardian/interfaces';
-import { MessageError, MessageResponse } from '@guardian/common';
+import { MessageError, MessageResponse, PinoLogger } from '@guardian/common';
 
 /**
  * Connect to the message broker methods of working with artifacts.
  */
-export async function aiSuggestionsAPI(aiManager: AIManager): Promise<void> {
+export async function aiSuggestionsAPI(aiManager: AIManager, logger: PinoLogger): Promise<void> {
     /**
      * AI Suggestions
      *
@@ -24,7 +24,7 @@ export async function aiSuggestionsAPI(aiManager: AIManager): Promise<void> {
             if (aiManager.vector !== null && aiManager.chain !== null) {
                 result = await aiManager.ask(msg.question);
             } else {
-                aiManager = new AIManager();
+                aiManager = new AIManager(logger);
                 await aiManager.rebuildVector();
 
                 result = await aiManager.ask(msg.question);
@@ -38,7 +38,7 @@ export async function aiSuggestionsAPI(aiManager: AIManager): Promise<void> {
 
     ApiResponse(MessageAPI.VECTOR_REBUILD, async () => {
         try {
-            aiManager = new AIManager();
+            aiManager = new AIManager(logger);
             await aiManager.rebuildVector();
 
             return new MessageResponse(true);

ai-service/src/app.ts

@@ -2,27 +2,29 @@ import { AISuggestionService } from './helpers/suggestions.js';
 import { aiSuggestionsAPI } from './api/service/ai-suggestions-service.js';
 import { AISuggestionsDB } from './helpers/ai-suggestions-db.js';
 import { AIManager } from './ai-manager.js';
-import { ApplicationState, Logger, MessageBrokerChannel } from '@guardian/common';
+import { ApplicationState, MessageBrokerChannel, mongoForLoggingInitialization, PinoLogger, pinoLoggerInitialization } from '@guardian/common';
 import * as process from 'process';
 import { ApplicationStates } from '@guardian/interfaces';
 
 Promise.all([
-    MessageBrokerChannel.connect('AI_SERVICE')
+    MessageBrokerChannel.connect('AI_SERVICE'),
+    mongoForLoggingInitialization()
 ]).then(async values => {
-    const [cn] = values;
+    const [cn, loggerMongo] = values;
+
+    const logger: PinoLogger = pinoLoggerInitialization(loggerMongo);
 
     const state = new ApplicationState();
     await state.setServiceName('AI_SERVICE').setConnection(cn).init();
 
-    state.updateState(ApplicationStates.INITIALIZING);
-    new Logger().setConnection(cn);
+    await state.updateState(ApplicationStates.INITIALIZING);
     await new AISuggestionService().setConnection(cn).init();
     await new AISuggestionsDB().setConnection(cn).init();
     try {
-        const aiManager = new AIManager();
-        await aiSuggestionsAPI(aiManager);
-        state.updateState(ApplicationStates.READY);
-        new Logger().info('Ai service started', ['AI_SERVICE']);
+        const aiManager = new AIManager(logger);
+        await aiSuggestionsAPI(aiManager, logger);
+        await state.updateState(ApplicationStates.READY);
+        await logger.info('Ai service started', ['AI_SERVICE']);
     } catch (error) {
         console.log(error);
         console.error(error);

ai-service/src/constants/index.ts

@@ -0,0 +1 @@
+export { DEFAULT as DEFAULT_MONGO } from './mongo.js';

ai-service/src/constants/mongo.ts

@@ -0,0 +1,5 @@
+export const DEFAULT = {
+    MIN_POOL_SIZE: '1',
+    MAX_POOL_SIZE: '5',
+    MAX_IDLE_TIME_MS: '30000',
+};

ai-service/src/helpers/files-manager-helper.ts

@@ -4,27 +4,27 @@ import * as path from 'path';
 import { GroupCategories } from './general-helper.js';
 import { PolicyDescription } from '../models/models.js';
 import { PolicyCategoryType } from '@guardian/interfaces';
-import { Logger, Policy, PolicyCategory } from '@guardian/common';
+import { PinoLogger, Policy, PolicyCategory } from '@guardian/common';
 
 const MIN_DESCRIPTION_WORDS = 5;
 
 export class FilesManager {
 
-    static async generateData(dirPath: string, policies: Policy[], categories: PolicyCategory[], policyDescriptions: PolicyDescription[]): Promise<boolean> {
+    static async generateData(dirPath: string, policies: Policy[], categories: PolicyCategory[], policyDescriptions: PolicyDescription[], logger: PinoLogger): Promise<boolean> {
         FilesManager.checkDir(dirPath);
         FilesManager.deleteAllFilesInDirectory(dirPath);
 
         try {
-            await FilesManager.generateMethodologyFiles(dirPath, policies, categories, policyDescriptions);
-            await FilesManager.generateMetadataFile(dirPath, policies, categories);
+            await FilesManager.generateMethodologyFiles(dirPath, policies, categories, policyDescriptions, logger);
+            await FilesManager.generateMetadataFile(dirPath, policies, categories, logger);
             return true;
         } catch (e) {
             console.log(e);
             return false;
         }
     }
 
-    static async generateMethodologyFiles(dirPath: string, policies: Policy[], categories: PolicyCategory[], policyDescriptions: PolicyDescription[]) {
+    static async generateMethodologyFiles(dirPath: string, policies: Policy[], categories: PolicyCategory[], policyDescriptions: PolicyDescription[], logger: PinoLogger) {
         if (!policies) {
             return false;
         }
@@ -38,17 +38,17 @@ export class FilesManager {
             const content = FilesManager.getFileData(policy, categories, descriptions);
 
             if (content) {
-                await FilesManager.generateFile(filePath, content);
+                await FilesManager.generateFile(filePath, content, logger);
             }
         }
     }
 
-    static async generateMetadataFile(dirPath: string, policies: Policy[], categories: PolicyCategory[]) {
+    static async generateMetadataFile(dirPath: string, policies: Policy[], categories: PolicyCategory[], logger: PinoLogger) {
         const content = FilesManager.getMetadataContent(policies, categories);
 
         if (content) {
             const fileName = `${dirPath}/metadata.txt`;
-            await FilesManager.generateFile(fileName, content);
+            await FilesManager.generateFile(fileName, content, logger);
         }
     }
 
@@ -159,14 +159,14 @@ export class FilesManager {
         return '';
     }
 
-    static async generateFile(filePath: string, content: string) {
+    static async generateFile(filePath: string, content: string, logger: PinoLogger) {
         return new Promise((resolve, reject) => {
-            fs.writeFile(filePath, content, (err) => {
+            fs.writeFile(filePath, content,  async (err) => {
                 if (err) {
                     console.error(err);
                     reject();
                 } else {
-                    new Logger().info(`File ${filePath} was created`, ['AI_SERVICE']);
+                    await logger.info(`File ${filePath} was created`, ['AI_SERVICE']);
                     resolve(true);
                 }
             });

ai-service/src/helpers/vector-storage-helper.ts

@@ -3,7 +3,8 @@ import { FaissStore } from 'langchain/vectorstores/faiss';
 import { DirectoryLoader } from 'langchain/document_loaders/fs/directory';
 import { TextLoader } from 'langchain/document_loaders/fs/text';
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
-import { Logger } from '@guardian/common';
+import { PinoLogger } from '@guardian/common';
+
 
 export class VectorStorage {
 
@@ -13,7 +14,7 @@ export class VectorStorage {
         return vectorData;
     }
 
-    static async create(docPath: string, vectorPath: string) {
+    static async create(docPath: string, vectorPath: string, logger: PinoLogger): Promise<void> {
         try {
             if (docPath && vectorPath) {
                 const textLoader = new DirectoryLoader(docPath, {
@@ -33,14 +34,14 @@ export class VectorStorage {
                     const vectorstore = await FaissStore.fromDocuments(documents, embeddings);
                     await vectorstore.save(vectorPath);
 
-                    new Logger().info('vector has been successfully created', ['AI_SERVICE']);
+                   await logger.info('vector has been successfully created', ['AI_SERVICE']);
                 } else {
-                    new Logger().warn('there is no data for vector creation', ['AI_SERVICE']);
+                   await logger.warn('there is no data for vector creation', ['AI_SERVICE']);
                 }
 
             }
         } catch (e) {
-            new Logger().error(e.message, ['AI_SERVICE']);
+           await logger.error(e.message, ['AI_SERVICE']);
         }
     }
 }

ai-service/tsconfig.json

@@ -13,7 +13,10 @@
         "module": "ESNext",
         "moduleResolution": "node",
         "outDir": "dist/",
-        "target": "es2022"
+        "target": "es2022",
+        "paths": {
+          "#constants/*": ["./src/constants/*"]
+        }
     },
     "exclude": [
         "node_modules",

ai-service/tsconfig.production.json

@@ -13,7 +13,10 @@
         "module": "ESNext",
         "moduleResolution": "node",
         "outDir": "dist/",
-        "target": "es2022"
+        "target": "es2022",
+        "paths": {
+          "#constants/*": ["./src/constants/*"]
+        }
     },
     "exclude": [
         "node_modules",