feat: Improve Brownie modules (#37)

* feat: Improve Brownie modules

* chore: Improve documentation

brownie/.env.example

@@ -10,20 +10,25 @@ REGISTRY_URL=ksp
 #IMAGE_TAG=$(git rev-parse --short HEAD)
 IMAGE_TAG="dev"
 
-# POSTGRES
-PGHOST=$LOCAL_HOST_IP
-PGPORT=5432
-PGUSER=postgres
-PGPASSWORD=admin
-PGDATABASE=postgres
+SERVER1_PGHOST=$LOCAL_HOST_IP
+SERVER1_PGPORT=5432
+SERVER1_PGUSER=postgres
+SERVER1_PGPASSWORD=admin
+SERVER1_PGDATABASE=postgres
+
+SERVER2_PGHOST=$LOCAL_HOST_IP
+SERVER2_PGPORT=5432
+SERVER2_PGUSER=postgres
+SERVER2_PGPASSWORD=admin
+SERVER2_PGDATABASE=postgres
 
 #KAFKA
 KAFKA_BROKERS="$LOCAL_HOST_IP:9092"
 KAFKA_CLIENT_ID="$K8S_NAMESPACE:$SERVICE_NAME"
 KAFKA_USERNAME=admin
 KAFKA_PASSWORD=admin
 
-ENABLED_MODULES=postgres,kafka
+ENABLED_MODULES="postgresql=server1;server2,kafka"
 BROWNIE_NODE_OPTIONS=--max-heap-size=256
 # Pattern with group wich extracts timestamp made of 14 digits: yyyyMMddHHmmss prefixed with "ts"
 TIMESTAMP_PATTRN="^.*pit.*_(ts\d{14,14}).*"

brownie/README.md

@@ -81,7 +81,44 @@ Brownie does not enforce any specific naming strategy for your resources other t
 
 ### Supported modules
 
-Brownie supports several modules. `--enabled-modules` parameter can beused to selectively enable them. By default all modules are disabled. Example: `--enabled-modules mod1,mod2` enables both "mod1" and "mod2".
+Brownie supports several modules. `--enabled-modules` parameter can be used to selectively enable them. By default all modules are disabled. Example: `--enabled-modules mod1,mod2` enables both "mod1" and "mod2".
+
+The parameter `--enabled-modules` supports two formats - the simple and extended format. The simple format looks like this: `--enabled-modules mod1,mod2`. 
+
+The extended format is designed to allow module to monitor more than one resource server. For example, when we need to do cleanups in multiple database servers we have to enable the module as usual, and in addition, we have to supply configuration describing a) how many servers do we want to monitor and b) how to connect to those multiple servers. The extended format looks like this: `--enabled-modules mod1=server1;server2,mod2`. In this example we have provided the extended module format for module named "mod1". Here "mod1" was given instruction to monitor two resource servers named "server1" and "server2". These names are arbitrary. They are aliases allowing to group relevant configuration details together.
+Following this example, we can now review how we can pass the connection details for those two servers "server1", "server2". Lets assume that our "mod1" needs four options for connecting to the resource server: HOST,PORT,USER,PASS. These four options need to have different values for each of our two servers. So, to make `mod1` fully functional it is expected that developer will pass eight options in total. The idea is to prefix each option with the resource server name. The following example will enable `mod1` and pass relevant connection details:
+
+```
+--enabled-modules mod1=server1;server2,mod2 \
+\
+--server1-host=127.0.0.1 \
+--server1-port=3000 \
+--server1-user=user1 \
+--server1-pass=admin \
+\
+--server2-host=127.0.0.2 \
+--server2-port=3000 \
+--server2-user=user2 \
+--server2-pass=admin
+```
+
+The same config can we re-written to use environment variables instead:
+
+```
+export SERVER1_HOST=127.0.0.1
+export SERVER1_PORT=3000
+export SERVER1_USER=user1
+export SERVER1_PASS=admin
+
+export SERVER2_HOST=127.0.0.2
+export SERVER2_PORT=3000
+export SERVER2_USER=user2
+export SERVER2_PASS=admin
+
+--enabled-modules mod1=server1;server2,mod2
+```
+
+Please note that names such as "mod1" and "mod2" were chosen only to simplify the explanation. Brownie does not recognise these module names. Below is the list of module names currently supported.
 
 *Postgresql*
 

brownie/src/bootstrap.ts

@@ -1,33 +1,11 @@
 import { logger } from "./logger.js"
-import { Config, PgConfig, KafkaConfig } from  "./config.js"
 
-// This is visible for testing only
-export const fnReadValue = (params: Map<string, string>, param: string, isRequired?: boolean, defaultValue?: any): any | undefined => {
-  const value = params.get(param)
-  if (value === undefined || (`${ value }`.trim().length == 0)) {
-    const envName = param.replaceAll("--", "").replaceAll("-", "_").toUpperCase()
-    logger.debug("fnReadValue(): Cannot find parameter '%s'. Reading environment variable: %s", param, envName)
+import * as cfg from  "./config.js"
+import { PgConfig } from "./modules/pg/config.js"
+import { KafkaConfig } from "./modules/kafka/config.js"
+import { readParameterValue } from "./utils.js"
 
-    const envValue = process.env[envName]
-    if (!envValue && !defaultValue && isRequired) throw new Error(`Missing required parameter "${ param }" or env variable: ${ envName }`)
-    if (envValue) {
-      logger.info("fnReadValue(): Parameter '%s' was loaded from env.%s", param, envName)
-      return envValue
-    }
-
-    if (!isRequired) {
-      logger.info("fnReadValue(): Parameter '%s' is optional, skipping...", param)
-      return undefined
-    }
-
-    logger.info("fnReadValue(): Cannot find env variable '%s'. Fallback to default value: '%s'", envName, defaultValue)
-    return defaultValue
-  }
-
-  return value
-}
-
-export const readParams = (): Config => {
+export const readParams = (): cfg.Config => {
   logger.debug("readParams()... ARGS \n%s", JSON.stringify(process.argv, null, 2))
   logger.debug("readParams()... ENV: \n%s", JSON.stringify(process.env, null, 2))
 
@@ -42,36 +20,24 @@ export const readParams = (): Config => {
 
   logger.info("Application started with arguments: \n%s", JSON.stringify(Object.fromEntries(params), null, 2))
 
-  const enabledModules = fnReadValue(params, Config.PARAM_ENABLED_MODULES)
-  let pgConfig: PgConfig | null = null
-  if (Config.isModuleEnabled(enabledModules, PgConfig.MODULE_NAME)) {
-    pgConfig = new PgConfig(
-      fnReadValue(params, PgConfig.PARAM_PGHOST),
-      fnReadValue(params, PgConfig.PARAM_PGPORT),
-      fnReadValue(params, PgConfig.PARAM_PGDATABASE),
-      fnReadValue(params, PgConfig.PARAM_PGUSER),
-      fnReadValue(params, PgConfig.PARAM_PGPASSWORD)
-    )
+  const enabledModules: Map<string, cfg.ModuleConfig> = cfg.parseModules(readParameterValue(params, cfg.Config.PARAM_ENABLED_MODULES, true))
+  let pgModules = new Map<string, PgConfig>()
+  let kafkaModules = new Map<string, KafkaConfig>()
+
+  if (enabledModules.has(PgConfig.MODULE_NAME)) {
+    pgModules = PgConfig.loadAll(enabledModules.get(PgConfig.MODULE_NAME), params, readParameterValue)
   }
 
-  let kafkaConfig: KafkaConfig | null = null
-  if (Config.isModuleEnabled(enabledModules, KafkaConfig.MODULE_NAME)) {
-    kafkaConfig = new KafkaConfig(
-      fnReadValue(params, KafkaConfig.PARAM_BROKERS),
-      fnReadValue(params, KafkaConfig.PARAM_CLIENT_ID),
-      fnReadValue(params, KafkaConfig.PARAM_USERNAME, false),
-      fnReadValue(params, KafkaConfig.PARAM_PASSWORD, false),
-      fnReadValue(params, KafkaConfig.PARAM_PORT, false),
-      fnReadValue(params, KafkaConfig.PARAM_SASL_MECHANISM, false)
-    )
+  if (enabledModules.has(KafkaConfig.MODULE_NAME)) {
+    kafkaModules = KafkaConfig.loadAll(enabledModules.get(KafkaConfig.MODULE_NAME), params, readParameterValue)
   }
 
-  return new Config(
+  return new cfg.Config(
     enabledModules,
-    pgConfig,
-    kafkaConfig,
-    new RegExp(fnReadValue(params, Config.PARAM_TIMESTAMP_PATTERN, true, Config.DEFAULT_TIMESTAMP_PATTERN)),
-    Config.parseRetention(fnReadValue(params, Config.PARAM_RETENTION_PERIOD, true, Config.DEFAULT_RETENTION_PERIOD)),
-    fnReadValue(params, Config.PARAM_DRY_RUN, false, false) === "true"
+    pgModules,
+    kafkaModules,
+    new RegExp(readParameterValue(params, cfg.Config.PARAM_TIMESTAMP_PATTERN, true, cfg.Config.DEFAULT_TIMESTAMP_PATTERN)),
+    cfg.Config.parseRetention(readParameterValue(params, cfg.Config.PARAM_RETENTION_PERIOD, true, cfg.Config.DEFAULT_RETENTION_PERIOD)),
+    readParameterValue(params, cfg.Config.PARAM_DRY_RUN, false, false) === "true"
   )
 }

brownie/src/config.ts

@@ -1,83 +1,35 @@
-import { logger } from "./logger.js"
+import { KafkaConfig } from "./modules/kafka/config.js"
+import { PgConfig } from "./modules/pg/config.js"
 
-export class PgConfig {
-  static MODULE_NAME: string = "postgresql"
-
-  static PARAM_PGHOST: string = "--pghost"
-  static PARAM_PGPORT: string = "--pgport"
-  static PARAM_PGDATABASE: string = "--pgdatabase"
-  static PARAM_PGUSER: string = "--pguser"
-  static PARAM_PGPASSWORD: string = "--pgpassword"
-
-  readonly port: number
-
-  constructor(
-    readonly host: string,
-    private readonly portAsText: string,
-    readonly database: string,
-    readonly username: string,
-    readonly password: string
-  ) {
-    this.port = parseInt(portAsText)
-    if (isNaN(this.port)) throw new Error(`The port should be a number: ${ portAsText }`)
-  }
-}
-
-export class KafkaConfig {
-  static MODULE_NAME: string = "kafka"
-
-  static PARAM_BROKERS: string = "--kafka-brokers"
-  static PARAM_PORT: string = "--kafka-port"
-  static PARAM_CLIENT_ID: string = "--kafka-client-id"
-  static PARAM_USERNAME: string = "--kafka-username"
-  static PARAM_PASSWORD: string = "--kafka-password"
-  static PARAM_SASL_MECHANISM: string = "--kafka-sasl-mechanism"
-
-  readonly brokers: Array<string>
-
-  constructor(
-    private readonly brokersCsv: string,
-    readonly clientId: string,
-    readonly username: string,
-    readonly password: string,
-    private readonly portAsText?: string,
-    readonly saslMechanism?: string,
-  ) {
-    const hosts = brokersCsv
-      .split(",")
-      .map(it => it.trim())
-
-    const parsed = new Array<string>()
-    for (let host of hosts) {
-      if (host.indexOf(":") === -1) {
-        if (!portAsText || (typeof(portAsText) === 'string' && (portAsText.trim().length === 0 || isNaN(parseInt(portAsText))))) {
-          throw Error(`The broker host should be given with port or default port should be provided. Cannot use: "${ host }" without port`)
-        }
-        host = `${ host }:${ portAsText }`
-      }
-      parsed.push(host)
-    }
-    this.brokers = parsed
-  }
-}
+export type ValueReader = (params: Map<string, string>, param: string, isRequired?: boolean, defaultValue?: any) => any | undefined
 
 export class Config {
+  // The format for value can be simple or extended
+  // Simple format:   --enabled-modules postgres,kafka
+  // Extended formats: --enabled-modules postgres=pg1;pg2,kafka
+  //                   --enabled-modules postgres,kafka=k1;k2
+  //                   --enabled-modules postgres=pg1;pg2,kafka=k1;k2
   static PARAM_ENABLED_MODULES: string = "--enabled-modules"
+
   static PARAM_DRY_RUN: string = "--dry-run"
   static PARAM_RETENTION_PERIOD: string = "--retention-period"
   static PARAM_TIMESTAMP_PATTERN: string = "--timestamp-pattern"
   static DEFAULT_TIMESTAMP_PATTERN: RegExp = /^.*pit.*(ts[0-9]{14,14}).*$/
   static DEFAULT_RETENTION_PERIOD: string = "3days"
 
   constructor(
-    readonly enabledModules: string,
-    readonly pg: PgConfig | null,
-    readonly kafka: KafkaConfig | null,
+    readonly enabledModules: Map<string, ModuleConfig>,
+    readonly pgModules: Map<string, PgConfig>,
+    readonly kafkaModules: Map<string, KafkaConfig>,
     readonly timestampPattern: RegExp,
     readonly retentionMinutes: number,
     readonly dryRun: boolean
   ) { }
 
+  isModuleEnabled = (moduleName: string): boolean => {
+    return this.enabledModules.has(moduleName)
+  }
+
   static parseRetention = (value: string): number => {
     if (new RegExp(/^1day$/).test(value)) return 24 * 60
     if (new RegExp(/^\d{1,}days$/).test(value)) return parseInt(value.replaceAll("days", "")) * 24 * 60
@@ -90,8 +42,69 @@ export class Config {
 
     throw new Error(`Invalid format for retention. Expected "<digit><unit>", got: ${ value }`)
   }
+}
 
-  static isModuleEnabled = (modules: string, module: string): boolean => {
-    return (modules.indexOf(module) !== -1)
-  }  
-}
+export class ModuleConfig {
+  constructor(readonly name: string, readonly ids: Array<string>) {}
+}
+
+export const parseModules = (rawConfig: string): Map<string, ModuleConfig> => {
+  // rawConfig = "postgres=pg1;pg2;pg3,kafka=k1;k2;k3,elastic"
+  const supportedModules = [ PgConfig.MODULE_NAME, KafkaConfig.MODULE_NAME ]
+  const rawModules = rawConfig.split(",")
+  if (rawModules.length > 5) {
+    // this is very unlikely scenario
+    throw new Error(`Invalid format for modules. We only support up to 5 modules: ${rawConfig}`)
+  }
+
+  const parsedModules: Array<ModuleConfig> = rawModules
+    .map(v => v.trim())
+    .filter(v => v.length > 0)
+    .map((rawModConfig) => {
+      if (rawModConfig.indexOf("=") == -1) {
+        // rawModConfig = 'elastic'
+        // this is simple module
+        return new ModuleConfig(rawModConfig, [rawModConfig])
+      }
+
+      // rawModConfig = 'postgres=pg1;pg2;pg3'
+      // parse it into map of arrays
+      const nameAndIds = rawModConfig.split("=").map(v => v.trim()).filter(v => v.length > 0)
+      if ( nameAndIds.length != 2) {
+        // unbalanced...
+        throw new Error(`Invalid format for module config. The correct example is: modue=id1;id2;id3 Current value is: ${rawModConfig}`)
+      }
+
+      const name = nameAndIds[0]   // postgres
+      const rawIds = nameAndIds[1] // pg1;pg2;pg3
+
+      const ids = rawIds.split(";").map(v => v.trim()).filter(v => v.length > 0)
+      if (ids.length == 0) {
+        // unbalanced...
+        throw new Error(`Invalid format of module ids. The correct example is: modue=id1;id2;id3 Current value is: ${rawModConfig}`)
+      }
+
+      const unique = new Set<string>(ids)
+      if (unique.size < ids.length) throw new Error(`Invalid format of module ids. Values must be unique. The correct example is: modue=id1;id2;id3 Current value is: ${rawModConfig}`)
+
+      return new ModuleConfig(name, ids)
+    })
+
+  const unique = new Set<string>()
+  for (let moduleConfig of parsedModules.values()) {
+    if (supportedModules.indexOf(moduleConfig.name) == -1) {
+      throw new Error(`Unsupported module name: ${moduleConfig.name}. We only support the following modules: ${supportedModules}`)
+    }
+    if (unique.has(moduleConfig.name)) {
+      throw new Error(`Invalid format of module names. Values must be unique. The correct example is: "module1=id1;id2;id3, module2" Current value is: ${rawConfig}`)
+    }
+
+    unique.add(moduleConfig.name)
+  }
+
+  const result = new Map<string, ModuleConfig>()
+  for (let module of parsedModules) {
+    result.set(module.name, module)
+  }
+  return result
+}