diff --git a/Kexa/main.ts b/Kexa/main.ts index 2171bf2e..9253a8ba 100644 --- a/Kexa/main.ts +++ b/Kexa/main.ts @@ -46,8 +46,7 @@ export async function main() { let settings = await gatheringRules(await getEnvVar("RULESDIRECTORY")??"./Kexa/rules"); context?.log("settings", settings); if(settings.length != 0){ - let resources = {}; - resources = await loadAddOns(resources); + let resources = await loadAddOns(settings); context?.log("resources", resources); if(args.o) createFileSync(JSON.stringify(resources), folderOutput + "/resources/"+ new Date().toISOString().slice(0, 16).replace(/[-T:/]/g, '') +".json", true); context?.log("good"); diff --git a/Kexa/models/settingFile/config.models.ts b/Kexa/models/settingFile/config.models.ts index d9776442..ec9f0777 100644 --- a/Kexa/models/settingFile/config.models.ts +++ b/Kexa/models/settingFile/config.models.ts @@ -27,4 +27,5 @@ export interface Config { name?: string; description?: string; prefix?: string; + ObjectNameNeed?: Array; } diff --git a/Kexa/services/addOn.service.ts b/Kexa/services/addOn.service.ts index c5580b37..2cba6c96 100644 --- a/Kexa/services/addOn.service.ts +++ b/Kexa/services/addOn.service.ts @@ -1,24 +1,26 @@ import { Provider, ProviderResource } from "../models/providerResource.models"; import { Header } from "../models/settingFile/header.models"; import { writeStringToJsonFile } from "../helpers/files" -const configuration = require('node-config-ts').config; +import { Capacity } from "../models/settingFile/capacity.models"; +import {getContext, getNewLogger} from "./logger.service"; +import { SettingFile } from "../models/settingFile/settingFile.models"; +const configuration = require('node-config-ts').config; const mainFolder = 'Kexa'; const serviceAddOnPath = './' + mainFolder + '/services/addOn'; const fs = require('fs'); - -import {getContext, getNewLogger} from "./logger.service"; -import { Capacity } from "../models/settingFile/capacity.models"; const logger = getNewLogger("LoaderAddOnLogger"); -export async function loadAddOns(resources: ProviderResource): Promise{ + +export async function loadAddOns(settings:SettingFile[]): Promise{ + let resources: ProviderResource = {}; let context = getContext(); logger.info("Loading addOns"); context?.log("Loading addOns"); const addOnNeed = require('../../config/addOnNeed.json'); const files = fs.readdirSync(serviceAddOnPath); const promises = files.map(async (file: string) => { - return await loadAddOn(file, addOnNeed); + return await loadAddOn(file, addOnNeed, settings); }); const results = await Promise.all(promises); results.forEach((result: { key: string; data: Provider[]; }) => { @@ -29,7 +31,7 @@ export async function loadAddOns(resources: ProviderResource): Promise { +async function loadAddOn(file: string, addOnNeed: any, settings:SettingFile[]): Promise<{ key: string; data: Provider|null; } | null> { let context = getContext(); try{ if (file.endsWith('Gathering.service.ts')){ @@ -43,6 +45,10 @@ async function loadAddOn(file: string, addOnNeed: any): Promise<{ key: string; d const { collectData } = await import(`./addOn/${file.replace(".ts", ".js") }`); let start = Date.now(); const addOnConfig = (configuration.hasOwnProperty(nameAddOn)) ? configuration[nameAddOn] : null; + addOnConfig?.forEach((config: any) => { + config.ObjectNameNeed = [] + config.rules.forEach((rulesName: string) => config.ObjectNameNeed = [...addOnNeed["objectNameNeed"][rulesName][nameAddOn], ...config.ObjectNameNeed]); + }); const data = await collectData(addOnConfig); let delta = Date.now() - start; context?.log(`AddOn ${nameAddOn} collect in ${delta}ms`); diff --git a/Kexa/services/addOn/awsGathering.service.ts b/Kexa/services/addOn/awsGathering.service.ts index cb4442b7..0ffde0e7 100644 --- a/Kexa/services/addOn/awsGathering.service.ts +++ b/Kexa/services/addOn/awsGathering.service.ts @@ -30,6 +30,7 @@ let rdsClient: RDS; let s3Client: S3; let ecsClient: ECS; let ecrClient: ECR; +let currentConfig: AwsConfig; //////////////////////////////////////////////////////////////////////////////////////////////////////// //// LISTING CLOUD RESOURCES /////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -37,6 +38,7 @@ export async function collectData(awsConfig: AwsConfig[]): Promise(); for (let oneConfig of awsConfig ?? []) { + currentConfig = oneConfig; let awsResource = { "ec2Instance": null, "ec2SG": null, @@ -169,6 +171,7 @@ function addRegion(resources:any, region:string) { } async function ec2SGListing(client: EC2, region: string): Promise { + if(!currentConfig.ObjectNameNeed?.includes("ec2SG")) return null; try { const data = await client.describeSecurityGroups().promise(); let jsonData = JSON.parse(JSON.stringify(data.SecurityGroups)); @@ -182,6 +185,7 @@ async function ec2SGListing(client: EC2, region: string): Promise { } async function ec2VolumesListing(client: EC2, region: string): Promise { + if(!currentConfig.ObjectNameNeed?.includes("ec2Volume")) return null; try { const data = await client.describeVolumes().promise(); let jsonData = JSON.parse(JSON.stringify(data.Volumes)); @@ -195,6 +199,7 @@ async function ec2VolumesListing(client: EC2, region: string): Promise { } async function ec2InstancesListing(client: EC2, region: string): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("ec2Instance")) return null; try { const data = await client.describeInstances().promise(); let jsonData = JSON.parse(JSON.stringify(data.Reservations)); @@ -208,6 +213,7 @@ async function ec2InstancesListing(client: EC2, region: string): Promise { + if(!currentConfig.ObjectNameNeed?.includes("rds")) return null; try { const data = await client.describeDBInstances().promise(); let jsonData = JSON.parse(JSON.stringify(data.DBInstances)); @@ -221,6 +227,7 @@ async function rdsInstancesListing(client: RDS, region: string): Promise { } async function resourceGroupsListing(client: ResourceGroups, region: string): Promise { + if(!currentConfig.ObjectNameNeed?.includes("resourceGroup")) return null; try { const data = await client.listGroups().promise(); let jsonData = JSON.parse(JSON.stringify(data.Groups)); @@ -234,6 +241,7 @@ async function resourceGroupsListing(client: ResourceGroups, region: string): Pr } async function tagsValueListing(client: ResourceGroupsTaggingAPI, region: string): Promise { + if(!currentConfig.ObjectNameNeed?.includes("tagsValue")) return null; try { interface TagParams {Key: string;} const dataKeys = await client.getTagKeys().promise(); @@ -255,6 +263,7 @@ async function tagsValueListing(client: ResourceGroupsTaggingAPI, region: string } async function s3BucketsListing(client: S3, region: string): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("s3")) return null; try { const data = await client.listBuckets().promise(); let jsonData = JSON.parse(JSON.stringify(data.Buckets)); @@ -268,6 +277,7 @@ async function s3BucketsListing(client: S3, region: string): Promise | } async function ecsClusterListing(client: ECS, region: string): Promise { + if(!currentConfig.ObjectNameNeed?.includes("ecsCluster")) return null; try { const data = await client.describeClusters().promise(); let jsonData = JSON.parse(JSON.stringify(data.clusters)); @@ -281,6 +291,7 @@ async function ecsClusterListing(client: ECS, region: string): Promise { } async function ecrImagesListing(client: ECR, region: string): Promise { + if(!currentConfig.ObjectNameNeed?.includes("ecrImage")) return null; try { const data = await client.describeRepositories().promise(); let jsonData = JSON.parse(JSON.stringify(data.repositories)); diff --git a/Kexa/services/addOn/azureGathering.service.ts b/Kexa/services/addOn/azureGathering.service.ts index 53e64600..77219646 100644 --- a/Kexa/services/addOn/azureGathering.service.ts +++ b/Kexa/services/addOn/azureGathering.service.ts @@ -41,6 +41,7 @@ const logger = getNewLogger("AzureLogger"); let computeClient: ComputeManagementClient; let resourcesClient : ResourceManagementClient ; let networkClient: NetworkManagementClient; +let currentConfig: AzureConfig; //////////////////////////////////////////////////////////////////////////////////////////////////////// //// LISTING CLOUD RESOURCES //////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -48,6 +49,7 @@ export async function collectData(azureConfig:AzureConfig[]): Promise(); for(let config of azureConfig??[]){ + currentConfig = config; let azureResource = { "vm": null, "rg": null, @@ -132,6 +134,7 @@ export async function collectData(azureConfig:AzureConfig[]): Promise { + if(!currentConfig.ObjectNameNeed?.includes("sp")) return null; const { GraphRbacManagementClient } = require("@azure/graph"); logger.info("starting getSPKeyInformation"); try { @@ -149,6 +152,7 @@ export async function getSPKeyInformation(credential: DefaultAzureCredential, su //ip list export async function ipListing(client:NetworkManagementClient): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("ip")) return null; logger.info("starting ipListing"); try{ const resultList = new Array; @@ -164,6 +168,7 @@ export async function ipListing(client:NetworkManagementClient): Promise { + if(!currentConfig.ObjectNameNeed?.includes("aks")) return null; logger.info("starting aksListing"); try{ const client = new ContainerServiceClient(credential, subscriptionId); @@ -180,6 +185,7 @@ export async function aksListing(credential: DefaultAzureCredential, subscriptio //network security group list export async function networkSecurityGroupListing(client:NetworkManagementClient): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("nsg")) return null; logger.info("starting networkSecurityGroupListing"); try { const resultList = new Array; @@ -196,6 +202,7 @@ export async function networkSecurityGroupListing(client:NetworkManagementClient //virtual network list export async function virtualNetworksListing(client:NetworkManagementClient): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("virtualNetwork")) return null; logger.info("starting virtualNetworksListing"); try { const resultList = new Array; @@ -212,6 +219,7 @@ export async function virtualNetworksListing(client:NetworkManagementClient): Pr //network list export async function networkInterfacesListing(client:NetworkManagementClient): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("networkInterfaces")) return null; logger.info("starting networkInterfacesListing"); try { const resultList = new Array; @@ -227,6 +235,7 @@ export async function networkInterfacesListing(client:NetworkManagementClient): //disks.list export async function disksListing(client:ComputeManagementClient): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("disk")) return null; logger.info("starting disksListing"); try { const resultList = new Array; @@ -242,6 +251,7 @@ export async function disksListing(client:ComputeManagementClient): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("vm")) return null; logger.info("starting virtualMachinesListing"); try { const resultList = new Array; @@ -256,6 +266,7 @@ export async function virtualMachinesListing(client:ComputeManagementClient): Pr } export async function resourceGroupListing(client:ResourceManagementClient): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("rg")) return null; logger.info("starting resourceGroupListing"); try { const resultList = new Array; @@ -269,7 +280,8 @@ export async function resourceGroupListing(client:ResourceManagementClient): Pro } } -export async function networkSecurityGroup_analyse(nsgList: Array): Promise|null> { +export async function networkSecurityGroup_analyze(nsgList: Array): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("nsg_analyze")) return null; try { const resultList = new Array; for await (let item of nsgList){ @@ -291,6 +303,12 @@ export async function networkSecurityGroup_analyse(nsgList: Array { + if( + !currentConfig.ObjectNameNeed?.includes("mlWorkspaces") + && !currentConfig.ObjectNameNeed?.includes("mlJobs") + && !currentConfig.ObjectNameNeed?.includes("mlComputes") + && !currentConfig.ObjectNameNeed?.includes("mlSchedules") + ) return null; logger.info("starting mlListing"); try{ const client = new AzureMachineLearningWorkspaces(credential, subscriptionId); @@ -322,6 +340,7 @@ export async function mlListing(credential: DefaultAzureCredential, subscription } export async function jobsListing(client: AzureMachineLearningWorkspaces, resourceGroupName: string, workspaceName: string): Promise { + if(!currentConfig.ObjectNameNeed?.includes("mlJobs")) return []; //logger.info("starting jobsListing"); try{ const resArray = new Array(); @@ -339,6 +358,7 @@ export async function jobsListing(client: AzureMachineLearningWorkspaces, resour } export async function computeOperationsListing(client: AzureMachineLearningWorkspaces, resourceGroupName: string, workspaceName: string): Promise { + if(!currentConfig.ObjectNameNeed?.includes("mlComputes")) return []; //logger.info("starting computeOperationsListing"); try{ const resArray = new Array(); @@ -356,6 +376,7 @@ export async function computeOperationsListing(client: AzureMachineLearningWorks } export async function schedulesListing(client: AzureMachineLearningWorkspaces, resourceGroupName: string, workspaceName: string): Promise { + if(!currentConfig.ObjectNameNeed?.includes("mlSchedules")) return []; //logger.info("starting schedulesListing"); try{ const resArray = new Array(); diff --git a/Kexa/services/addOn/gcpGathering.service.ts b/Kexa/services/addOn/gcpGathering.service.ts index 236055a8..0e006851 100644 --- a/Kexa/services/addOn/gcpGathering.service.ts +++ b/Kexa/services/addOn/gcpGathering.service.ts @@ -40,6 +40,7 @@ import { GcpConfig } from "../../models/gcp/config.models"; import {getContext, getNewLogger} from "../logger.service"; const logger = getNewLogger("GcpLogger"); +let currentConfig: GcpConfig; ///////////////////////////////////////// ////// LISTING CLOUD RESOURCES ///// @@ -51,6 +52,7 @@ export async function collectData(gcpConfig:GcpConfig[]): Promise(); let defaultPathCred = await getEnvVar("GOOGLE_APPLICATION_CREDENTIALS"); for (let config of gcpConfig??[]) { + currentConfig = config; setEnvVar("GOOGLE_APPLICATION_CREDENTIALS", "./config/gcp.json"); let prefix = config.prefix??(gcpConfig.indexOf(config).toString()); let gcpResources = { @@ -303,6 +305,7 @@ async function executeAllRegions(projectId: number, serviceFunction: Function, c const {CloudTasksClient} = require('@google-cloud/tasks').v2; async function listTasks(projectId: number, regionsList: Array): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("tasks_queue")) return null; let jsonData = []; try { const tasksClient = new CloudTasksClient(); @@ -316,6 +319,7 @@ async function listTasks(projectId: number, regionsList: Array): Promise const compute = require('@google-cloud/compute'); async function listAllComputes(projectId: string): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("compute")) return null; let jsonData = []; const instancesClient = new compute.InstancesClient(); @@ -336,6 +340,7 @@ async function listAllComputes(projectId: string): Promise|null> { } async function listSSHKey(projectId: string): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("compute_item")) return null; let jsonData = []; const instancesClient = new compute.InstancesClient(); @@ -353,6 +358,7 @@ async function listSSHKey(projectId: string): Promise|null> { return jsonData ?? null; } async function listPersistentDisks(projectId: any) { + if(!currentConfig.ObjectNameNeed?.includes("disk")) return null; let jsonData = []; const disksClient = new compute.DisksClient(); const aggListRequest = await disksClient.aggregatedListAsync({ @@ -372,6 +378,7 @@ async function listPersistentDisks(projectId: any) { } async function listAllBucket(): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("bucket")) return null; let jsonReturn = []; try { const storage = new Storage(); @@ -392,8 +399,10 @@ async function listAllBucket(): Promise|null> { import { ClusterManagerClient } from '@google-cloud/container'; import {CloudBillingClient} from "@google-cloud/billing"; import {VpcAccessServiceClient} from "@google-cloud/vpc-access"; +import { config } from "dotenv"; async function listAllClusters(): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("cluster")) return null; let jsonData = []; try { @@ -414,6 +423,7 @@ async function listAllClusters(): Promise|null> { const {ProjectsClient} = require('@google-cloud/resource-manager'); async function listAllProject(): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("project")) return null; let jsonData = []; try { @@ -430,6 +440,7 @@ async function listAllProject(): Promise|null> { } async function getBillingAccount(projectId: any): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("billingAccount")) return null; const {CloudBillingClient} = require('@google-cloud/billing'); let jsonData = []; @@ -449,6 +460,7 @@ async function getBillingAccount(projectId: any): Promise|null> { } async function listWorkflows(projectId: any): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("workflow")) return null; const {WorkflowsClient} = require('@google-cloud/workflows'); let jsonData = []; try { @@ -467,6 +479,7 @@ async function listWorkflows(projectId: any): Promise|null> { } async function listWebSecurityConfig(projectId: any): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("websecurity")) return null; const { WebSecurityScannerClient } = require('@google-cloud/web-security-scanner'); let jsonData = []; try { @@ -484,6 +497,7 @@ async function listWebSecurityConfig(projectId: any): Promise|null> { } async function listVpcConnectors(projectId: any, regionsList: Array): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("connector")) return null; const {VpcAccessServiceClient} = require('@google-cloud/vpc-access'); let jsonData = []; try { @@ -497,6 +511,7 @@ async function listVpcConnectors(projectId: any, regionsList: Array): Pr } async function listVMWareEngine(projectId: any): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("vmware_engine")) return null; const {VmwareEngineClient} = require('@google-cloud/vmwareengine').v1; let jsonData = []; const parent = 'projects/' + projectId + '/locations/-'; @@ -519,6 +534,7 @@ async function listVMWareEngine(projectId: any): Promise|null> { } async function listNamespaces(projectId: any, regionsList: Array): Promise|null> { ///// KO REGION + if(!currentConfig.ObjectNameNeed?.includes("namespace")) return null; const {RegistrationServiceClient,} = require('@google-cloud/service-directory'); let jsonData = []; @@ -534,6 +550,7 @@ async function listNamespaces(projectId: any, regionsList: Array): Promi } export async function listSecrets(projectId: any): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("secret")) return null; const {SecretManagerServiceClient,} = require('@google-cloud/secret-manager').v1; const parent = 'projects/globalInnovtech'; let jsonData = []; @@ -553,6 +570,7 @@ export async function listSecrets(projectId: any): Promise|null> { } async function listConnectivityTests(projectId: any): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("connectivity_test")) return null; const {ReachabilityServiceClient} = require('@google-cloud/network-management'); let jsonData = []; @@ -570,6 +588,7 @@ async function listConnectivityTests(projectId: any): Promise|null> } async function listResourceSettings(projectId: any): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("resource_settings")) return null; let jsonData = []; const { ResourceSettingsServiceClient } = require('@google-cloud/resource-settings'); @@ -588,6 +607,7 @@ async function listResourceSettings(projectId: any): Promise|null> { } async function listRedisInstances(projectId: any, regionsList: Array): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("redis_instance")) return null; const {CloudRedisClient} = require('@google-cloud/redis'); let jsonData = []; @@ -602,6 +622,7 @@ async function listRedisInstances(projectId: any, regionsList: Array): P } async function listOSConfig(projectId: any): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("os_config")) return null; const {OsConfigServiceClient} = require('@google-cloud/os-config'); let jsonData = []; @@ -619,6 +640,7 @@ async function listOSConfig(projectId: any): Promise|null> { } async function listOrgPolicyContraints(projectId: any): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("org_policy_constraint")) return null; const {OrgPolicyClient} = require('@google-cloud/org-policy'); let jsonData = []; @@ -636,6 +658,7 @@ async function listOrgPolicyContraints(projectId: any): Promise|null> } async function listOrchestrationAirflow(projectId: any, regionsList: Array): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("airflow_image_version")) return null; const {ImageVersionsClient} = require('@google-cloud/orchestration-airflow'); let jsonData = []; @@ -650,6 +673,7 @@ async function listOrchestrationAirflow(projectId: any, regionsList: Array | null> { + if(!currentConfig.ObjectNameNeed?.includes("notebook")) return null; const {NotebookServiceClient} = require('@google-cloud/notebooks'); let jsonData = []; @@ -669,6 +693,7 @@ async function listNotebookInstances(projectId: any): Promise | null> } async function listDashboards(projectId: any): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("dashboard")) return null; const { DashboardsServiceClient } = require('@google-cloud/monitoring-dashboards'); const parent = 'projects/' + projectId; let jsonData = []; @@ -687,6 +712,7 @@ async function listDashboards(projectId: any): Promise | null> { } async function listIdentitiesDomain(projectId: any): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("identity_domain")) return null; const { ManagedIdentitiesServiceClient } = require('@google-cloud/managed-identities'); let jsonData = []; @@ -723,6 +749,7 @@ async function listLineageProcesses(projectId: any): Promise | null> } async function listKMSCryptoKeys(projectId: any): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("kms_crypto_key")) return null; const {KeyDashboardServiceClient} = require('@google-cloud/kms-inventory').v1; let jsonData = []; const parent = 'projects/' + projectId; @@ -743,6 +770,7 @@ async function listKMSCryptoKeys(projectId: any): Promise | null> { } async function listKMSKeyRings(projectId: any): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("kms_key_ring")) return null; const {KeyManagementServiceClient} = require('@google-cloud/kms'); let jsonData = []; @@ -763,6 +791,7 @@ async function listKMSKeyRings(projectId: any): Promise | null> { } async function listDomainsRegistration(projectId: any): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("domain_registration")) return null; const {DomainsClient} = require('@google-cloud/domains'); let jsonData = []; @@ -780,6 +809,7 @@ async function listDomainsRegistration(projectId: any): Promise | nul } async function listDnsZones(projectId: any): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("dns_zone")) return null; const {DNS} = require('@google-cloud/dns'); let jsonData = []; @@ -797,6 +827,7 @@ async function listDnsZones(projectId: any): Promise | null> { } async function listDeliveryPipelines(projectId: any, regionsList: Array): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("pipeline")) return null; const {CloudDeployClient} = require('@google-cloud/deploy').v1; let jsonData = []; @@ -811,6 +842,7 @@ async function listDeliveryPipelines(projectId: any, regionsList: Array) } async function listCertificates(projectId: any): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("certificate")) return null; const {CertificateManagerClient} = require('@google-cloud/certificate-manager').v1; let jsonData = []; const parent = 'projects/' + projectId + '/locations/global'; @@ -830,6 +862,7 @@ async function listCertificates(projectId: any): Promise | null> { } async function listBatchJobs(projectId: any, regionsList: Array): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("batch_job")) return null; let jsonData = []; const {BatchServiceClient} = require('@google-cloud/batch').v1; @@ -844,7 +877,7 @@ async function listBatchJobs(projectId: any, regionsList: Array): Promis } async function listWorkloads(projectId: any): Promise | null> { - + if(!currentConfig.ObjectNameNeed?.includes("workload")) return null; const { AssuredWorkloadsServiceClient } = require('@google-cloud/assured-workloads'); const {ProjectsClient} = require('@google-cloud/resource-manager').v3; let jsonData; @@ -867,6 +900,7 @@ async function listWorkloads(projectId: any): Promise | null> { } async function listArtifactsRepositories(projectId: any, regionsList: Array): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("artifact_repository")) return null; const {ArtifactRegistryClient} = require('@google-cloud/artifact-registry'); let jsonData = []; @@ -881,6 +915,7 @@ async function listArtifactsRepositories(projectId: any, regionsList: Array): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("app_gateway")) return null; const {AppGatewaysServiceClient} = require('@google-cloud/appgateways').v1; let jsonData = []; @@ -899,6 +934,7 @@ async function listAppGateways(projectId: any, regionsList: Array): Prom ///////////////////////////////////////////////////////// /* async function listAppConnectors(projectId: any): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("app_connector")) return null; const {AppConnectorsServiceClient} = require('@google-cloud/appconnectors').v1; const parent = 'projects/' + projectId + '/locations/global'; let jsonData = []; @@ -919,6 +955,7 @@ async function listAppConnectors(projectId: any): Promise | null> { */ /* async function listApiKeys(projectId: any): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("api_key")) return null; const {ApiKeysClient} = require('@google-cloud/apikeys').v2; const parent = 'projects/' + projectId; let jsonData = []; @@ -938,6 +975,7 @@ async function listApiKeys(projectId: any): Promise | null> { }*/ /* async function listApi(projectId: any): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("api")) return null; const {ApiGatewayServiceClient} = require('@google-cloud/api-gateway'); let jsonData = []; @@ -957,6 +995,7 @@ async function listApi(projectId: any): Promise | null> { }*/ /* async function listAccessPolicy(projectId: any): Promise | null> { + if(!currentConfig.ObjectNameNeed?.includes("access_policy")) return null; const {AccessApprovalClient} = require('@google-cloud/access-approval'); let jsonData = []; @@ -977,6 +1016,7 @@ async function listAccessPolicy(projectId: any): Promise | null> { // Workstations : timeout because no instance to be tested // /* async function listWorkstations(projectId: any, regionsList: Array): Promise|null> { // KO + if(!currentConfig.ObjectNameNeed?.includes("workstation")) return null; const {WorkstationsClient} = require('@google-cloud/workstations').v1; let jsonData; try { @@ -995,6 +1035,7 @@ async function listWorkstations(projectId: any, regionsList: Array): Pro // Storage Config : Deadline/Timeout // /* async function listStorageConfig(projectId: any): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("storage_config")) return null; const {StorageInsightsClient} = require('@google-cloud/storageinsights').v1; let jsonData = []; const parent = 'projects/' + projectId + '/locations/-'; @@ -1019,6 +1060,7 @@ async function listStorageConfig(projectId: any): Promise|null> { // Private CA : Timeout // /* async function listPrivateCertificates(projectId: any): Promise|null> { + if(!currentConfig.ObjectNameNeed?.includes("private_certificate")) return null; const {CertificateAuthorityServiceClient} = require('@google-cloud/security-private-ca'); let jsonData = []; diff --git a/Kexa/services/addOn/githubGathering.service.ts b/Kexa/services/addOn/githubGathering.service.ts index 6627ca53..ad5b2ddd 100644 --- a/Kexa/services/addOn/githubGathering.service.ts +++ b/Kexa/services/addOn/githubGathering.service.ts @@ -27,11 +27,13 @@ env.config(); import {getContext, getNewLogger} from "../logger.service"; const logger = getNewLogger("GithubLogger"); let githubToken = ""; +let currentConfig:GitConfig export async function collectData(gitConfig:GitConfig[]): Promise{ let context = getContext(); let resources = new Array(); for(let config of gitConfig??[]){ + currentConfig = config; let prefix = config.prefix??(gitConfig.indexOf(config).toString()); githubToken = await getConfigOrEnvVar(config, "GITHUBTOKEN", prefix); if(!githubToken){ @@ -192,6 +194,11 @@ async function getOctokit(): Promise{ } export async function collectRepo(){ + if( + !currentConfig?.ObjectNameNeed?.includes("repositories") + && !currentConfig?.ObjectNameNeed?.includes("branches") + && !currentConfig?.ObjectNameNeed?.includes("issues") + ) return []; let page = 1; try{ let octokit = await getOctokit(); @@ -214,6 +221,7 @@ export async function collectRepo(){ } export async function collectBranch(repo: string, owner: string): Promise{ + if(!currentConfig?.ObjectNameNeed?.includes("branches")) return []; let page = 1; try{ let octokit = await getOctokit(); @@ -242,6 +250,7 @@ export async function collectBranch(repo: string, owner: string): Promise } export async function collectIssues(repo: string, owner: string): Promise{ + if(!currentConfig?.ObjectNameNeed?.includes("issues")) return []; let page = 1; try{ let octokit = await getOctokit(); @@ -270,6 +279,15 @@ export async function collectIssues(repo: string, owner: string): Promise } export async function collectOrganizations(): Promise{ + if( + !currentConfig?.ObjectNameNeed?.includes("organizations") + && !currentConfig?.ObjectNameNeed?.includes("members") + && !currentConfig?.ObjectNameNeed?.includes("teams") + && !currentConfig?.ObjectNameNeed?.includes("teamMembers") + && !currentConfig?.ObjectNameNeed?.includes("teamRepositories") + && !currentConfig?.ObjectNameNeed?.includes("teamProjects") + && !currentConfig?.ObjectNameNeed?.includes("outsideCollaborators") + ) return []; try{ return (await (await getOctokit()).request('GET /user/orgs')).data; }catch(e){ @@ -279,6 +297,7 @@ export async function collectOrganizations(): Promise{ } export async function collectMembers(org: string): Promise{ + if(!currentConfig?.ObjectNameNeed?.includes("members")) return []; let page = 1; try{ let octokit = await getOctokit(); @@ -315,6 +334,7 @@ export async function collectMembers(org: string): Promise{ } export async function collectOutsideCollaborators(org: string): Promise{ + if(!currentConfig?.ObjectNameNeed?.includes("outsideCollaborators")) return []; let page = 1; try{ let octokit = await getOctokit(); @@ -342,6 +362,7 @@ export async function collectOutsideCollaborators(org: string): Promise{ } export async function collectTeams(org: string): Promise{ + if(!currentConfig?.ObjectNameNeed?.includes("teams")) return []; let page = 1; try{ let octokit = await getOctokit(); @@ -369,6 +390,7 @@ export async function collectTeams(org: string): Promise{ } export async function collectTeamMembers(org:string, team: string): Promise{ + if(!currentConfig?.ObjectNameNeed?.includes("teamMembers")) return []; let page = 1; try{ let octokit = await getOctokit(); @@ -397,6 +419,7 @@ export async function collectTeamMembers(org:string, team: string): Promise } export async function collectTeamRepos(org:string, team: string): Promise{ + if(!currentConfig?.ObjectNameNeed?.includes("teamRepositories")) return []; let page = 1; try{ let octokit = await getOctokit(); @@ -425,6 +448,7 @@ export async function collectTeamRepos(org:string, team: string): Promise{ } export async function collectTeamProjects(org:string, team: string): Promise{ + if(!currentConfig?.ObjectNameNeed?.includes("teamProjects")) return []; let page = 1; try{ let octokit = await getOctokit(); diff --git a/Kexa/services/addOn/googleWorkspaceGathering.service.ts b/Kexa/services/addOn/googleWorkspaceGathering.service.ts index 2739cd52..d1c1c14c 100644 --- a/Kexa/services/addOn/googleWorkspaceGathering.service.ts +++ b/Kexa/services/addOn/googleWorkspaceGathering.service.ts @@ -33,6 +33,7 @@ const fs = require('fs').promises; const path = require('path'); const {authenticate} = require('@google-cloud/local-auth'); const {google} = require('googleapis'); +let currentConfig: googleWorkspaceConfig; ///////////////////////////////////////// ////// LISTING CLOUD RESOURCES ///// @@ -64,6 +65,7 @@ export async function collectData(googleWorkspaceConfig:googleWorkspaceConfig[]) for (let config of googleWorkspaceConfig??[]) { + currentConfig = config; let googleWorkspaceResources = { "user": null, "domain": null, @@ -156,6 +158,7 @@ async function authorize() { } async function listUsers(auth: any): Promise | null> { + if(!currentConfig?.ObjectNameNeed?.includes("user")) return null; let jsonData = []; const service = google.admin({version: 'directory_v1', auth}); @@ -199,7 +202,8 @@ async function listUsers(auth: any): Promise | null> { return jsonData ?? null; } async function listDomains(auth: any): Promise | null> { - let jsonData = []; + if(!currentConfig?.ObjectNameNeed?.includes("domain")) return null; + let jsonData = []; const admin = google.admin({version: 'directory_v1', auth}); try { @@ -232,6 +236,7 @@ async function listDomains(auth: any): Promise | null> { } async function listGroups(auth: any): Promise | null> { + if(!currentConfig?.ObjectNameNeed?.includes("group")) return null; let jsonData = []; const admin = google.admin({version: 'directory_v1', auth}); @@ -251,6 +256,7 @@ async function listGroups(auth: any): Promise | null> { } async function listRoles(auth: any): Promise | null> { + if(!currentConfig?.ObjectNameNeed?.includes("role")) return null; let jsonData = []; const service = google.admin({version: 'directory_v1', auth}); @@ -267,6 +273,7 @@ async function listRoles(auth: any): Promise | null> { } async function listOrganizationalUnits(auth: any): Promise | null> { + if(!currentConfig?.ObjectNameNeed?.includes("orgaunit")) return null; let jsonData = []; try { @@ -285,6 +292,7 @@ async function listOrganizationalUnits(auth: any): Promise | null> { return jsonData ?? null; } async function listCalendars(auth: any): Promise | null> { + if(!currentConfig?.ObjectNameNeed?.includes("calendar")) return null; let jsonData = []; try { @@ -309,6 +317,7 @@ async function listCalendars(auth: any): Promise | null> { } async function listFiles(auth: any): Promise | null> { + if(!currentConfig?.ObjectNameNeed?.includes("file")) return null; let jsonData = []; try { @@ -330,6 +339,7 @@ async function listFiles(auth: any): Promise | null> { } async function listDrive(auth: any): Promise | null> { + if(!currentConfig?.ObjectNameNeed?.includes("drive")) return null; let jsonData = []; try { diff --git a/Kexa/services/addOn/kubernetesGathering.service.ts b/Kexa/services/addOn/kubernetesGathering.service.ts index 332e5c01..d62c5fea 100644 --- a/Kexa/services/addOn/kubernetesGathering.service.ts +++ b/Kexa/services/addOn/kubernetesGathering.service.ts @@ -21,10 +21,12 @@ import {getNewLogger} from "../logger.service"; const logger = getNewLogger("KubernetesLogger"); const k8s = require('@kubernetes/client-node'); +let currentConfig:KubernetesConfig; export async function collectData(kubernetesConfig:KubernetesConfig[]): Promise{ let resources = new Array(); for(let config of kubernetesConfig??[]){ + currentConfig = config; let prefix = config.prefix??(kubernetesConfig.indexOf(config).toString()); try { let pathKubeFile = await getConfigOrEnvVar(config, "KUBECONFIG", prefix); @@ -78,6 +80,7 @@ export async function kubernetesListing(isPathKubeFile: boolean): Promise { } async function collectHelm(namespace: string): Promise { + if(!currentConfig?.ObjectNameNeed?.includes("helm")) return null; try{ let helmData = await helm.list({ namespace: namespace }); return helmData; @@ -88,6 +91,7 @@ async function collectHelm(namespace: string): Promise { } async function collectPods(k8sApiCore: any, namespace: string): Promise { + if(!currentConfig?.ObjectNameNeed?.includes("pods")) return null; try{ const pods = await k8sApiCore.listNamespacedPod(namespace); return pods; diff --git a/Kexa/services/addOn/o365Gathering.service.ts b/Kexa/services/addOn/o365Gathering.service.ts index 2200f680..f5853d2a 100644 --- a/Kexa/services/addOn/o365Gathering.service.ts +++ b/Kexa/services/addOn/o365Gathering.service.ts @@ -32,6 +32,7 @@ import { o365Config } from "../../models/o365/config.models"; import {getContext, getNewLogger} from "../logger.service"; const logger = getNewLogger("o365Logger"); +let currentConfig:o365Config; ///////////////////////////////////////// ////// LISTING CLOUD RESOURCES ///// @@ -42,6 +43,7 @@ export async function collectData(o365Config:o365Config[]): Promise(); for (let config of o365Config??[]) { + currentConfig = config; let o365Resources = { "sku": null, "user": null, @@ -173,6 +175,7 @@ async function getToken(tenantId: string, clientId: string, clientSecret: string return accessToken ?? null; } async function listUsers(endpoint: string, accessToken: string, headers: Headers): Promise | null> { + if(!currentConfig?.ObjectNameNeed?.includes("user")) return null; const axios = require("axios"); let jsonData = []; @@ -223,9 +226,9 @@ async function listUsers(endpoint: string, accessToken: string, headers: Header } async function listSubscribedSkus(endpoint: string, accessToken: string, headers: Headers): Promise | null> { + if(!currentConfig?.ObjectNameNeed?.includes("sku")) return null; let jsonData = []; - try { const response = await axios.get(`${endpoint}/subscribedSkus`, { headers: { @@ -262,6 +265,7 @@ async function listSubscribedSkus(endpoint: string, accessToken: string, header } async function genericListing(endpoint: string, accessToken: string, queryEndpoint: string, operationName: string): Promise | null> { + //if(!currentConfig?.ObjectNameNeed?.includes(operationName.toLowerCase().replace(' ', '_'))) return null; let jsonData = []; try { @@ -301,6 +305,7 @@ async function listSecureScore(endpoint: string, accessToken: string, headers: H } async function listAuthMethods(endpoint: string, accessToken: string, userList: any): Promise | null> { + if(!currentConfig?.ObjectNameNeed?.includes("auth_methods")) return null; let jsonData = []; for (const element of userList) { @@ -377,6 +382,7 @@ async function listIncidents(endpoint: string, accessToken: string, headers: Hea } async function listAppAccessPolicy(endpoint: string, accessToken: string, headers: Headers, userList: any): Promise | null> { + if(!currentConfig?.ObjectNameNeed?.includes("app_access_policy")) return null; const axios = require("axios"); let jsonData: any | []; for (let i = 0; i < userList.length; i++) { diff --git a/Kexa/services/analyse.service.ts b/Kexa/services/analyse.service.ts index 73fd0ff4..c102a34b 100644 --- a/Kexa/services/analyse.service.ts +++ b/Kexa/services/analyse.service.ts @@ -33,7 +33,7 @@ const varEnvMin = { const config = require('node-config-ts').config; const levelAlert = ["info", "warning", "error", "critical"]; let headers: any; -//Analyse list +// Analyze list // read the yaml file with rules // exam each rules and raise alarm or not export async function gatheringRules(rulesDirectory:string, getAll:boolean=false): Promise { @@ -46,7 +46,7 @@ export async function gatheringRules(rulesDirectory:string, getAll:boolean=false let listNeedRules = getListNeedRules(); for(const p of paths) { logger.debug("getting "+rulesDirectory+"/"+p.name+" rules."); - let setting = await analyseRule(rulesDirectory+"/"+p.name, listNeedRules, getAll); + let setting = await analyzeRule(rulesDirectory+"/"+p.name, listNeedRules, getAll); if(setting){ setting.alert.global.name = p.name.split(".")[0]; settingFileList.push(setting); @@ -60,12 +60,16 @@ export async function gatheringRules(rulesDirectory:string, getAll:boolean=false export function extractAddOnNeed(settingFileList: SettingFile[]){ let providerList = new Array(); + let objectNameList:any = {}; settingFileList.forEach((ruleFile) => { + objectNameList[ruleFile.alert.global.name] = {}; ruleFile.rules.forEach((rule) => { if(!providerList.includes(rule.cloudProvider)) providerList.push(rule.cloudProvider); + if(!objectNameList[ruleFile.alert.global.name][rule.cloudProvider]) objectNameList[ruleFile.alert.global.name][rule.cloudProvider] = new Array(); + if(!objectNameList[ruleFile.alert.global.name][rule.cloudProvider].includes(rule.objectName)) objectNameList[ruleFile.alert.global.name][rule.cloudProvider].push(rule.objectName); }); }); - writeStringToJsonFile(JSON.stringify({ "addOn" : providerList}), "./config/addOnNeed.json"); + writeStringToJsonFile(JSON.stringify({ "addOn" : providerList, "objectNameNeed": objectNameList }), "./config/addOnNeed.json"); } function getListNeedRules(): string[]{ @@ -89,8 +93,8 @@ function getListNeedRules(): string[]{ return listNeedRules; } -export async function analyseRule(ruleFilePath:string, listNeedRules:string[], getAll:boolean=false): Promise { - logger.debug("analyse:"+ruleFilePath); +export async function analyzeRule(ruleFilePath:string, listNeedRules:string[], getAll:boolean=false): Promise { + logger.debug("analyze:"+ruleFilePath); try { let lastBlockSplited = ruleFilePath.split('/')[ruleFilePath.split('/').length -1].split("."); if(lastBlockSplited.length == 1){ @@ -130,9 +134,9 @@ export async function checkDoc(doc:SettingFile): Promise { logger.debug("check doc"); let result:string[] = []; if(!doc.hasOwnProperty("version")) result.push("info - version not found in doc"); - else if(doc.version.match(/^[0-9]+\.[0-9]+\.[0-9]+$/) === null) result.push("debug - version not valid in doc : "+ doc.version); + else if(RegExp(/^[0-9]+\.[0-9]+\.[0-9]+$/).exec(doc.version) === null) result.push("debug - version not valid in doc : "+ doc.version); if(!doc.hasOwnProperty("date")) result.push("info - date not found in doc"); - else if(doc.date.match(/^(0[1-9]|[12][0-9]|3[01])-(0[1-9]|1[012])-(19|20)\d\d$/) === null) result.push("debug - date not valid in doc : "+ doc.date); + else if(RegExp(/^(0[1-9]|[12][0-9]|3[01])-(0[1-9]|1[012])-(19|20)\d\d$/).exec(doc.date) === null) result.push("debug - date not valid in doc : "+ doc.date); (await checkDocAlert(doc.alert)).forEach((value) => result.push(value)); checkDocRules(doc.rules).forEach((value) => result.push(value)); return result; diff --git a/config/default.json b/config/default.json index f774db90..6b0d7820 100644 --- a/config/default.json +++ b/config/default.json @@ -1,5 +1,12 @@ { "azure": [ + { + "description": "organization 4urcloud", + "rules": [ + "OperationalExcellence", + "Security" + ] + }, { "description": "organization 4urcloud", "rules": [ diff --git a/documentation/Documentation-Kexa.md b/documentation/Documentation-Kexa.md index 0ece0c53..80820116 100644 --- a/documentation/Documentation-Kexa.md +++ b/documentation/Documentation-Kexa.md @@ -873,7 +873,7 @@ We've set up a system to facilitate the development of new features. This system ### **Gathering data** -A file to collect data whose path will be "./Kexa/services/addOn". It is named as follows: [extension's name]Gathering.service.ts . The entry point for this file is a function named "collectData", which takes one arguments. The argument is a list containing all the configs for your addOn. The return format of this function is as shown in the following example. exemple : +A file to collect data whose path will be "./Kexa/services/addOn". It is named as follows: [extension's name]Gathering.service.ts . The entry point for this file is a function named "collectData", which takes one arguments. The argument is a list containing all the configs for your addOn. This list corresponds to what you can find in the default.json file in the key that has the same name as your addOn. As additional information per item in the list, you have "ObjectNameNeed" which corresponds to the CategoryItem solicity in the rules set up. This helps you, to optimize your addOn and make faster, more targeted collections. The return format of this function is as shown in the following example. exemple : ```json [ @@ -900,7 +900,7 @@ A file to collect data whose path will be "./Kexa/services/addOn". It is named a } ] ``` - + The data format is the following for several reasons. The first list corresponds to the different subscriptions or accounts in a sector. To illustrate, in the case of a cloud provider such as Azure, we might need different identifiers to scan all our accounts, and each account scan result is an item in the list. The dictionaries in the main list correspond to your dictionary that you find relevant to create to identify the different resources of your addOn, in the rules this corresponds to your "objectName". Finally, the lowest-level lists correspond to the list of items collected by your addOn. For project maintainability, we require addOn modules to contain a header to quickly identify certain information. It's also important to update this header according to the capabilities of your addOn. In fact, this header serves as a basis for checking the feasibility of certain rules. This header is a comment that must contain at least 2 pieces of information: the name of the module and the "categoryItems" you've included in your module. example for an "azureComplement" module: file name : azureComplementGathering.service.ts @@ -916,8 +916,8 @@ For project maintainability, we require addOn modules to contain a header to qui * - azFunction */ -export async function collectData(myGlobalConfig: any[]){ - //the type of myGlobalConfig is any but you can make an interface if you want +export async function collectData(myAddonConfig: any[]){ + //the type of myAddonConfig is any but you can make an interface if you want //insert your stuff here }