Skip to content

Commit

Permalink
To handle max socket warning while writing the item on DDb
Browse files Browse the repository at this point in the history
  • Loading branch information
Rakhi Mundhada authored and Rakhi Mundhada committed Jul 26, 2024
1 parent 279bac0 commit 8ece285
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 5 deletions.
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@alertlogic/paws-collector",
"version": "2.2.4",
"version": "2.2.5",
"license": "MIT",
"description": "Alert Logic AWS based API Poll Log Collector Library",
"repository": {
Expand Down Expand Up @@ -29,6 +29,7 @@
"@aws-sdk/client-s3": "^3.600.0",
"@aws-sdk/client-sqs": "^3.600.0",
"@aws-sdk/client-ssm": "^3.600.0",
"@smithy/node-http-handler":"3.1.3",
"clone": "*",
"jshint": "^2.13.6",
"mocha": "^10.4.0",
Expand Down
24 changes: 20 additions & 4 deletions paws_collector.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ const {
{
SSM
} = require("@aws-sdk/client-ssm");
const { NodeHttpHandler } = require("@smithy/node-http-handler");
const http = require("http");
const https = require("https");
const fs = require('fs');
const moment = require('moment');
const ddLambda = require('datadog-lambda-js');
Expand All @@ -52,10 +55,23 @@ const ERROR_CODE_COMPLETED_STATE = 'COMPLETED_STATE';
const SQS_VISIBILITY_TIMEOUT = 900;
const DDB_TTL_DAYS = 14;
const DEDUP_LOG_TTL_SECONDS = 86400;
const DDB_OPTIONS = {
maxRetries: 10,
ConsistentRead: true
let maxSocket = process.env.maxSocket ? parseInt(process.env.maxSocket, 10) : 100;

const agent = {
maxSockets: maxSocket,
keepAlive: true
};

// Configure the NodeHttpHandler with the custom agents
const nodeHttpHandler = new NodeHttpHandler({
httpAgent: new http.Agent(agent),
httpsAgent: new https.Agent(agent)
});

const DDB_OPTIONS = {
maxAttempts: 10,
requestHandler: nodeHttpHandler
}
const DDB_DELETE_BATCH_OPTIONS = {
maxBatchSize: 25,
maxBatchSizeBytes: 16 * 1024 * 1024
Expand Down Expand Up @@ -925,7 +941,7 @@ class PawsCollector extends AlAwsCollector {
*/
removeDuplicatedItem(logs, paramName, asyncCallback) {
let collector = this;
const ddb = new DynamoDB();
const ddb = new DynamoDB(DDB_OPTIONS);
let uniqueLogs = [];
var promises = [];
let duplicateCount = 0;
Expand Down

0 comments on commit 8ece285

Please sign in to comment.