[GCP ]Filter the logs base on logNames and timestamps

collectors/googlestackdriver/cfn/googlestackdriver-collector.template

@@ -57,6 +57,10 @@
             "Description": "JSON list of google resources to poll logs from. In the format <resourceType>/<resourceID>",
             "Type": "String"
         },
+        "LogNameFilters": {
+            "Description": "A JSON list of Google Cloud log names used to filter logs. The format should be 'service.googleapis.com%2Flog_type', for example: 'cloudaudit.googleapis.com%2Factivity'.",
+            "Type": "String"
+        },
         "CollectionStartTs": {
             "Description": "Timestamp when log collection starts. For example, 2020-01-13T16:00:00Z",
             "Type": "String",
@@ -113,6 +117,9 @@
                     "CollectorStreams": {
                         "Ref": "GoogleResourceIds"
                     },
+                    "CollectorParamString2": {
+                        "Ref": "LogNameFilters"
+                    },
                     "CollectionStartTs": {
                         "Ref": "CollectionStartTs"
                     }

collectors/googlestackdriver/collector.js

@@ -61,13 +61,15 @@ class GooglestackdriverCollector extends PawsCollector {
             endTs = moment(startTs).add(this.pollInterval, 'seconds').toISOString();
         }
         const resourceNames = JSON.parse(process.env.collector_streams);
-        const initialStates = resourceNames.map(stream => ({
-            stream,
-            nextPage:null,
-            since: startTs,
-            until: endTs,
-            poll_interval_sec: 1
-        }));
+        const initialStates = resourceNames
+            .filter(stream => stream && stream.trim() !== "")  // Filter out empty or invalid values
+            .map(stream => ({
+                stream,
+                nextPage: null,
+                since: startTs,
+                until: endTs,
+                poll_interval_sec: 1
+            }));
         return callback(null, initialStates, 1);
     }
 
@@ -97,9 +99,7 @@ class GooglestackdriverCollector extends PawsCollector {
 
         AlLogger.info(`GSTA000001 Collecting data from ${state.since} till ${state.until} for ${state.stream}`);
 
-        // TODO: figure out a better way to format this. I'm pretty sure that it needs the newlines in it.
-        const filter = `timestamp >= "${state.since}"
-timestamp < "${state.until}"`;
+        const filter = collector.generateFilter(state);
 
         let pagesRetireved = 0;
 
@@ -182,6 +182,39 @@ timestamp < "${state.until}"`;
             });
     }
 
+    generateFilter(state) {
+        const logNames = process.env.paws_collector_param_string_2 ? JSON.parse(process.env.paws_collector_param_string_2) : [];
+        const filterConditions = [];
+        let logFilterCondition;
+
+        if (logNames.length > 0) {
+            logNames.forEach(logName => {
+                const trimmedlogName = logName?.trim();
+
+                if (trimmedlogName) {
+                    const encodelogName = this.isUriEncoded(logName) ? logName : encodeURIComponent(logName);
+                    filterConditions.push(`logName:"${encodelogName}"`);
+                } else {
+                    AlLogger.warn("Skipping empty log ID.");
+                }
+            });
+            if (filterConditions.length > 0) {
+                logFilterCondition = filterConditions.join(" OR ");
+            }
+        }
+        // Construct the basic timestamp filter
+        let filterQuery = `timestamp >= "${state.since}" AND timestamp < "${state.until}"`;
+        if (logFilterCondition) {
+            // Combine the LogName and timesamp filter
+            filterQuery = `${filterQuery} AND (${logFilterCondition})`;
+        }
+        return filterQuery;
+    }
+
+    isUriEncoded(logName) {
+        return decodeURIComponent(logName) !== logName;
+    }
+
 
     _getNextCollectionState(curState, nextPage) {
         // Reset the page size for the next collection if it's less than the maximum

collectors/googlestackdriver/package.json

@@ -1,6 +1,6 @@
 {
   "name": "googlestackdriver-collector",
-  "version": "1.2.14",
+  "version": "1.2.15",
   "description": "Alert Logic AWS based Googlestackdriver Log Collector",
   "repository": {},
   "private": true,
@@ -9,28 +9,28 @@
     "test": "JUNIT_REPORT_PATH=./test/report.xml nyc --reporter=text mocha --colors"
   },
   "devDependencies": {
-    "@aws-sdk/client-cloudformation": "^3.632.0",
-    "@aws-sdk/client-cloudwatch": "^3.632.0",
-    "@aws-sdk/client-dynamodb": "^3.632.0",
-    "@aws-sdk/client-kms": "^3.632.0",
-    "@aws-sdk/client-lambda": "^3.632.0",
-    "@aws-sdk/client-s3": "^3.633.0",
-    "@aws-sdk/client-sqs": "^3.632.0",
-    "@aws-sdk/client-ssm": "^3.632.0",
+    "@aws-sdk/client-cloudformation": "^3.666.0",
+    "@aws-sdk/client-cloudwatch": "^3.666.0",
+    "@aws-sdk/client-dynamodb": "^3.666.0",
+    "@aws-sdk/client-kms": "^3.666.0",
+    "@aws-sdk/client-lambda": "^3.666.0",
+    "@aws-sdk/client-s3": "^3.666.0",
+    "@aws-sdk/client-sqs": "^3.666.0",
+    "@aws-sdk/client-ssm": "^3.666.0",
     "jshint": "^2.13.6",
     "mocha": "^10.7.3",
     "mocha-jenkins-reporter": "^0.4.8",
-    "nyc": "^17.0.0",
+    "nyc": "^17.1.0",
     "rewire": "^7.0.0",
-    "sinon": "^18.0.0"
+    "sinon": "^19.0.2"
   },
   "dependencies": {
-    "@alertlogic/al-collector-js": "3.0.12",
-    "@alertlogic/paws-collector": "2.2.5",
-    "async": "^3.2.5",
-    "debug": "^4.3.6",
-    "google-auth-library": "^9.13.0",
-    "googleapis": "^140.0.1",
+    "@alertlogic/al-collector-js": "3.0.14",
+    "@alertlogic/paws-collector": "2.2.6",
+    "async": "^3.2.6",
+    "debug": "^4.3.7",
+    "google-auth-library": "^9.14.1",
+    "googleapis": "^144.0.0",
     "moment": "2.30.1"
   },
   "author": "Alert Logic Inc."

collectors/googlestackdriver/test/test.js

@@ -293,10 +293,8 @@ describe('Unit Tests', function() {
             const startDate = moment().subtract(3, 'days');
             let since = startDate.toISOString();
             let until = startDate.add(2, 'days').toISOString();
-            const filter = `timestamp >= "${since}"
-timestamp < "${until}"`;
+            const filter = `timestamp >= "${since}" AND timestamp < "${until}"`;
             let nextPage = { pageToken: 'http://somenextpage.com', "pageSize": 1000, "resourceNames": ["projects/a-fake-project"], filter };
-
             logginClientStub.callsFake(() => {
                 return new Promise((res, rej) => {
                     res({
@@ -539,4 +537,106 @@ timestamp < "${until}"`;
             });
         });
     });
-    });
+    describe('log filter Tests', function() {
+        it('should generate correct filter excluding empty logType values', function (done) {
+            let ctx = {
+                invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN,
+                fail: function (error) {
+                    assert.fail(error);
+                    done();
+                },
+                succeed: function () {
+                    done();
+                }
+            };
+
+            GooglestackdriverCollector.load().then(function (creds) {
+                var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver');
+                const startDate = moment().subtract(20, 'minutes');
+                let since = startDate.toISOString();
+                let until = startDate.add(collector.pollInterval, 'seconds').toISOString();
+                process.env.paws_collector_param_string_2 = "[\"cloudaudit.googleapis.com%2Factivity\",\"\",\"cloudfunctions.googleapis.com/cloud-functions\"]";
+                const curState = {
+                    since: since,
+                    until: until,
+                    poll_interval_sec: 1,
+                    stream: 'projects/imran-49253',
+                };
+                // Expected filter string
+                const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}" AND (logName:"cloudaudit.googleapis.com%2Factivity" OR logName:"cloudfunctions.googleapis.com%2Fcloud-functions")`;
+
+                // Call the function to generate the filter
+                const filter = collector.generateFilter(curState);
+                assert.equal(expectedFilter, filter);
+                done();
+            });
+        });
+
+        it('should generate filter without logNameFilter when all logTypes are empty', function (done) {
+            let ctx = {
+                invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN,
+                fail: function (error) {
+                    assert.fail(error);
+                    done();
+                },
+                succeed: function () {
+                    done();
+                }
+            };
+
+            GooglestackdriverCollector.load().then(function (creds) {
+                var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver');
+                const startDate = moment().subtract(20, 'minutes');
+                let since = startDate.toISOString();
+                let until = startDate.add(collector.pollInterval, 'seconds').toISOString();
+                process.env.paws_collector_param_string_2 = "[\"\",\"\"]";
+                const curState = {
+                    since: since,
+                    until: until,
+                    poll_interval_sec: 1,
+                    stream: 'projects/imran-49253',
+                };
+                // Expected filter string
+                const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}"`;
+
+                // Call the function to generate the filter
+                const filter = collector.generateFilter(curState);
+                assert.equal(expectedFilter, filter);
+                done();
+            });
+        });
+        it('should handle case when logTypes is undefined or null', function (done) {
+            let ctx = {
+                invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN,
+                fail: function (error) {
+                    assert.fail(error);
+                    done();
+                },
+                succeed: function () {
+                    done();
+                }
+            };
+
+            GooglestackdriverCollector.load().then(function (creds) {
+                var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver');
+                const startDate = moment().subtract(20, 'minutes');
+                let since = startDate.toISOString();
+                let until = startDate.add(collector.pollInterval, 'seconds').toISOString();
+                const curState = {
+                    since: since,
+                    until: until,
+                    poll_interval_sec: 1,
+                    stream: 'projects/imran-49253',
+                };
+                // Expected filter string
+                const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}"`;
+
+                // Call the function to generate the filter
+                const filter = collector.generateFilter(curState);
+                assert.equal(expectedFilter, filter);
+                done();
+            });
+        });     
+    });
+});
+

ps_spec.yml

@@ -177,7 +177,7 @@ stages:
       - ./build_collector.sh googlestackdriver
     env:
       ALPS_SERVICE_NAME: "paws-googlestackdriver-collector"
-      ALPS_SERVICE_VERSION: "1.2.14" #set the value from collector package json
+      ALPS_SERVICE_VERSION: "1.2.15" #set the value from collector package json
     outputs:
       file: ./googlestackdriver-collector*
     packagers: