Filter the logs base on logNames and timestamps

rakhimundhada15 · Oct 8, 2024 · 53f29ac · 53f29ac
1 parent 4f42e22
commit 53f29ac
Show file tree

Hide file tree

Showing 3 changed files with 140 additions and 6 deletions.
diff --git a/collectors/googlestackdriver/cfn/googlestackdriver-collector.template b/collectors/googlestackdriver/cfn/googlestackdriver-collector.template
@@ -57,6 +57,10 @@
             "Description": "JSON list of google resources to poll logs from. In the format <resourceType>/<resourceID>",
             "Type": "String"
         },
+        "LogNameFilters": {
+            "Description": "A JSON list of Google Cloud log names used to filter logs. The format should be 'service.googleapis.com%2Flog_type', for example: 'compute.googleapis.com%2Factivity_log'.",
+            "Type": "String"
+        },
         "CollectionStartTs": {
             "Description": "Timestamp when log collection starts. For example, 2020-01-13T16:00:00Z",
             "Type": "String",
@@ -113,6 +117,9 @@
                     "CollectorStreams": {
                         "Ref": "GoogleResourceIds"
                     },
+                    "CollectorParamString2": {
+                        "Ref": "LogNameFilters"
+                    },
                     "CollectionStartTs": {
                         "Ref": "CollectionStartTs"
                     }

diff --git a/collectors/googlestackdriver/collector.js b/collectors/googlestackdriver/collector.js
@@ -98,8 +98,7 @@ class GooglestackdriverCollector extends PawsCollector {
         AlLogger.info(`GSTA000001 Collecting data from ${state.since} till ${state.until} for ${state.stream}`);
 
         // TODO: figure out a better way to format this. I'm pretty sure that it needs the newlines in it.
-        const filter = `timestamp >= "${state.since}"
-timestamp < "${state.until}"`;
+        const filter = collector.generateFilter(state);
 
         let pagesRetireved = 0;
 
@@ -182,6 +181,33 @@ timestamp < "${state.until}"`;
             });
     }
 
+    generateFilter(state) {
+        const logTypes = process.env.paws_collector_param_string_2 ? JSON.parse(process.env.paws_collector_param_string_2) : [];
+        let filterParts = [];
+        let logNameFilter;
+
+        if (logTypes && logTypes.length > 0) {
+            logTypes.forEach(logType => {
+                if (state.stream && logType && logType.trim() !== "") {  // Check that logType is not empty
+                    filterParts.push(`logName="${state.stream}/logs/${logType}"`);
+                } else if (!logType || logType.trim() === "") {
+                    AlLogger.warn("Skipping empty log type.");
+                }
+            });
+            if (filterParts.length > 0) {
+                logNameFilter = filterParts.join(" OR ");
+            }
+        }
+        // Construct the basic timestamp filter
+        let filter = `timestamp >= "${state.since}" AND timestamp < "${state.until}"`;
+
+        if (logNameFilter) {
+            // Combine the LogName and timesamp filter
+            filter = `${filter} AND (${logNameFilter})`;
+        }
+        return filter;
+    }
+
 
     _getNextCollectionState(curState, nextPage) {
         // Reset the page size for the next collection if it's less than the maximum

diff --git a/collectors/googlestackdriver/test/test.js b/collectors/googlestackdriver/test/test.js
@@ -293,10 +293,8 @@ describe('Unit Tests', function() {
             const startDate = moment().subtract(3, 'days');
             let since = startDate.toISOString();
             let until = startDate.add(2, 'days').toISOString();
-            const filter = `timestamp >= "${since}"
-timestamp < "${until}"`;
+            const filter = `timestamp >= "${since}" AND timestamp < "${until}"`;
             let nextPage = { pageToken: 'http://somenextpage.com', "pageSize": 1000, "resourceNames": ["projects/a-fake-project"], filter };
-
             logginClientStub.callsFake(() => {
                 return new Promise((res, rej) => {
                     res({
@@ -539,4 +537,107 @@ timestamp < "${until}"`;
             });
         });
     });
-    });
+    describe('log filter Tests', function() {
+        it('should generate correct filter excluding empty logType values', function (done) {
+            let ctx = {
+                invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN,
+                fail: function (error) {
+                    assert.fail(error);
+                    done();
+                },
+                succeed: function () {
+                    done();
+                }
+            };
+
+            GooglestackdriverCollector.load().then(function (creds) {
+                var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver');
+                const startDate = moment().subtract(20, 'minutes');
+                let since = startDate.toISOString();
+                let until = startDate.add(collector.pollInterval, 'seconds').toISOString();
+                process.env.paws_collector_param_string_2 = "[\"cloudaudit.googleapis.com%2Factivity\",\"\",\"cloudfunctions.googleapis.com%2Fcloud-functions\"]";
+                const curState = {
+                    since: since,
+                    until: until,
+                    poll_interval_sec: 1,
+                    stream: 'projects/imran-49253',
+                };
+                // Expected filter string
+                const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}" AND (logName="projects/imran-49253/logs/cloudaudit.googleapis.com%2Factivity" OR logName="projects/imran-49253/logs/cloudfunctions.googleapis.com%2Fcloud-functions")`;
+
+                // Call the function to generate the filter
+                const filter = collector.generateFilter(curState);
+                assert.equal(expectedFilter, filter);
+                done();
+            });
+        });
+
+        it('should generate filter without logNameFilter when all logTypes are empty', function (done) {
+            let ctx = {
+                invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN,
+                fail: function (error) {
+                    assert.fail(error);
+                    done();
+                },
+                succeed: function () {
+                    done();
+                }
+            };
+
+            GooglestackdriverCollector.load().then(function (creds) {
+                var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver');
+                const startDate = moment().subtract(20, 'minutes');
+                let since = startDate.toISOString();
+                let until = startDate.add(collector.pollInterval, 'seconds').toISOString();
+                process.env.paws_collector_param_string_2 = "[\"\",\"\"]";
+                const curState = {
+                    since: since,
+                    until: until,
+                    poll_interval_sec: 1,
+                    stream: 'projects/imran-49253',
+                };
+                // Expected filter string
+                const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}"`;
+
+                // Call the function to generate the filter
+                const filter = collector.generateFilter(curState);
+                assert.equal(expectedFilter, filter);
+                done();
+            });
+        });
+        it('should handle case when logTypes is undefined or null', function (done) {
+            let ctx = {
+                invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN,
+                fail: function (error) {
+                    assert.fail(error);
+                    done();
+                },
+                succeed: function () {
+                    done();
+                }
+            };
+
+            GooglestackdriverCollector.load().then(function (creds) {
+                var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver');
+                const startDate = moment().subtract(20, 'minutes');
+                let since = startDate.toISOString();
+                let until = startDate.add(collector.pollInterval, 'seconds').toISOString();
+                process.env.paws_collector_param_string_2 = null;
+                const curState = {
+                    since: since,
+                    until: until,
+                    poll_interval_sec: 1,
+                    stream: 'projects/imran-49253',
+                };
+                // Expected filter string
+                const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}"`;
+
+                // Call the function to generate the filter
+                const filter = collector.generateFilter(curState);
+                assert.equal(expectedFilter, filter);
+                done();
+            });
+        });     
+    });
+});
+