diff --git a/collectors/googlestackdriver/cfn/googlestackdriver-collector.template b/collectors/googlestackdriver/cfn/googlestackdriver-collector.template index aafa8d1f..df110ea5 100644 --- a/collectors/googlestackdriver/cfn/googlestackdriver-collector.template +++ b/collectors/googlestackdriver/cfn/googlestackdriver-collector.template @@ -57,6 +57,10 @@ "Description": "JSON list of google resources to poll logs from. In the format /", "Type": "String" }, + "LogNameFilters": { + "Description": "A JSON list of Google Cloud log names used to filter logs. The format should be 'service.googleapis.com%2Flog_type', for example: 'compute.googleapis.com%2Factivity_log'.", + "Type": "String" + }, "CollectionStartTs": { "Description": "Timestamp when log collection starts. For example, 2020-01-13T16:00:00Z", "Type": "String", @@ -113,6 +117,9 @@ "CollectorStreams": { "Ref": "GoogleResourceIds" }, + "CollectorParamString2": { + "Ref": "LogNameFilters" + }, "CollectionStartTs": { "Ref": "CollectionStartTs" } diff --git a/collectors/googlestackdriver/collector.js b/collectors/googlestackdriver/collector.js index 606d4bce..cb18e7b8 100644 --- a/collectors/googlestackdriver/collector.js +++ b/collectors/googlestackdriver/collector.js @@ -98,8 +98,7 @@ class GooglestackdriverCollector extends PawsCollector { AlLogger.info(`GSTA000001 Collecting data from ${state.since} till ${state.until} for ${state.stream}`); // TODO: figure out a better way to format this. I'm pretty sure that it needs the newlines in it. - const filter = `timestamp >= "${state.since}" -timestamp < "${state.until}"`; + const filter = collector.generateFilter(state); let pagesRetireved = 0; @@ -182,6 +181,33 @@ timestamp < "${state.until}"`; }); } + generateFilter(state) { + const logTypes = process.env.paws_collector_param_string_2 ? JSON.parse(process.env.paws_collector_param_string_2) : []; + let filterParts = []; + let logNameFilter; + + if (logTypes && logTypes.length > 0) { + logTypes.forEach(logType => { + if (state.stream && logType && logType.trim() !== "") { // Check that logType is not empty + filterParts.push(`logName="${state.stream}/logs/${logType}"`); + } else if (!logType || logType.trim() === "") { + AlLogger.warn("Skipping empty log type."); + } + }); + if (filterParts.length > 0) { + logNameFilter = filterParts.join(" OR "); + } + } + // Construct the basic timestamp filter + let filter = `timestamp >= "${state.since}" AND timestamp < "${state.until}"`; + + if (logNameFilter) { + // Combine the LogName and timesamp filter + filter = `${filter} AND (${logNameFilter})`; + } + return filter; + } + _getNextCollectionState(curState, nextPage) { // Reset the page size for the next collection if it's less than the maximum diff --git a/collectors/googlestackdriver/test/test.js b/collectors/googlestackdriver/test/test.js index eab9f07f..96a8a0d2 100644 --- a/collectors/googlestackdriver/test/test.js +++ b/collectors/googlestackdriver/test/test.js @@ -293,10 +293,8 @@ describe('Unit Tests', function() { const startDate = moment().subtract(3, 'days'); let since = startDate.toISOString(); let until = startDate.add(2, 'days').toISOString(); - const filter = `timestamp >= "${since}" -timestamp < "${until}"`; + const filter = `timestamp >= "${since}" AND timestamp < "${until}"`; let nextPage = { pageToken: 'http://somenextpage.com', "pageSize": 1000, "resourceNames": ["projects/a-fake-project"], filter }; - logginClientStub.callsFake(() => { return new Promise((res, rej) => { res({ @@ -539,4 +537,107 @@ timestamp < "${until}"`; }); }); }); - }); \ No newline at end of file + describe('log filter Tests', function() { + it('should generate correct filter excluding empty logType values', function (done) { + let ctx = { + invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + done(); + }, + succeed: function () { + done(); + } + }; + + GooglestackdriverCollector.load().then(function (creds) { + var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver'); + const startDate = moment().subtract(20, 'minutes'); + let since = startDate.toISOString(); + let until = startDate.add(collector.pollInterval, 'seconds').toISOString(); + process.env.paws_collector_param_string_2 = "[\"cloudaudit.googleapis.com%2Factivity\",\"\",\"cloudfunctions.googleapis.com%2Fcloud-functions\"]"; + const curState = { + since: since, + until: until, + poll_interval_sec: 1, + stream: 'projects/imran-49253', + }; + // Expected filter string + const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}" AND (logName="projects/imran-49253/logs/cloudaudit.googleapis.com%2Factivity" OR logName="projects/imran-49253/logs/cloudfunctions.googleapis.com%2Fcloud-functions")`; + + // Call the function to generate the filter + const filter = collector.generateFilter(curState); + assert.equal(expectedFilter, filter); + done(); + }); + }); + + it('should generate filter without logNameFilter when all logTypes are empty', function (done) { + let ctx = { + invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + done(); + }, + succeed: function () { + done(); + } + }; + + GooglestackdriverCollector.load().then(function (creds) { + var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver'); + const startDate = moment().subtract(20, 'minutes'); + let since = startDate.toISOString(); + let until = startDate.add(collector.pollInterval, 'seconds').toISOString(); + process.env.paws_collector_param_string_2 = "[\"\",\"\"]"; + const curState = { + since: since, + until: until, + poll_interval_sec: 1, + stream: 'projects/imran-49253', + }; + // Expected filter string + const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}"`; + + // Call the function to generate the filter + const filter = collector.generateFilter(curState); + assert.equal(expectedFilter, filter); + done(); + }); + }); + it('should handle case when logTypes is undefined or null', function (done) { + let ctx = { + invokedFunctionArn: googlestackdriverMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + done(); + }, + succeed: function () { + done(); + } + }; + + GooglestackdriverCollector.load().then(function (creds) { + var collector = new GooglestackdriverCollector(ctx, creds, 'googlestackdriver'); + const startDate = moment().subtract(20, 'minutes'); + let since = startDate.toISOString(); + let until = startDate.add(collector.pollInterval, 'seconds').toISOString(); + process.env.paws_collector_param_string_2 = null; + const curState = { + since: since, + until: until, + poll_interval_sec: 1, + stream: 'projects/imran-49253', + }; + // Expected filter string + const expectedFilter = `timestamp >= "${since}" AND timestamp < "${until}"`; + + // Call the function to generate the filter + const filter = collector.generateFilter(curState); + assert.equal(expectedFilter, filter); + done(); + }); + }); + }); +}); + \ No newline at end of file