From 74b3a009a3e4b79e6e9cb166621f1fb528fda211 Mon Sep 17 00:00:00 2001 From: Chronicle Team Date: Tue, 23 Apr 2024 20:19:27 +0000 Subject: [PATCH] No public description PiperOrigin-RevId: 627484841 --- README.md | 10 +- common/options.py | 1 + parsers/commands/classify_log_type.py | 145 +++++++++ parsers/commands/classify_log_type_test.py | 324 +++++++++++++++++++++ parsers/constants/key_constants.py | 2 + parsers/parsers.py | 2 + parsers/parsers_test.py | 1 + parsers/tests/fixtures.py | 11 + parsers/url.py | 1 + 9 files changed, 492 insertions(+), 5 deletions(-) create mode 100644 parsers/commands/classify_log_type.py create mode 100644 parsers/commands/classify_log_type_test.py diff --git a/README.md b/README.md index 374bf57..25a581f 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -# Chronicle CLI +# Google Security Operations CLI -Command line tool to interact with Chronicle's APIs. +Command line tool to interact with Google Security Operations' APIs. -Chronicle CLI allows customers to manage various operations that can be -performed on Chronicle. This script provides a command line tool to interact +Google Security Operations CLI allows customers to manage various operations that can be +performed on Google Security Operations. This script provides a command line tool to interact with Feed, Parser, Forwarder and BigQuery APIs. It will gradually expand to cover other APIs. @@ -12,7 +12,7 @@ cover other APIs. Follow these instructions: https://cloud.google.com/python/setup You may skip installing the Cloud Client Libraries and the Cloud SDK, they are -unnecessary for interacting with Chronicle. +unnecessary for interacting with Google Security Operations. After creating and activating the virtual environment `venv`, clone the repository using following command: diff --git a/common/options.py b/common/options.py index 39e7a0f..666d4d4 100644 --- a/common/options.py +++ b/common/options.py @@ -29,6 +29,7 @@ "EUROPE-WEST6", "ME-CENTRAL2", "ME-WEST1", + "NORTHAMERICA-NORTHEAST2", "US", ] diff --git a/parsers/commands/classify_log_type.py b/parsers/commands/classify_log_type.py new file mode 100644 index 0000000..745dcf1 --- /dev/null +++ b/parsers/commands/classify_log_type.py @@ -0,0 +1,145 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Classify the provided logs to the corresponding log types.""" + +import base64 +import os + +import click + +from common import api_utility +from common import chronicle_auth +from common import exception_handler +from common import options +from common.constants import key_constants as common_constants +from common.constants import status +from parsers import url +from parsers.constants import key_constants as parser_constants + + +@click.command( + name="classify_log_type", + help="[New]Classify the provided logs to the log types.") +@click.argument("project_id", required=True, default="") +@click.argument("customer_id", required=True, default="") +@click.argument("log_file", required=True, default="") +@options.env_option +@options.region_option +@options.verbose_option +@options.credential_file_option +@options.v2_option +@exception_handler.catch_exception() +def classify_log_type( + v2: bool, + credential_file: str, + verbose: bool, + region: str, + env: str, + project_id: str, + customer_id: str, + log_file: str) -> None: + """Classify the provided logs to the corresponding log types. + + Args: + v2 (bool): Option for enabling v2 commands. + credential_file (AnyStr): Path of Service Account JSON. + verbose (bool): Option for printing verbose output to console. + region (str): Option for selecting regions. Available options - US, EUROPE, + ASIA_SOUTHEAST1. + env (str): Option for selection environment. Available options - prod, test. + project_id (str): The GCP Project ID. + customer_id (str): The Customer ID. + log_file (str): Path of log file containing a single log line. + + Raises: + OSError: Failed to read the given file, e.g. not found, no read access + (https://docs.python.org/library/exceptions.html#os-exceptions). + ValueError: Invalid file contents. + KeyError: Required key is not present in dictionary. + TypeError: If response data is not JSON. + """ + if not v2: + click.echo("--v2 flag not provided. " + "Please provide the flag to run the new commands") + return + + if not project_id: + click.echo("Project ID not provided. Please enter Project ID") + return + + if not customer_id: + click.echo("Customer ID not provided. Please enter Customer ID") + return + + if not os.path.exists(log_file): + click.echo(f"{log_file} does not exist. " + "Please enter valid log file path") + return + + click.echo("Classifying the provided log to the corresponding log types...\n") + + resources = { + "project": project_id, + "location": region.lower(), + "instance": customer_id + } + + with open(log_file, "r") as f: + log_lines = f.readlines() + + log_data = [] + for log_line in log_lines: + log_line = log_line.strip(" \n") + log_data.append(base64.b64encode(log_line.encode()).decode()) + + data = { + parser_constants.KEY_LOG_DATA: log_data, + } + + classify_log_type_url = url.get_dataplane_url( + region, + "classify_log_type", + env, + resources) + client = chronicle_auth.initialize_dataplane_http_session(credential_file) + method = "POST" + response = client.request( + method, classify_log_type_url, + json=data, timeout=url.HTTP_REQUEST_TIMEOUT_IN_SECS) + parsed_response = api_utility.check_content_type(response.text) + + if response.status_code != status.STATUS_OK: + click.echo( + f"Error while classifying the logs.\n" + f"Response Code: {response.status_code}\n" + f"Error: " + f"{parsed_response[common_constants.KEY_ERROR][common_constants.KEY_MESSAGE]}" + ) + return + + if parser_constants.KEY_PREDICTIONS not in parsed_response: + click.echo("No predictions found in the response.") + return + + results = parsed_response.get(parser_constants.KEY_PREDICTIONS, []) + for result in results: + # Handle log type and score + log_type = result[parser_constants.KEY_LOGTYPE] + score = result[parser_constants.KEY_SCORE] + click.echo(f"Log Type: {log_type} , Score: {score}") + + if verbose: + api_utility.print_request_details( + classify_log_type_url, method, None, parsed_response) diff --git a/parsers/commands/classify_log_type_test.py b/parsers/commands/classify_log_type_test.py new file mode 100644 index 0000000..6b6c6da --- /dev/null +++ b/parsers/commands/classify_log_type_test.py @@ -0,0 +1,324 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Tests for classify_log_type.py.""" + +from unittest import mock + +from click import testing + +from google3.third_party.chronicle.cli import mock_test_utility +from parsers import url +from parsers.commands import classify_log_type +from parsers.tests.fixtures import * # pylint: disable=wildcard-import +from parsers.tests.fixtures import create_temp_log_file +from parsers.tests.fixtures import TEMP_SUBMIT_LOG_FILE + + +runner = testing.CliRunner() +RESOURCES = { + "project": "test_project", + "location": "us", + "instance": "test_instance", + "log_type": "test_log_type", +} +CLASSIFY_LOG_TYPE_URL = url.get_dataplane_url( + "us", "classify_log_type", "prod", RESOURCES +) + + +@mock.patch("time.time") +@mock.patch( + "common.chronicle_auth.initialize_dataplane_http_session" +) +@mock.patch("parsers.url.get_dataplane_url") +def test_classify_log_type( + mock_get_dataplane_url: mock.MagicMock, + mock_http_session: mock.MagicMock, + mock_time: mock.MagicMock, + test_data_classify_log_type: mock_test_utility.MockResponse, +) -> None: + """Test case to check success response. + + Args: + mock_get_dataplane_url (mock.MagicMock): Mock object + mock_http_session (mock.MagicMock): Mock object + mock_time (mock.MagicMock): Mock object + test_data_classify_log_type (mock_test_utility.MockResponse): Test input + data + """ + mock_time.return_value = 0.0 + create_temp_log_file(TEMP_SUBMIT_LOG_FILE, "test_log1\ntest_log2") + mock_get_dataplane_url.return_value = CLASSIFY_LOG_TYPE_URL + client = mock.Mock() + client.request.side_effect = [test_data_classify_log_type] + mock_http_session.return_value = client + result = runner.invoke( + classify_log_type.classify_log_type, + [ + "test_project", + "test_instance", + TEMP_SUBMIT_LOG_FILE, + "--v2", + "--env", + "PROD", + "--region", + "US", + ], + ) + assert """Classifying the provided log to the corresponding log types... + +Log Type: LOG_TYPE_1 , Score: 0.998 +Log Type: LOG_TYPE_2 , Score: 0.001 +Log Type: LOG_TYPE_3 , Score: 0.001 +""" == result.output + + +@mock.patch( + "common.chronicle_auth.initialize_dataplane_http_session" +) +@mock.patch("parsers.url.get_dataplane_url") +def test_classify_log_type_v2_flag_not_provided( + mock_get_dataplane_url: mock.MagicMock, + mock_http_session: mock.MagicMock, + test_v2flag_not_provided: mock_test_utility.MockResponse, +) -> None: + """Test case to check response for v2 flag not provided. + + Args: + mock_get_dataplane_url (mock.MagicMock): Mock object + mock_http_session (mock.MagicMock): Mock object + test_v2flag_not_provided (mock_test_utility.MockResponse): Test input data + """ + mock_get_dataplane_url.return_value = CLASSIFY_LOG_TYPE_URL + client = mock.Mock() + client.request.side_effect = [test_v2flag_not_provided] + mock_http_session.return_value = client + result = runner.invoke(classify_log_type.classify_log_type, []) + assert ( + "--v2 flag not provided. " + "Please provide the flag to run the new commands\n" + ) == result.output + + +@mock.patch( + "common.chronicle_auth.initialize_dataplane_http_session" +) +@mock.patch("parsers.url.get_dataplane_url") +def test_classify_log_type_empty_project_id( + mock_get_dataplane_url: mock.MagicMock, + mock_http_session: mock.MagicMock, + test_empty_project_id: mock_test_utility.MockResponse, +) -> None: + """Test case to check response for empty Project ID. + + Args: + mock_get_dataplane_url (mock.MagicMock): Mock object + mock_http_session (mock.MagicMock): Mock object + test_empty_project_id (mock_test_utility.MockResponse): Test input data + """ + mock_get_dataplane_url.return_value = CLASSIFY_LOG_TYPE_URL + client = mock.Mock() + client.request.side_effect = [test_empty_project_id] + mock_http_session.return_value = client + result = runner.invoke( + classify_log_type.classify_log_type, + ["--v2", "--env", "PROD", "--region", "US"], + ) + assert """Project ID not provided. Please enter Project ID +""" == result.output + + +@mock.patch( + "common.chronicle_auth.initialize_dataplane_http_session" +) +@mock.patch("parsers.url.get_dataplane_url") +def test_classify_log_type_empty_customer_id( + mock_get_dataplane_url: mock.MagicMock, + mock_http_session: mock.MagicMock, + test_empty_customer_id: mock_test_utility.MockResponse, +) -> None: + """Test case to check response for empty Customer ID. + + Args: + mock_get_dataplane_url (mock.MagicMock): Mock object + mock_http_session (mock.MagicMock): Mock object + test_empty_customer_id (mock_test_utility.MockResponse): Test input data + """ + mock_get_dataplane_url.return_value = CLASSIFY_LOG_TYPE_URL + client = mock.Mock() + client.request.side_effect = [test_empty_customer_id] + mock_http_session.return_value = client + result = runner.invoke( + classify_log_type.classify_log_type, + ["test_project", "--v2", "--env", "PROD", "--region", "US"], + ) + assert """Customer ID not provided. Please enter Customer ID +""" == result.output + + +@mock.patch( + "common.chronicle_auth.initialize_dataplane_http_session" +) +@mock.patch("parsers.url.get_dataplane_url") +def test_classify_log_type_non_existing_log_file( + mock_get_dataplane_url: mock.MagicMock, + mock_http_session: mock.MagicMock, + test_data_non_existing_log_file: mock_test_utility.MockResponse, +) -> None: + """Test case to check response for non existing log file. + + Args: + mock_get_dataplane_url (mock.MagicMock): Mock object + mock_http_session (mock.MagicMock): Mock object + test_data_non_existing_log_file (mock_test_utility.MockResponse): Test input + data + """ + mock_get_dataplane_url.return_value = CLASSIFY_LOG_TYPE_URL + client = mock.Mock() + client.request.side_effect = [test_data_non_existing_log_file] + mock_http_session.return_value = client + result = runner.invoke( + classify_log_type.classify_log_type, + [ + "test_project", + "test_instance", + "test_log_file", + "--v2", + "--env", + "PROD", + "--region", + "US", + ], + ) + assert """test_log_file does not exist. Please enter valid log file path +""" == result.output + + +@mock.patch( + "common.chronicle_auth.initialize_dataplane_http_session" +) +@mock.patch("parsers.url.get_dataplane_url") +def test_classify_log_type_empty_response( + mock_get_dataplane_url: mock.MagicMock, mock_http_session: mock.MagicMock +) -> None: + """Test case to check empty response. + + Args: + mock_get_dataplane_url (mock.MagicMock): Mock object + mock_http_session (mock.MagicMock): Mock object + """ + create_temp_log_file(TEMP_SUBMIT_LOG_FILE, "test_log1\ntest_log2") + mock_get_dataplane_url.return_value = CLASSIFY_LOG_TYPE_URL + client = mock.Mock() + client.request.side_effect = [ + mock_test_utility.MockResponse(status_code=200, text="""{}""") + ] + mock_http_session.return_value = client + result = runner.invoke( + classify_log_type.classify_log_type, + [ + "test_project", + "test_instance", + TEMP_SUBMIT_LOG_FILE, + "--v2", + "--env", + "PROD", + "--region", + "US", + ], + ) + assert """Classifying the provided log to the corresponding log types... + +No predictions found in the response. +""" == result.output + + +@mock.patch( + "common.chronicle_auth.initialize_dataplane_http_session" +) +@mock.patch("parsers.url.get_dataplane_url") +def test_classify_log_type_500( + mock_get_dataplane_url: mock.MagicMock, + mock_http_session: mock.MagicMock, + test_500_resp: mock_test_utility.MockResponse, +) -> None: + """Test case to check response for 500 response code. + + Args: + mock_get_dataplane_url (mock.MagicMock): Mock object + mock_http_session (mock.MagicMock): Mock object + test_500_resp (mock_test_utility.MockResponse): Test input data + """ + create_temp_log_file(TEMP_SUBMIT_LOG_FILE, "test_log1\ntest_log2") + mock_get_dataplane_url.return_value = CLASSIFY_LOG_TYPE_URL + client = mock.Mock() + client.request.side_effect = [test_500_resp] + mock_http_session.return_value = client + result = runner.invoke( + classify_log_type.classify_log_type, + [ + "test_project", + "test_instance", + TEMP_SUBMIT_LOG_FILE, + "--v2", + "--env", + "PROD", + "--region", + "US", + ], + ) + assert """Classifying the provided log to the corresponding log types... + +Error while classifying the logs. +Response Code: 500 +Error: test error +""" == result.output + + +@mock.patch( + "common.chronicle_auth.initialize_dataplane_http_session" +) +@mock.patch("parsers.url.get_dataplane_url") +def test_classify_log_type_exception( + mock_get_dataplane_url: mock.MagicMock, mock_http_session: mock.MagicMock +) -> None: + """Test case to verify console output in case of exception. + + Args: + mock_get_dataplane_url (mock.MagicMock): Mock object + mock_http_session (mock.MagicMock): Mock object + """ + create_temp_log_file(TEMP_SUBMIT_LOG_FILE, "test_log1\ntest_log2") + mock_get_dataplane_url.return_value = CLASSIFY_LOG_TYPE_URL + client = mock.Mock() + client.request.side_effect = Exception("test error message") + mock_http_session.return_value = client + result = runner.invoke( + classify_log_type.classify_log_type, + [ + "test_project", + "test_instance", + TEMP_SUBMIT_LOG_FILE, + "--v2", + "--env", + "PROD", + "--region", + "US", + ], + ) + assert """Classifying the provided log to the corresponding log types... + +Failed with exception: test error message +""" == result.output diff --git a/parsers/constants/key_constants.py b/parsers/constants/key_constants.py index 8be0bb2..3bbbfa9 100644 --- a/parsers/constants/key_constants.py +++ b/parsers/constants/key_constants.py @@ -77,6 +77,7 @@ KEY_PARSING_ERRORS = 'parsingErrors' KEY_PARSING_ERRORS = 'parsingErrors' KEY_PARSING_ERRORS = 'parsingErrors' +KEY_PREDICTIONS = 'predictions' KEY_PROJECT = 'project' KEY_PROJECTS = 'projects' KEY_RELEASE_STAGE = 'releaseStage' @@ -84,6 +85,7 @@ KEY_RUN_PARSER_RESULTS = 'runParserResults' KEY_SHA256 = 'sha256' KEY_SKIP_VALIDATION_ON_NO_LOGS = 'skipValidationOnNoLogs' +KEY_SCORE = 'score' KEY_SOURCE = 'source' KEY_START_TIME = 'start_time' KEY_STATE = 'state' diff --git a/parsers/parsers.py b/parsers/parsers.py index 3b62128..3da33b5 100644 --- a/parsers/parsers.py +++ b/parsers/parsers.py @@ -18,6 +18,7 @@ from parsers.commands import activate_parser from parsers.commands import archive +from parsers.commands import classify_log_type from parsers.commands import deactivate_parser from parsers.commands import delete_extension from parsers.commands import delete_parser @@ -46,6 +47,7 @@ def parsers() -> None: parsers.add_command(activate_parser.activate_parser) parsers.add_command(archive.archive) +parsers.add_command(classify_log_type.classify_log_type) parsers.add_command(deactivate_parser.deactivate_parser) parsers.add_command(delete_extension.delete_extension) parsers.add_command(delete_parser.delete_parser) diff --git a/parsers/parsers_test.py b/parsers/parsers_test.py index 066ff96..05a6f2b 100644 --- a/parsers/parsers_test.py +++ b/parsers/parsers_test.py @@ -28,6 +28,7 @@ def test_parser() -> None: expected_output = """Commands: activate_parser [New]Activate a parser archive Archives a parser given the config ID + classify_log_type [New]Classify the provided logs to the log types. deactivate_parser [New]Deactivate a parser delete_extension [New]Delete an extension delete_parser [New]Delete a parser diff --git a/parsers/tests/fixtures.py b/parsers/tests/fixtures.py index 2d4721e..ddafddd 100644 --- a/parsers/tests/fixtures.py +++ b/parsers/tests/fixtures.py @@ -143,6 +143,17 @@ def test_archive_data() -> MockResponse: "config": "test_config"}""") +@pytest.fixture() +def test_data_classify_log_type() -> MockResponse: + """Test response data.""" + return MockResponse( + status_code=200, + text="""{"predictions": [ + { "logType": "LOG_TYPE_1", "score": 0.998 }, + { "logType": "LOG_TYPE_2", "score": 0.001 }, + { "logType": "LOG_TYPE_3", "score": 0.001 }]}""") + + @pytest.fixture() def error_list() -> MockResponse: """Test input data.""" diff --git a/parsers/url.py b/parsers/url.py index b04b23f..3ae1e78 100644 --- a/parsers/url.py +++ b/parsers/url.py @@ -35,6 +35,7 @@ # Dataplane APIs 'activate_parser': f'{PARENT}/parsers/{{parser}}:activate', 'deactivate_parser': f'{PARENT}/parsers/{{parser}}:deactivate', + 'classify_log_type': 'projects/{project}/locations/{location}/instances/{instance}/logs:classify', 'delete_parser': f'{PARENT}/parsers/{{parser}}', 'delete_extension': f'{PARENT}/parserExtensions/{{parser_extension}}', 'get_parser': f'{PARENT}/parsers/{{parser}}',