From 46641d0c7ff202d62e74952382e1906b10634740 Mon Sep 17 00:00:00 2001 From: Dan Dye Date: Thu, 18 Jan 2024 12:55:34 -0800 Subject: [PATCH] v1alpha samples for: Create Ref List, Rule, UDM Event PiperOrigin-RevId: 599597439 --- common/project_id.py | 24 ++++ common/project_instance.py | 28 +++++ detect/v1alpha/create_rule.py | 134 +++++++++++++++++++++ detect/v1alpha/list_rules.py | 67 +++++++++++ ingestion/v1alpha/create_udm_events.py | 129 ++++++++++++++++++++ ingestion/v1alpha/get_udm_event.py | 97 +++++++++++++++ lists/v1alpha/create_list.py | 158 +++++++++++++++++++++++++ 7 files changed, 637 insertions(+) create mode 100644 common/project_id.py create mode 100644 common/project_instance.py create mode 100644 detect/v1alpha/create_rule.py create mode 100644 detect/v1alpha/list_rules.py create mode 100644 ingestion/v1alpha/create_udm_events.py create mode 100644 ingestion/v1alpha/get_udm_event.py create mode 100644 lists/v1alpha/create_list.py diff --git a/common/project_id.py b/common/project_id.py new file mode 100644 index 0000000..baedf03 --- /dev/null +++ b/common/project_id.py @@ -0,0 +1,24 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Support for Project ID for v1alpha Chronicle API calls.""" +import argparse + + +def add_argument_project_id(parser: argparse.ArgumentParser): + """Adds a shared command-line argument to all the sample modules.""" + parser.add_argument( + "-p", "--project_id", type=str, required=True, + help="Your BYOP, project id", + ) diff --git a/common/project_instance.py b/common/project_instance.py new file mode 100644 index 0000000..92f71dc --- /dev/null +++ b/common/project_instance.py @@ -0,0 +1,28 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Support for Project INSTANCE for v1alpha Chronicle API calls.""" + +import argparse + + +def add_argument_project_instance(parser: argparse.ArgumentParser): + """Adds a shared command-line argument to all the sample modules.""" + parser.add_argument( + "-g", + "--project_instance", + type=str, + required=True, + help="Customer ID for Chronicle instance", + ) diff --git a/detect/v1alpha/create_rule.py b/detect/v1alpha/create_rule.py new file mode 100644 index 0000000..a8d2768 --- /dev/null +++ b/detect/v1alpha/create_rule.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +r"""Executable and reusable sample for creating a detection rule. + + HTTP request + POST https://chronicle.googleapis.com/v1alpha/{parent}/rules + + python3 -m detect.v1alpha.create_rule \ + --project_instance $project_instance \ + --project_id $PROJECT_ID \ + --rule_file=./ip_in_abuseipdb_blocklist.yaral + + Requires the following IAM permission on the parent resource: + chronicle.rules.create + + API reference: + https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.rules/create + https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.rules#Rule +""" + +import argparse +import json +from typing import Any, Mapping + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + + +INGESTION_API_BASE_URL = "https://malachiteingestion-pa.googleapis.com" +AUTHORIZATION_SCOPES = ["https://www.googleapis.com/auth/malachite-ingestion"] + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + + +def create_rule( + http_session: requests.AuthorizedSession, + rule_file_path: str, +) -> Mapping[str, Any]: + """Creates a new detection rule to find matches in logs. + + Args: + http_session: Authorized session for HTTP requests. + rule_file_path: Content of the new detection rule, used to evaluate logs. + + Returns: + New detection rule. + + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + """ + parent = f"projects/{args.project_id}/locations/{args.region}/instances/{args.project_instance}" + url = f"https://{args.region}-chronicle.googleapis.com/v1alpha/{parent}/rules" + body = { + "text": rule_file_path.read(), + } + response = http_session.request("POST", url, json=body) + # Expected server response: + # { + # # pylint: disable=line-too-long + # "name": "projects/{project}/locations/{location}/instances/{instance}/rules/{rule_id}", + # "revisionId": "v_{10_digits}_{9_digits}", + # "displayName": "{rule_name}", + # "text": "{rule_content}", + # "author": str, + # "severity": { + # "displayName": str + # }, + # "metadata": { + # "{key_1}": "{value_1}", + # ... + # }, + # "createTime": "yyyy-MM-ddThh:mm:ss.ssssssZ", + # "revisionCreateTime": "yyyy-MM-ddThh:mm:ss.ssssssZ" + # "compilationState": "SUCCEEDED", + # "type": "{{SINGLE,MULTI}_EVENT,RULE_TYPE_UNSPECIFIED}", + # "referenceLists": [str], + # "allowedRunFrequencies": [ + # str, + # ... + # ], + # "etag": str + # } + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + return response.json() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # common + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + # local + parser.add_argument( + "-f", + "--rule_file", + type=argparse.FileType("r"), + required=True, + # File example: python3 create_rule.py -f + # STDIN example: cat rule.txt | python3 create_rule.py -f - + help="path of a file with the desired rule's content, or - for STDIN", + ) + args = parser.parse_args() + + auth_session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES + ) + new_rule = create_rule(auth_session, args.rule_file) + print(json.dumps(new_rule, indent=2)) diff --git a/detect/v1alpha/list_rules.py b/detect/v1alpha/list_rules.py new file mode 100644 index 0000000..28be05e --- /dev/null +++ b/detect/v1alpha/list_rules.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Executable and reusable sample for retrieving a list of rules.""" + +import argparse +import json +from typing import Mapping, Any + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + + +def list_rules(http_session: requests.AuthorizedSession) -> Mapping[str, Any]: + """Gets a list of rules. + + Args: + http_session: Authorized session for HTTP requests. + Returns: + Array containing each line of the feed's content. + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + """ + parent = f"projects/{args.project_id}/locations/{args.region}/instances/{args.project_instance}" + url = f"https://{args.region}-chronicle.googleapis.com/v1alpha/{parent}/rules" + + response = http_session.request("GET", url) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + return response.json() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + args = parser.parse_args() + session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES + ) + print(json.dumps(list_rules(session), indent=2)) diff --git a/ingestion/v1alpha/create_udm_events.py b/ingestion/v1alpha/create_udm_events.py new file mode 100644 index 0000000..4eac3f4 --- /dev/null +++ b/ingestion/v1alpha/create_udm_events.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +r"""Executable and reusable sample for ingesting events in UDM format. + + The Unified Data Model (UDM) is a way of representing events across all log + sources. See + https://cloud.google.com/chronicle/docs/unified-data-model/udm-field-list for a + description of UDM fields, and see + https://cloud.google.com/chronicle/docs/unified-data-model/format-events-as-udm + for how to describe a log as an event in UDM format. + + This command accepts a path to a file (--json_events_file) that contains an + array of JSON formatted events in UDM format. See + ./example_input/sample_udm_events.json for an example. + + So, assuming you've created a credentials file at $HOME/.chronicle_credentials.json, + and you are using environment variables for your PROJECT_INSTANCE and PROJECT_ID, + you can run this command using the sample input like so: + + Sample Command: + python3 -m ingestion.v1alpha.create_udm_events \ + --project_instance $PROJECT_INSTANCE \ + --project_id $PROJECT_ID \ + --json_events_file=./ingestion/example_input/sample_udm_events.json + + API reference: + https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.events/import + https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.events/import#EventsInlineSource + https://cloud.google.com/chronicle/docs/reference/udm-field-list + https://cloud.google.com/chronicle/docs/unified-data-model/udm-usage +""" + +import argparse +import json + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + + +INGESTION_API_BASE_URL = "https://malachiteingestion-pa.googleapis.com" +AUTHORIZATION_SCOPES = ["https://www.googleapis.com/auth/malachite-ingestion"] + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + + +def create_udm_events( + http_session: requests.AuthorizedSession, json_events: str +) -> None: + """Sends a collection of UDM events to the Chronicle backend for ingestion. + + A Unified Data Model (UDM) event is a structured representation of an event + regardless of the log source. + + Args: + http_session: Authorized session for HTTP requests. + json_events: A collection of UDM events in (serialized) JSON format. + + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + + Requires the following IAM permission on the parent resource: + chronicle.events.import + + POST https://chronicle.googleapis.com/v1alpha/{parent}/events:import + + https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.events/import + """ + parent = f"projects/{args.project_id}/locations/{args.region}/instances/{args.project_instance}" + url = f"https://{args.region}-chronicle.googleapis.com/v1alpha/{parent}/events:import" + + body = { + "inline_source": { + "events": [{ + "udm": json.loads(json_events)[0], + }] + } + } + + response = http_session.request("POST", url, json=body) + if response.status_code >= 400: + print(body) + print(response.text) + response.raise_for_status() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # common + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + # local + parser.add_argument( + "--json_events_file", + type=argparse.FileType("r"), + required=True, + help=( + 'path to a file (or "-" for STDIN) containing a list of UDM ' + "events in json format" + ), + ) + args = parser.parse_args() + auth_session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES, + ) + create_udm_events(auth_session, args.json_events_file.read()) diff --git a/ingestion/v1alpha/get_udm_event.py b/ingestion/v1alpha/get_udm_event.py new file mode 100644 index 0000000..c0a01cb --- /dev/null +++ b/ingestion/v1alpha/get_udm_event.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=line-too-long +r"""Executable and reusable v1alpha API sample for getting a UDM event by ID. + + API reference: + https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.events/get +""" +# pylint: enable=line-too-long + +import argparse +import json +from typing import Dict + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + +INGESTION_API_BASE_URL = "https://malachiteingestion-pa.googleapis.com" +AUTHORIZATION_SCOPES = ["https://www.googleapis.com/auth/malachite-ingestion"] + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + + +def get_udm_event( + http_session: requests.AuthorizedSession, event_id: str +) -> Dict[any]: + """Get a UDM event by metadata.id. + + A Unified Data Model (UDM) event is a structured representation of an event + regardless of the log source. + + Args: + http_session: Authorized session for HTTP requests. + event_id: URL-encoded Base64 for the UDM Event ID. + Returns: + dict/json respresentation of UDM Event + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + + Requires the following IAM permission on the parent resource: + chronicle.events.get + + https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.events/get + """ + parent = f"projects/{args.project_id}/locations/{args.region}/instances/{args.project_instance}" + url = f"https://{args.region}-chronicle.googleapis.com/v1alpha/{parent}/events/{event_id}" + + response = http_session.request("GET", url) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + return response.json() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # common + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + # local + parser.add_argument( + "--event_id", + type=str, + required=True, + help=("URL-encoded Base64 ID of the Event"), + ) + args = parser.parse_args() + + auth_session = chronicle_auth.initialize_http_session( + args.credentials_file, + SCOPES, + ) + event = get_udm_event(auth_session, args.event_id) + print(json.dumps(event, indent=2)) diff --git a/lists/v1alpha/create_list.py b/lists/v1alpha/create_list.py new file mode 100644 index 0000000..2677d73 --- /dev/null +++ b/lists/v1alpha/create_list.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pylint: disable=line-too-long +"""Executable and reusable sample for creating a Reference List. + + Requires the following IAM permission on the parent resource: + chronicle.referenceLists.create + + https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.referenceLists/create +""" +# pylint: enable=line-too-long + +import argparse +from typing import Sequence + +from google.auth.transport import requests + +from common import chronicle_auth +from common import project_id +from common import project_instance +from common import regions + + +SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", +] + +PREFIX = "REFERENCE_LIST_SYNTAX_TYPE_" +SYNTAX_TYPE_ENUM = [ + f"{PREFIX}UNSPECIFIED", # Defaults to ..._PLAIN_TEXT_STRING. + f"{PREFIX}PLAIN_TEXT_STRING", # List contains plain text patterns. + f"{PREFIX}REGEX", # List contains only Regular Expression patterns. + f"{PREFIX}CIDR", # List contains only CIDR patterns. +] + + +def create_list( + http_session: requests.AuthorizedSession, + name: str, + description: str, + content_lines: Sequence[str], + content_type: str, +) -> str: + """Creates a list. + + Args: + http_session: Authorized session for HTTP requests. + name: Unique name for the list. + description: Description of the list. + content_lines: Array containing each line of the list's content. + content_type: Type of list content, indicating how to interpret this list. + + Returns: + Creation timestamp of the new list. + + Raises: + requests.exceptions.HTTPError: HTTP request resulted in an error + (response.status_code >= 400). + """ + + # pylint: disable=line-too-long + parent = f"projects/{args.project_id}/locations/{args.region}/instances/{args.project_instance}" + url = f"https://{args.region}-chronicle.googleapis.com/v1alpha/{parent}/referenceLists" + # pylint: enable=line-too-long + + # Test auth with a Get List + response = http_session.request("GET", url) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + + # entries are list like [{"value": }, ...] + # pylint: disable-next=line-too-long + # https://cloud.google.com/chronicle/docs/reference/rest/v1alpha/projects.locations.instances.referenceLists#resource:-referencelist + entries = [] + for content_line in content_lines: + entries.append({"value": content_line.strip()}) + + body = { + "name": name, + "description": description, + "entries": entries, + "syntax_type": content_type, + } + url_w_query_string = f"{url}?referenceListId={name}" + response = http_session.request("POST", url_w_query_string, json=body) + # Expected server response: + # ['name', 'displayName', 'revisionCreateTime', 'description', + # 'entries', 'syntaxType']) + if response.status_code >= 400: + print(response.text) + response.raise_for_status() + return response.json()["revisionCreateTime"] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + chronicle_auth.add_argument_credentials_file(parser) + project_instance.add_argument_project_instance(parser) + project_id.add_argument_project_id(parser) + regions.add_argument_region(parser) + parser.add_argument( + "-n", "--name", type=str, required=True, help="unique name for the list" + ) + parser.add_argument( + "-d", + "--description", + type=str, + required=True, + help="description of the list", + ) + parser.add_argument( + "-t", + "--syntax_type", + type=str, + required=False, + default="REFERENCE_LIST_SYNTAX_TYPE_PLAIN_TEXT_STRING", + choices=SYNTAX_TYPE_ENUM, + # pylint: disable-next=line-too-long + help="syntax type of the list, used for validation (default: REFERENCE_LIST_SYNTAX_TYPE_PLAIN_TEXT_STRING)", + ) + parser.add_argument( + "-f", + "--list_file", + type=argparse.FileType("r"), + required=True, + # File example: + # python3 -m lists.v1alpha.create_list -f + # STDIN example: + # cat | python3 -m lists.v1alpha.create_list -f - + help="path of a file containing the list content, or - for STDIN", + ) + args = parser.parse_args() + + # pylint: disable-next=line-too-long + auth_session = chronicle_auth.initialize_http_session(args.credentials_file, SCOPES) + new_list_create_time = create_list( + auth_session, + args.name, + args.description, + args.list_file.read().splitlines(), + args.syntax_type, + ) + print(f"New list created successfully, at {new_list_create_time}")