From c246fa7d4e372a6f611fd46c0acd96b3b54a3676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Thu, 24 Oct 2024 13:11:43 -0400 Subject: [PATCH 01/20] Fix a bug where App_Base.getSplunkbasePath() was broken and did not work at all. Fixes #295. --- contentctl/objects/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 9057a4c4..c50ab900 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -47,7 +47,7 @@ class App_Base(BaseModel,ABC): def getSplunkbasePath(self)->HttpUrl: - return HttpUrl(SPLUNKBASE_URL.format(uid=self.uid, release=self.version)) + return HttpUrl(SPLUNKBASE_URL.format(uid=self.uid, version=self.version)) @abstractmethod def getApp(self, config:test, stage_file:bool=False)->str: From f7a09637f4382f79cc6c6d3c779fdcdc6b71b843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Thu, 24 Oct 2024 13:47:38 -0400 Subject: [PATCH 02/20] Fix a bug where the jinja template for `analyticsories_detection` crashed when specifying a `detection.tags.asset_type`. Fixes #313. --- contentctl/output/templates/analyticstories_detections.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/output/templates/analyticstories_detections.j2 b/contentctl/output/templates/analyticstories_detections.j2 index e97f82a8..d24a1217 100644 --- a/contentctl/output/templates/analyticstories_detections.j2 +++ b/contentctl/output/templates/analyticstories_detections.j2 @@ -5,7 +5,7 @@ {% if (detection.type == 'TTP' or detection.type == 'Anomaly' or detection.type == 'Hunting' or detection.type == 'Correlation') %} [savedsearch://{{ detection.get_conf_stanza_name(app) }}] type = detection -asset_type = {{ detection.tags.asset_type.value }} +asset_type = {{ detection.tags.asset_type }} confidence = medium explanation = {{ (detection.explanation if detection.explanation else detection.description) | escapeNewlines() }} {% if detection.how_to_implement is defined %} From ea34bb1459bf249d1d68cd879b293a2cfae3d1be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Thu, 24 Oct 2024 15:45:58 -0400 Subject: [PATCH 03/20] Do not serialize TestApp.hardcoded_path if it has value None. Fixes #319. --- contentctl/objects/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 9057a4c4..6f5ee070 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -65,7 +65,7 @@ class TestApp(App_Base): hardcoded_path: Optional[Union[FilePath,HttpUrl]] = Field(default=None, description="This may be a relative or absolute link to a file OR an HTTP URL linking to your app.") - @field_serializer('hardcoded_path',when_used='always') + @field_serializer('hardcoded_path',when_used='unless-none') def serialize_path(path: Union[AnyUrl, pathlib.Path])->str: return str(path) From 62f81f143d4dc39b9d4148e13480a202c8f3f11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Thu, 24 Oct 2024 20:20:30 +0000 Subject: [PATCH 04/20] Merged PR 112820: add azure-pipelines.yml add azure-pipelines.yml --- azure-pipelines.yml | 89 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..850f870b --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,89 @@ +variables: + package: $(Build.Repository.Name) + srcDirectory: $(package) + python.version: '3.12' + artifactName: dist + +# Trigger only when simple_package or its build has been modified +trigger: + branches: + include: + - "main" + +# Jobs are collections of related steps +jobs: + +- job: Build + + pool: + vmImage: 'ubuntu-latest' + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + + - script: | + python -m pip install --upgrade pip wheel setuptools + displayName: 'Install build dependencies' + + # Our built wheel will land in dist + - bash: python setup.py bdist_wheel --universal + displayName: Build package + + # Upload everything in src/simple_package/dist (including subfolders) to the build artifacts for later use or debugging + # Add pythonVersion to the artifact name to avoid conflicts and ensure we capture all build output + - task: PublishPipelineArtifact@0 + displayName: Publish artifacts + inputs: + artifactName: 'dist' + targetPath: dist + + +# If all Build steps for all Python versions have succeeded, +# we will download one of the already-validated build assets and publish it to an Azure Artifacts feed +- job: Publish + + # Run on a Microsoft-hosted agent running Ubuntu Latest + # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops + pool: + vmImage: 'ubuntu-latest' + + # We want to wait for all Build Jobs to complete before running the Publish Job + dependsOn: Build + + # Only publish when the previous Jobs are successful and we're on main + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) + + # Steps are the specific tasks that execute code and do things + steps: + + # Explicitly disable source checkout to keep a pristine environment for publishing + - checkout: none + + # Download from build artifacts + - download: current + artifact: $(artifactName) + + # Set the version of Python to use for publishing (which may or may not match the version the package was built with or tested against) + - task: UsePythonVersion@0 + displayName: Use Python $(python.version) + inputs: + versionSpec: $(python.version) + + # Install tools needed for publishing + - bash: python -m pip install twine + displayName: Install twine + + # Authenticate to Azure Artifacts + # This sets the PYPIRC_PATH environment variable, which contains credentials for the feed + - task: TwineAuthenticate@0 + displayName: Configure twine authentication + inputs: + artifactFeeds: TH-Investigation/TH-Investigation + + # Upload everything in the dist folder to the private Artifacts feed + - bash: twine upload -r TH-Investigation/TH-Investigation --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/$(artifactName)/*.whl + displayName: Publish artifacts + From 792f3cc6f5cf0d01ba27bfc8656268c64beea9c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Thu, 24 Oct 2024 20:23:26 +0000 Subject: [PATCH 05/20] Merged PR 112823: fix --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 850f870b..949e49a8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,7 +29,7 @@ jobs: displayName: 'Install build dependencies' # Our built wheel will land in dist - - bash: python setup.py bdist_wheel --universal + - bash: python -m build displayName: Build package # Upload everything in src/simple_package/dist (including subfolders) to the build artifacts for later use or debugging From 8bf35a8bd768d52da3832f369e2144d6a922134e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Thu, 24 Oct 2024 20:26:20 +0000 Subject: [PATCH 06/20] Merged PR 112825: fix azure-pipelines.yml fix azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 949e49a8..874caf00 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,7 +25,7 @@ jobs: displayName: 'Use Python $(python.version)' - script: | - python -m pip install --upgrade pip wheel setuptools + python -m pip install --upgrade pip wheel setuptools build displayName: 'Install build dependencies' # Our built wheel will land in dist From 438d00f70b65a3e59050c5cede03a571867bbbfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Thu, 24 Oct 2024 20:30:27 +0000 Subject: [PATCH 07/20] Merged PR 112827: fix build fix build --- pyproject.toml | 4 ++-- setup.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index ef0803af..46b0e27d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,5 +32,5 @@ setuptools = ">=69.5.1,<76.0.0" [tool.poetry.dev-dependencies] [build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" +requires = ["setuptools>=70.3.0", "calver==2022.6.26"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..ab7494c5 --- /dev/null +++ b/setup.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from setuptools import setup + + +setup( + use_calver="%Y%m%d.%H.%M", + setup_requires=['calver==2022.6.26'] +) From 8bb94c60cfc14952bf6803bd1819703fc7125b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Thu, 24 Oct 2024 20:45:40 +0000 Subject: [PATCH 08/20] Merged PR 112831: fix build fix build --- pyproject.toml | 50 ++++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 46b0e27d..23b76dc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,35 +1,33 @@ -[tool.poetry] +[project] +dynamic = ["version"] name = "contentctl" -version = "4.4.1" - description = "Splunk Content Control Tool" -authors = ["STRT "] -license = "Apache 2.0" readme = "README.md" -[tool.poetry.scripts] +dependencies = [ + "pydantic>=2.8.2,<3", + "PyYAML>=6.0.2,<7", + "requests>=2.32.3,<3", + "pycvesearch>=1.2,<2", + "xmltodict>=0.13,<0.15", + "attackcti>=0.4.0,<1", + "Jinja2>=3.1.4,<4", + "questionary>=2.0.1,<3", + "docker>=7.1.0,<8", + "splunk-sdk>=2.0.2,<3", + "semantic-version>=2.10.0,<3", + "bottle>=0.12.25,<0.14.0", + "tqdm>=4.66.5,<5", + "pygit2>=1.15.1,<2", + "tyro>=0.8.3,<1", + "gitpython>=3.1.43,<4", +] + +[project.scripts] contentctl = 'contentctl.contentctl:main' -[tool.poetry.dependencies] -python = "^3.11" -pydantic = "^2.8.2" -PyYAML = "^6.0.2" -requests = "~2.32.3" -pycvesearch = "^1.2" -xmltodict = ">=0.13,<0.15" -attackcti = "^0.4.0" -Jinja2 = "^3.1.4" -questionary = "^2.0.1" -docker = "^7.1.0" -splunk-sdk = "^2.0.2" -semantic-version = "^2.10.0" -bottle = ">=0.12.25,<0.14.0" -tqdm = "^4.66.5" -pygit2 = "^1.15.1" -tyro = "^0.8.3" -gitpython = "^3.1.43" -setuptools = ">=69.5.1,<76.0.0" -[tool.poetry.dev-dependencies] +[project.urls] +Repository = "https://dev.azure.com/mvtdesjardins/TH-Investigation/_git/contentctl" [build-system] requires = ["setuptools>=70.3.0", "calver==2022.6.26"] From 801930dffa133a3fd22f184982d60da5b57ffb1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Fri, 25 Oct 2024 14:15:51 +0000 Subject: [PATCH 09/20] Merged PR 112907: fix build fix build --- .../infrastructures/DetectionTestingInfrastructure.py | 2 +- contentctl/objects/config.py | 2 +- pyproject.toml | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 8e816025..d443ea22 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -1358,7 +1358,7 @@ def hec_raw_replay( url_with_hec_path = urllib.parse.urljoin( url_with_port, "services/collector/raw" ) - with open(tempfile, "rb") as datafile: + with open(tempfile, "r") as datafile: try: res = requests.post( url_with_hec_path, diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 6f5ee070..cbf6929f 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -47,7 +47,7 @@ class App_Base(BaseModel,ABC): def getSplunkbasePath(self)->HttpUrl: - return HttpUrl(SPLUNKBASE_URL.format(uid=self.uid, release=self.version)) + return HttpUrl(SPLUNKBASE_URL.format(uid=self.uid, version=self.version)) @abstractmethod def getApp(self, config:test, stage_file:bool=False)->str: diff --git a/pyproject.toml b/pyproject.toml index 23b76dc4..3eacdf0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,13 @@ contentctl = 'contentctl.contentctl:main' [project.urls] Repository = "https://dev.azure.com/mvtdesjardins/TH-Investigation/_git/contentctl" +[tool.setuptools] + +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] + [build-system] requires = ["setuptools>=70.3.0", "calver==2022.6.26"] build-backend = "setuptools.build_meta" \ No newline at end of file From b6eadbf4797b2a439d585da3d2c7c8a4655d24ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Fri, 25 Oct 2024 15:07:43 +0000 Subject: [PATCH 10/20] Merged PR 112936: fix build :) --- pyproject.toml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3eacdf0f..d760067e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,12 +30,17 @@ contentctl = 'contentctl.contentctl:main' Repository = "https://dev.azure.com/mvtdesjardins/TH-Investigation/_git/contentctl" [tool.setuptools] - -include-package-data = true +#include-package-data = true [tool.setuptools.packages.find] where = ["."] +[tool.setuptools.package-data] +"contentctl" = ["**/*.*"] + +[tool.setuptools.exclude-package-data] +"contentctl" = ["**/__pycache__/**"] + [build-system] -requires = ["setuptools>=70.3.0", "calver==2022.6.26"] +requires = ["setuptools>=75.2.0", "calver==2022.6.26"] build-backend = "setuptools.build_meta" \ No newline at end of file From f74ae37c6ec9e3b5da84029cb3afade71cca48f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Mon, 4 Nov 2024 20:34:30 +0000 Subject: [PATCH 11/20] Merged PR 114299: Allow test.splunk_api_username and test.splunk_api_password to be specified w... Allow test.splunk_api_username and test.splunk_api_password to be specified with environment variables SPLUNKBASE_USERNAME and SPLUNKBASE_PASSWORD to avoid putting credentials in YAML files. --- contentctl/objects/config.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index cbf6929f..4228ba98 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from os import environ from datetime import datetime, UTC from typing import Optional, Any, List, Union, Self @@ -830,10 +831,17 @@ class test(test_common): model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) container_settings:ContainerSettings = ContainerSettings() test_instances: List[Container] = Field([], exclude = True, validate_default=True) - splunk_api_username: Optional[str] = Field(default=None, exclude = True,description="Splunk API username used for running appinspect or installating apps from Splunkbase") - splunk_api_password: Optional[str] = Field(default=None, exclude = True, description="Splunk API password used for running appinspect or installaing apps from Splunkbase") - - + splunk_api_username: Optional[str] = Field(default=None, exclude = True,description="Splunk API username used for running appinspect or installating apps from Splunkbase. Can be replaced by the 'SPLUNKBASE_USERNAME' environment variable.") + splunk_api_password: Optional[str] = Field(default=None, exclude = True, description="Splunk API password used for running appinspect or installaing apps from Splunkbase. Can be replaced by the 'SPLUNKBASE_PASSWORD' environment variable.") + + def __init__(self, **kwargs): + if "SPLUNKBASE_USERNAME" in os.environ: + breakpoint() + kwargs['splunk_api_username'] = os.environ["SPLUNKBASE_USERNAME"] + if "SPLUNKBASE_PASSWORD" in os.environ: + kwargs['splunk_api_password'] = os.environ["SPLUNKBASE_PASSWORD"] + super().__init__(**kwargs) + def getContainerInfrastructureObjects(self)->Self: try: self.test_instances = self.container_settings.getContainers() From e33011560e8f12ece38609189a524ae90e19f369 Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Mon, 4 Nov 2024 15:47:36 -0800 Subject: [PATCH 12/20] Manually port changed raised by Res260. --- contentctl/objects/config.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index cbf6929f..7c9afb52 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -830,10 +830,18 @@ class test(test_common): model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True) container_settings:ContainerSettings = ContainerSettings() test_instances: List[Container] = Field([], exclude = True, validate_default=True) - splunk_api_username: Optional[str] = Field(default=None, exclude = True,description="Splunk API username used for running appinspect or installating apps from Splunkbase") - splunk_api_password: Optional[str] = Field(default=None, exclude = True, description="Splunk API password used for running appinspect or installaing apps from Splunkbase") + splunk_api_username: None | str = Field(default=None, exclude = True,description="Splunk API username used for running appinspect or installating apps from Splunkbase. Can be replaced by the 'SPLUNKBASE_USERNAME' environment variable.") + splunk_api_password: None | str = Field(default=None, exclude = True, description="Splunk API password used for running appinspect or installaing apps from Splunkbase. Can be replaced by the 'SPLUNKBASE_PASSWORD' environment variable.") + + def __init__(self, **kwargs: Any): + if "SPLUNKBASE_USERNAME" in environ: + kwargs['splunk_api_username'] = environ["SPLUNKBASE_USERNAME"] + if "SPLUNKBASE_PASSWORD" in environ: + kwargs['splunk_api_password'] = environ["SPLUNKBASE_PASSWORD"] + super().__init__(**kwargs) + def getContainerInfrastructureObjects(self)->Self: try: self.test_instances = self.container_settings.getContainers() From 5bcfb6c49fc42b121a4567837a35d5358af71bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Wed, 6 Nov 2024 01:43:14 +0000 Subject: [PATCH 13/20] =?UTF-8?q?Merged=20PR=20114606:=20Fix=20du=20setup?= =?UTF-8?q?=20initial=20+=20t=C3=A9l=C3=A9chargement=20depuis=20splunkbase?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix du setup initial + téléchargement depuis splunkbase. --- .../DetectionTestingInfrastructure.py | 44 +++++++++++++++++-- ...DetectionTestingInfrastructureContainer.py | 5 +++ contentctl/objects/config.py | 6 +-- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index d443ea22..8bb1f1eb 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -1,3 +1,4 @@ +import logging import time import uuid import abc @@ -17,12 +18,13 @@ import requests # type: ignore import splunklib.client as client # type: ignore from splunklib.binding import HTTPError # type: ignore +from splunklib.client import Service from splunklib.results import JSONResultsReader, Message # type: ignore import splunklib.results from urllib3 import disable_warnings import urllib.parse -from contentctl.objects.config import test_common, Infrastructure +from contentctl.objects.config import test_common, Infrastructure, ENTERPRISE_SECURITY_UID from contentctl.objects.enums import PostTestBehavior, AnalyticsType from contentctl.objects.detection import Detection from contentctl.objects.base_test import BaseTest @@ -42,6 +44,12 @@ TestingStates ) +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) +handler = logging.StreamHandler() +handler.setLevel(logging.DEBUG) +LOG.addHandler(handler) + class SetupTestGroupResults(BaseModel): exception: Union[Exception, None] = None @@ -107,6 +115,7 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC): def __init__(self, **data): super().__init__(**data) + self._conn: Optional[Service] = None # TODO: why not use @abstractmethod def start(self): @@ -138,7 +147,8 @@ def setup(self): try: for func, msg in [ (self.start, "Starting"), - (self.get_conn, "Waiting for App Installation"), + (self.get_conn, "Getting initial connection"), + (self.wait_for_app_installation, "Waiting for App Installation"), (self.configure_conf_file_datamodels, "Configuring Datamodels"), (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"), (self.get_all_indexes, "Getting all indexes from server"), @@ -210,6 +220,29 @@ def get_all_indexes(self) -> None: except Exception as e: raise (Exception(f"Failure getting indexes: {str(e)}")) + def wait_for_app_installation(self): + config_apps = self.global_config.apps + installed_config_apps = [] + while len(installed_config_apps) < len(config_apps): + try: + # Get apps installed in the Splunk instance + splunk_instance_apps = self.get_conn().apps.list() + + # Try to find all the apps we want to be installed (config_apps) + installed_config_apps = [] + for config_app in config_apps: + for splunk_instance_app in splunk_instance_apps: + if config_app.appid == splunk_instance_app.name: + # For Enterprise Security, we need to make sure the app is also configured. + if config_app.uid == ENTERPRISE_SECURITY_UID and splunk_instance_app.content.get('configured') != '1': + continue + installed_config_apps.append(config_app.appid) + LOG.debug("Apps in the Splunk instance: " + str(list(map(lambda x: x.name, splunk_instance_apps)))) + LOG.debug("apps in contentctl package found in Splunk instance: " + installed_config_apps) + except Exception as e: + LOG.exception(e) + time.sleep(5) + def get_conn(self) -> client.Service: try: if not self._conn: @@ -218,9 +251,11 @@ def get_conn(self) -> client.Service: # continue trying to re-establish a connection until after # the server has restarted self.connect_to_api() - except Exception: + except Exception as e: # there was some issue getting the connection. Try again just once + LOG.exception(e) self.connect_to_api() + return self._conn def check_for_teardown(self): @@ -295,7 +330,7 @@ def configure_imported_roles( ): try: # Set which roles should be configured. For Enterprise Security/Integration Testing, - # we must add some extra foles. + # we must add some extra roles. if self.global_config.enable_integration_testing: roles = imported_roles + enterprise_security_roles else: @@ -334,6 +369,7 @@ def wait_for_conf_file(self, app_name: str, conf_file_name: str): self.check_for_teardown() time.sleep(1) try: + print('woahhhhh') _ = self.get_conn().get( f"configs/conf-{conf_file_name}", app=app_name ) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py index f5887033..37326f94 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py @@ -1,3 +1,5 @@ +import time + from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import ( DetectionTestingInfrastructure, ) @@ -25,6 +27,9 @@ def start(self): self.container = self.make_container() self.container.start() + # There might be a small delay between the starting of the container and the binding of the ports for splunk. + # To avoid a "connection refused" error, wait a little bit before finishing the method call. + time.sleep(20) def finish(self): if self.container is not None: diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 4228ba98..1931c151 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -31,7 +31,8 @@ ENTERPRISE_SECURITY_UID = 263 COMMON_INFORMATION_MODEL_UID = 1621 -SPLUNKBASE_URL = "https://splunkbase.splunk.com/app/{uid}/release/{version}/download" +SPLUNKBASE_BASE_URL = "https://splunkbase.splunk.com" +SPLUNKBASE_URL = SPLUNKBASE_BASE_URL + "/app/{uid}/release/{version}/download" # TODO (#266): disable the use_enum_values configuration @@ -836,7 +837,6 @@ class test(test_common): def __init__(self, **kwargs): if "SPLUNKBASE_USERNAME" in os.environ: - breakpoint() kwargs['splunk_api_username'] = os.environ["SPLUNKBASE_USERNAME"] if "SPLUNKBASE_PASSWORD" in os.environ: kwargs['splunk_api_password'] = os.environ["SPLUNKBASE_PASSWORD"] @@ -886,7 +886,7 @@ def getContainerEnvironmentString(self,stage_file:bool=False, include_custom_app container_paths = [] for path in paths: - if path.startswith(SPLUNKBASE_URL): + if path.startswith(SPLUNKBASE_BASE_URL): container_paths.append(path) else: container_paths.append((self.getContainerAppDir()/pathlib.Path(path).name).as_posix()) From e658e7dd7c0039c867a306b893d998e06cf10aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Wed, 6 Nov 2024 16:42:04 +0000 Subject: [PATCH 14/20] Merged PR 114699: fix typo fix typo --- .../infrastructures/DetectionTestingInfrastructure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 8bb1f1eb..a4bfcbba 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -238,7 +238,7 @@ def wait_for_app_installation(self): continue installed_config_apps.append(config_app.appid) LOG.debug("Apps in the Splunk instance: " + str(list(map(lambda x: x.name, splunk_instance_apps)))) - LOG.debug("apps in contentctl package found in Splunk instance: " + installed_config_apps) + LOG.debug(f"apps in contentctl package found in Splunk instance: {installed_config_apps}") except Exception as e: LOG.exception(e) time.sleep(5) From ccd62b697c3490f7d18ca80aa1a8295ffda67bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Wed, 6 Nov 2024 18:57:47 +0000 Subject: [PATCH 15/20] Merged PR 114746: fix tests maybe fix tests maybe --- .../infrastructures/DetectionTestingInfrastructure.py | 4 ++-- contentctl/objects/unit_test_result.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index a4bfcbba..beb226d2 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -369,7 +369,6 @@ def wait_for_conf_file(self, app_name: str, conf_file_name: str): self.check_for_teardown() time.sleep(1) try: - print('woahhhhh') _ = self.get_conn().get( f"configs/conf-{conf_file_name}", app=app_name ) @@ -1136,7 +1135,7 @@ def retry_search_until_timeout( threat_object_fields_set = set([o.name for o in detection.tags.observable if "Attacker" in o.role]) # just the "threat objects" # Ensure the search had at least one result - if int(job.content.get("resultCount", "0")) > 0: + if int(job.content["resultCount"]) > 0: # Initialize the test result test.result = UnitTestResult() @@ -1239,6 +1238,7 @@ def retry_search_until_timeout( self.infrastructure, TestResultStatus.FAIL, duration=time.time() - search_start_time, + message=f"Search had 0 result. {job.content}" ) tick += 1 diff --git a/contentctl/objects/unit_test_result.py b/contentctl/objects/unit_test_result.py index 8c40da10..d7ab0bd0 100644 --- a/contentctl/objects/unit_test_result.py +++ b/contentctl/objects/unit_test_result.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Union,TYPE_CHECKING +from typing import Union, TYPE_CHECKING, Optional from splunklib.data import Record from contentctl.objects.base_test_result import BaseTestResult, TestResultStatus @@ -23,6 +23,7 @@ def set_job_content( status: TestResultStatus, exception: Union[Exception, None] = None, duration: float = 0, + message: Optional[str] = None ) -> bool: """ Sets various fields in the result, pulling some fields from the provided search job's @@ -75,4 +76,8 @@ def set_job_content( self.message = f"ERROR with no more specific message available." self.sid_link = NO_SID + if message: + # Override the message + self.message = message + return self.success From 929a2e3e5a891bfeea653169d4ca70415ff5df7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Wed, 6 Nov 2024 21:36:10 +0000 Subject: [PATCH 16/20] Merged PR 114823: Add support for dynamically creating custom_index for tests Add support for dynamically creating custom_index for tests --- .../DetectionTestingInfrastructure.py | 33 ++++++++----------- contentctl/contentctl.py | 5 +-- contentctl/helper/utils.py | 12 +++++++ 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index beb226d2..a109ab9c 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -44,11 +44,7 @@ TestingStates ) -LOG = logging.getLogger(__name__) -LOG.setLevel(logging.DEBUG) -handler = logging.StreamHandler() -handler.setLevel(logging.DEBUG) -LOG.addHandler(handler) +LOG = Utils.get_logger() class SetupTestGroupResults(BaseModel): @@ -180,13 +176,10 @@ def wait_for_ui_ready(self): def configure_hec(self): self.hec_channel = str(uuid.uuid4()) try: - res = self.get_conn().input( - path="/servicesNS/nobody/splunk_httpinput/data/inputs/http/http:%2F%2FDETECTION_TESTING_HEC" - ) - self.hec_token = str(res.token) - return - except Exception: - # HEC input does not exist. That's okay, we will create it + # Delete old HEC + self.get_conn().inputs.delete("DETECTION_TESTING_HEC", kind='http') + except HTTPError as e: + # HEC input didn't exist in the first place, everything is good. pass try: @@ -1283,16 +1276,16 @@ def replay_attack_data_file( test_group_start_time: float, ): # Before attempting to replay the file, ensure that the index we want - # to replay into actuall exists. If not, we should throw a detailed - # exception that can easily be interpreted by the user. + # to replay into actually exists. If not, we create the index. if attack_data_file.custom_index is not None and \ attack_data_file.custom_index not in self.all_indexes_on_server: - raise ReplayIndexDoesNotExistOnServer( - f"Unable to replay data file {attack_data_file.data} " - f"into index '{attack_data_file.custom_index}'. " - "The index does not exist on the Splunk Server. " - f"The only valid indexes on the server are {self.all_indexes_on_server}" - ) + index = self.get_conn().indexes.create(name=attack_data_file.custom_index) + LOG.debug(f"Created Index {attack_data_file.custom_index}: {index}") + LOG.debug("Re-retup of the HEC and roles and indexes...") + self.get_all_indexes() + self.configure_imported_roles() + self.configure_delete_indexes() + self.configure_hec() tempfile = mktemp(dir=tmp_dir) if not (str(attack_data_file.data).startswith("http://") or diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index dbf434a7..1a1e001b 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -5,6 +5,7 @@ import tyro from contentctl.actions.initialize import Initialize +from contentctl.helper.utils import Utils from contentctl.objects.config import init, validate, build, new, deploy_acs, test, test_servers, inspect, report, test_common, release_notes from contentctl.actions.validate import Validate from contentctl.actions.new_content import NewContent @@ -51,8 +52,7 @@ # """ # ) - - +LOG = Utils.get_logger() def init_func(config:test): Initialize().execute(config) @@ -154,6 +154,7 @@ def main(): t = test.model_validate(config_obj) except Exception as e: print(f"Error validating 'contentctl.yml':\n{str(e)}") + LOG.exception(e) sys.exit(1) diff --git a/contentctl/helper/utils.py b/contentctl/helper/utils.py index 261ecb64..61f9b18d 100644 --- a/contentctl/helper/utils.py +++ b/contentctl/helper/utils.py @@ -1,3 +1,4 @@ +import logging import os import git import shutil @@ -19,6 +20,13 @@ TOTAL_BYTES = 0 ALWAYS_PULL = True +LOG = logging.getLogger("main") +LOG.setLevel(logging.DEBUG) +handler = logging.StreamHandler() +handler.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - [%(levelname)-8s] %(message)s') +handler.setFormatter(formatter) +LOG.addHandler(handler) class Utils: @@ -485,3 +493,7 @@ def getPercent(numerator: float, denominator: float, decimal_places: int) -> str ratio = numerator / denominator percent = ratio * 100 return Utils.getFixedWidth(percent, decimal_places) + "%" + + @staticmethod + def get_logger() -> logging.Logger: + return LOG From 9b3dcc8185fc479acbffd1af9b83a070b3f82320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Wed, 6 Nov 2024 21:59:52 +0000 Subject: [PATCH 17/20] Merged PR 114830: fix fix --- .../infrastructures/DetectionTestingInfrastructure.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index a109ab9c..087cec73 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -178,9 +178,13 @@ def configure_hec(self): try: # Delete old HEC self.get_conn().inputs.delete("DETECTION_TESTING_HEC", kind='http') - except HTTPError as e: + except (HTTPError, KeyError) as e: # HEC input didn't exist in the first place, everything is good. pass + except Exception as e: + LOG.error("Error when deleting input DETECTION_TESTING_HEC.") + LOG.exception(e) + raise e try: res = self.get_conn().inputs.create( @@ -191,10 +195,11 @@ def configure_hec(self): useACK=True, ) self.hec_token = str(res.token) + breakpoint() return except Exception as e: - raise (Exception(f"Failure creating HEC Endpoint: {str(e)}")) + raise Exception(f"Failure creating HEC Endpoint: {str(e)}") def get_all_indexes(self) -> None: """ @@ -232,6 +237,8 @@ def wait_for_app_installation(self): installed_config_apps.append(config_app.appid) LOG.debug("Apps in the Splunk instance: " + str(list(map(lambda x: x.name, splunk_instance_apps)))) LOG.debug(f"apps in contentctl package found in Splunk instance: {installed_config_apps}") + if len(installed_config_apps) >= len(config_apps): + break except Exception as e: LOG.exception(e) time.sleep(5) From 344f7713789cec535bb8c832c5f14a507c4a98f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Wed, 6 Nov 2024 22:00:40 +0000 Subject: [PATCH 18/20] Merged PR 114831: fix From a789726a099c7c14e7f5eeca4d2771d6acb0a5e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Thu, 7 Nov 2024 18:59:55 +0000 Subject: [PATCH 19/20] Merged PR 114975: fix fix --- .../infrastructures/DetectionTestingInfrastructure.py | 1 - 1 file changed, 1 deletion(-) diff --git a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py index 087cec73..c7182dff 100644 --- a/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +++ b/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py @@ -195,7 +195,6 @@ def configure_hec(self): useACK=True, ) self.hec_token = str(res.token) - breakpoint() return except Exception as e: From 8207247e6f1bc6398c76ca51239e65647ec0064c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milio=20Gonzalez?= Date: Thu, 7 Nov 2024 21:54:17 +0000 Subject: [PATCH 20/20] Merged PR 115051: Fix validation for apps Fix validation for apps --- contentctl/objects/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 1931c151..f57c9e6b 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -95,6 +95,10 @@ def getApp(self, config:test,stage_file:bool=False)->str: destination = config.getLocalAppDir() / server_path.name if stage_file: Utils.download_file_from_http(file_url_string, str(destination)) + # Needed for `contentctl validate` and `contentctl build` else it fails without the splunkbase creds, + # which shouldn't be mandatory for validation or building the app. + elif self.version is not None and self.uid is not None: + destination = self.getSplunkbasePath() else: raise Exception(f"Unknown path for app '{self.title}'")