From a3d84ee0af02ff3ee9224ca51298b8ad7226fa81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Mon, 20 Nov 2023 17:33:37 +0100 Subject: [PATCH 1/7] Add CI/CD --- .github/workflows/publish.yml | 33 ++++++++++++++++++++++++++ .github/workflows/release.yml | 44 +++++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 15 ++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..820ab14 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,33 @@ +name: Build and push docker images +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + build_core: + name: Build image + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + if: ${{ github.event_name == 'push' }} + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build and push the image + uses: docker/build-push-action@v4 + with: + tags: | + certpl/${{ github.event.repository.name }}:${{ github.sha }} + push: ${{ github.event_name == 'push' }} + # Flux v1 doesn't support OCI-compliant manifests + provenance: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..26eca14 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,44 @@ +name: Release a new version + +on: + release: + types: [published] + +jobs: + release_pypi: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build package + run: | + pip3 install setuptools wheel + python3 setup.py bdist_wheel + - name: Publish to PyPi + uses: pypa/gh-action-pypi-publish@v1.3.0 + with: + user: __token__ + password: ${{ secrets.pypi_password }} + release_dockerhub: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build and push the image + uses: docker/build-push-action@v4 + with: + tags: | + certpl/${{ github.event.repository.name }}:${{ github.sha }} + certpl/${{ github.event.repository.name }}:${{ github.event.release.tag_name }} + certpl/${{ github.event.repository.name }}:latest + push: true + # Flux v1 doesn't support OCI-compliant manifests + provenance: false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..b7c1693 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,15 @@ +name: Test the code +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: CERT-Polska/lint-python-action@v2 + with: + source: karton/ From 1cd6076eaf5dba7b8e0dc5251af3f73f9ce7eb6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Mon, 20 Nov 2023 17:36:26 +0100 Subject: [PATCH 2/7] Add Dockerfile --- Dockerfile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f3594c2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.10-slim + +WORKDIR /usr/src/app + +RUN apt-get update && apt-get install -y \ + tshark \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +COPY ./README.md ./README.md +COPY ./karton ./karton +COPY ./setup.py ./setup.py + +RUN pip install . +ENTRYPOINT karton-pcap-miner From 30141b6b81ce67f154da39b9cdd91e3595c18c97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Mon, 20 Nov 2023 17:41:34 +0100 Subject: [PATCH 3/7] Fix dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f3594c2..29ef457 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,8 @@ RUN apt-get update && apt-get install -y \ tshark \ && rm -rf /var/lib/apt/lists/* -COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt +COPY ./requirements.txt ./requirements.txt +RUN pip install --no-cache-dir -r ./requirements.txt COPY ./README.md ./README.md COPY ./karton ./karton From 2d65155165df11baa1b067aa7b56763c54711ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Mon, 20 Nov 2023 17:49:36 +0100 Subject: [PATCH 4/7] Fix lint --- karton/pcap_miner/pcap_miner.py | 60 ++++++++++++++++++++++----------- pyproject.toml | 14 ++++++++ setup.cfg | 3 ++ 3 files changed, 58 insertions(+), 19 deletions(-) create mode 100644 pyproject.toml create mode 100644 setup.cfg diff --git a/karton/pcap_miner/pcap_miner.py b/karton/pcap_miner/pcap_miner.py index a93bfa5..a2962ab 100644 --- a/karton/pcap_miner/pcap_miner.py +++ b/karton/pcap_miner/pcap_miner.py @@ -1,11 +1,11 @@ +import ipaddress import json import re import tempfile from pathlib import Path from subprocess import check_output -import ipaddress -from karton.core import Karton, Task, Resource +from karton.core import Karton, Resource, Task def extract_ip(ip: str) -> str: @@ -34,7 +34,8 @@ def convert_tlsmon(directory: Path) -> None: class KartonPcapMiner(Karton): """ - Extract network indicators from analysis PCAPs and add push them to MWDB as attributes + Extract network indicators from analysis PCAPs and add push them to MWDB as + attributes """ identity = "karton.pcap-miner" @@ -56,21 +57,21 @@ def parse_tcp_conv(self, output: str) -> list[str]: PAT = r"([\d.]+:\d+)\s+<->\s+([\d.]+:\d+)" matches = re.findall(PAT, output) - output = set() + results: set[str] = set() for source, destination in matches: - output.add(self.select_nonlocal_ip(source, destination)) + results.add(self.select_nonlocal_ip(source, destination)) - return list(output) + return list(results) def parse_sni_output(self, output: str) -> list[str]: PAT = r"^(\S+)\s+(\d+)$" matches = re.findall(PAT, output) - output = set() + results: set[str] = set() for hostname, port in matches: - output.add(f"{hostname}:{port}") + results.add(f"{hostname}:{port}") - return list(output) + return list(results) def default_parser(self, output: str) -> list[str]: return list(set(filter(None, output.splitlines()))) @@ -79,7 +80,9 @@ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) # analysis VM range, used for detecting direction in connections - self.vm_ip_range = ipaddress.ip_network(self.config.get("pcap-miner", "vm_ip_range", "10.0.0.0/8")) + self.vm_ip_range = ipaddress.ip_network( + self.config.get("pcap-miner", "vm_ip_range", "10.0.0.0/8") + ) # do not report artifacts if number of results exceeds max_results self.max_results = self.config.getint("pcap-miner", "max_results", fallback=24) @@ -90,10 +93,28 @@ def __init__(self, *args, **kwargs) -> None: self.ignorelist = json.load(f) self.analyzers = { - "network-http": (["-T", "fields", "-e", "http.request.full_uri"], self.default_parser), + "network-http": ( + ["-T", "fields", "-e", "http.request.full_uri"], + self.default_parser, + ), "network-tcp": (["-z", "conv,tcp"], self.parse_tcp_conv), - "network-sni": (["-Y", 'ssl.handshake.extension.type == "server_name"', "-T", "fields", "-e", "tls.handshake.extensions_server_name", "-e", "tcp.dstport"], self.parse_sni_output), - "network-dns": (["-Y", "dns.flags.response == 0", "-T", "fields", "-e", "dns.qry.name"], self.default_parser), + "network-sni": ( + [ + "-Y", + 'ssl.handshake.extension.type == "server_name"', + "-T", + "fields", + "-e", + "tls.handshake.extensions_server_name", + "-e", + "tcp.dstport", + ], + self.parse_sni_output, + ), + "network-dns": ( + ["-Y", "dns.flags.response == 0", "-T", "fields", "-e", "dns.qry.name"], + self.default_parser, + ), } def mine_pcap(self, directory: Path) -> dict[str, list[str]]: @@ -123,7 +144,9 @@ def filter_results(self, results: dict[str, list[str]]) -> dict[str, list[str]]: filtered = [x for x in v if x not in filter_list] if self.max_results != -1 and len(filtered) > self.max_results: - self.log.warning("Dropping results for %s due to high count: %s", k, len(filtered)) + self.log.warning( + "Dropping results for %s due to high count: %s", k, len(filtered) + ) elif filtered: output[k] = sorted(filtered) @@ -135,10 +158,7 @@ def report_results(self, sample: Resource, results: dict[str, list[str]]) -> Non "type": "sample", "stage": "analyzed", }, - payload={ - "sample": sample, - "attributes": results - } + payload={"sample": sample, "attributes": results}, ) self.send_task(enrichment_task) @@ -169,4 +189,6 @@ def process(self, task: Task) -> None: self.log.info("Results:") for k, v in results_filtered.items(): self.log.info("%s: %s", k, len(v)) - self.report_results(task.get_payload("sample"), results=results_filtered) + self.report_results( + task.get_payload("sample"), results=results_filtered + ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bb6be48 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,14 @@ +[tool.lint-python] +lint-version = "2" +source = "karton/" + +[tool.black] +line-length = 88 + +[tool.isort] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 88 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..e4e5eb3 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203, W503 From e6f1bc531859cc09100d2f98eb850bbde505834a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Mon, 20 Nov 2023 17:53:59 +0100 Subject: [PATCH 5/7] Bump python version --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b7c1693..0eb989e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,6 +10,9 @@ jobs: lint: runs-on: ubuntu-latest steps: + - uses: actions/setup-python@v4 + with: + python-version: '3.10' - uses: CERT-Polska/lint-python-action@v2 with: source: karton/ From 609f14d2b6da51387f987608251b0696a8d140a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Mon, 20 Nov 2023 18:03:29 +0100 Subject: [PATCH 6/7] Fix python bump --- .github/workflows/test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0eb989e..d4b83d8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,9 +10,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - uses: CERT-Polska/lint-python-action@v2 with: source: karton/ + python-version: 3.10 From a6c7aefe97de6668e88b5cd96fd4c949f4740d11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Mon, 20 Nov 2023 18:04:51 +0100 Subject: [PATCH 7/7] Thanks, yaml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d4b83d8..a54fa96 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,4 +13,4 @@ jobs: - uses: CERT-Polska/lint-python-action@v2 with: source: karton/ - python-version: 3.10 + python-version: "3.10"