Initial release

.env.dist

@@ -0,0 +1,4 @@
+MWDB_TOKEN=
+PROXY_METHOD=
+PROXY_URL=
+PROXY_DEFAULT_PROXY=

.github/workflows/push_hub.yml

@@ -0,0 +1,81 @@
+name: Build and push docker images
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+jobs:
+  build_web:
+    name: Build image
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push the image
+        uses: docker/build-push-action@v4
+        with:
+          file: ./deploy/docker/web.Dockerfile
+          repository: nazywam/mtracker-web
+          tags: |
+            nazywam/mtracker-web:master
+            nazywam/mtracker-web:${{ github.sha }}
+          push: ${{ github.event_name == 'push' }}
+          # Flux v1 doesn't support OCI-compliant manifests
+          provenance: false
+  build_scheduler:
+    name: Build image
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push the image
+        uses: docker/build-push-action@v4
+        with:
+          file: ./deploy/docker/scheduler.Dockerfile
+          repository: nazywam/mtracker-scheduler
+          tags: |
+            nazywam/mtracker-scheduler:master
+            nazywam/mtracker-scheduler:${{ github.sha }}
+          push: ${{ github.event_name == 'push' }}
+          # Flux v1 doesn't support OCI-compliant manifests
+          provenance: false
+  build_worker:
+    name: Build image
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push the image
+        uses: docker/build-push-action@v4
+        with:
+          file: ./deploy/docker/worker.Dockerfile
+          repository: nazywam/mtracker-worker
+          tags: |
+            nazywam/mtracker-worker:master
+            nazywam/mtracker-worker:${{ github.sha }}
+          push: ${{ github.event_name == 'push' }}
+          # Flux v1 doesn't support OCI-compliant manifests
+          provenance: false

.gitignore

@@ -0,0 +1,12 @@
+mtracker.ini
+venv
+*.pyc
+__pycache__/
+.mypy_cache/
+.vscode
+results
+data
+.env
+src/private_modules
+build/
+*.egg-info

.gitlab-ci.yml

@@ -0,0 +1,14 @@
+image: python:3.7
+test_style:
+  script:
+  - pip3 install black==19.10b0 flake8==3.7.9
+  - black --check src/*.py
+  - flake8 src/
+
+image: python:3.7
+test_types:
+  script:
+  - pip3 install -r requirements.txt
+  - pip3 install -r src/modules/requirements.txt
+  - pip3 install mypy==0.770
+  - mypy src/

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2024, CERT Polska
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of mailgoose nor the names of its contributors
+      may be used to endorse or promote products derived from this software
+      without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

MANIFEST.in

@@ -0,0 +1,2 @@
+graft src/templates
+graft src/static

README.md

@@ -0,0 +1,109 @@
+# mtracker
+
+Mtracker is a system dedicated to tracking malware C2 servers and automatically downloading actionable information from them. 
+
+Mtracker receives "configurations" that are used to create "trackers". Each tracker tries to connect to C2 using the provided configuration (C2 address, host, port, etc) and download more binaries, commands or data.
+
+Logically, mtracker is divided into several components:
+
+![](docs/mtracker.png)
+
+The user-visible component is the mtracker web interface, which provides
+a HTTP UI and API:
+
+![](docs/ui.png)
+
+The second most important component are workers, which communicate with
+the interface using redis task queue (using [rq](https://python-rq.org/docs/)), and execute the tasks.
+
+Finally there is a scheduler, that makes sure tasks are executed in a regular intervals.
+
+All the components communicate through the shared postgres database to manage the state, but the results are only uploaded to [mwdb-core](https://github.com/CERT-Polska/mwdb-core/). Mtracker is tightly tied to MWDB. Users who want to customize this behaviour and use another backend need to override the BotModule class and provide their own implmentation of appropriate methods.
+
+### Installation: docker-compose
+
+The easiest way to setup mtracker is using a provided docker-compose file
+
+```bash
+docker-compose build
+cp .env.dist .env
+vim .env  # set mwdb token and proxy configuration
+mkdir src/private_modules  # a place for your modules
+echo "trackers = {}" > src/private_modules/__init__.py  # empty module list
+docker-compose up # You can use '-d' to detach
+
+# finally, in another tab apply migrations from db/folder in order, using:
+docker-compose exec -T postgres psql -U mtracker3 --dbname mtracker3 < db/000_init.sql
+docker-compose exec -T postgres psql -U mtracker3 --dbname mtracker3 < db/001_bots_tasks.sql
+docker-compose exec -T postgres psql -U mtracker3 --dbname mtracker3 < db/002_update_bots.sql
+docker-compose exec -T postgres psql -U mtracker3 --dbname mtracker3 < db/003_bots_add_last_error.sql
+docker-compose exec -T postgres psql -U mtracker3 --dbname mtracker3 < db/004_bots_add_family.sql
+docker-compose exec -T postgres psql -U mtracker3 --dbname mtracker3 < db/005_results.sql
+docker-compose exec -T postgres psql -U mtracker3 --dbname mtracker3 < db/006_add_proxy.sql
+docker-compose exec -T postgres psql -U mtracker3 --dbname mtracker3 < db/007_alter_proxy_add_username_password.sql
+```
+
+You can now access the web interface on port 80.
+
+### Installation: bare metal
+
+You can also install mtracker like you would install any other Python package:
+
+```
+# Assuming mtracker is cloned into /opt/mtracker directory
+# And your modules are stored in /opt/mtrackermodules
+# Ensure you have postgres and redis running
+cp /opt/mtracker/mtracker.example.ini /opt/mtracker/mtracker.ini
+# Edit the config, especially mwdb_token, Db config, Redis config, and ProxyConfig
+pip install /opt/mtracker
+pip install -r /opt/mtrackermodules/requirements.txt
+python3 -m mtracker.worker /opt/mtrackermodules/ # start worker process
+python3 -m mtracker.server  # in another shell, start the server
+python3 -m mtracker.scheduler  # in another shell, start the scheduler
+```
+
+### Installation: proxy configuration
+
+Mtracker needs proxy servers to work. There are two ways to configure proxies:
+
+* Set proxy.method to "file", and proxy.path to a json file.
+* Set proxy.method to "url", and proxy.url to a url that returns a json.
+
+In both cases, the JSON should look like this (additional fields are allowed):
+```json
+[
+  {
+    "host": "10.0.0.1234",
+    "port": 9050,
+    "password": "",
+    "username": "",
+    "country": "pl"
+  },
+  // more proxies in the same format
+]
+```
+
+All the proxies are SOCKS5h proxies. If you don't want to host your own proxies, the easiest solution is to proxy traffic through tor. 
+
+### Module development
+
+Mtracker is just a framework, and **won't work** without implementing modules. See [the documentation](docs/moduledev.md) for more information about module development.
+
+### Contact
+
+If you have any problems, bugs or feature requests related to mquery, you're
+encouraged to create a GitHub issue.
+
+You can chat about this project on Discord:
+
+[![](https://dcbadge.vercel.app/api/server/3FcP6GQNzd)](https://discord.gg/3FcP6GQNzd)
+
+If you have questions unsuitable for Github or discord, you can email CERT.PL
+([email protected]) directly.
+
+### Acknowledgements
+
+This project was created and is maintained by [CERT Polska](https://cert.pl/en/).
+
+Additional thanks to [Checkpoint Research](research.checkpoint.com), especially [@ITAYC0HEN](https://github.com/ITAYC0HEN),
+who contributed to the development and testing of mtracker

RELEASES.md

@@ -0,0 +1,3 @@
+# Version 1.0.0
+
+First public release

db/000_init.sql

@@ -0,0 +1,9 @@
+CREATE TYPE status AS ENUM ('crashed', 'inprogress', 'working', 'failing', 'new', 'archived');
+
+CREATE TABLE trackers (
+    tracker_id      SERIAL PRIMARY KEY,
+    config_hash     CHAR(64),
+    config          TEXT,
+    family          TEXT,
+    status          status
+);

db/001_bots_tasks.sql

@@ -0,0 +1,18 @@
+CREATE TABLE bots (
+    bot_id          SERIAL PRIMARY KEY,
+    tracker_id      INT REFERENCES trackers(tracker_id),
+    status          status,
+    state           TEXT,
+    failing_spree   INT,
+    next_execution  DATE,
+    country         TEXT
+);
+
+CREATE TABLE tasks (
+    task_id         SERIAL PRIMARY KEY,
+    bot_id          INT REFERENCES bots(bot_id),
+    status          status,
+    report_time     DATE,
+    results         TEXT,
+    logs            TEXT
+);

db/002_update_bots.sql

@@ -0,0 +1,2 @@
+ALTER table bots ALTER next_execution TYPE timestamp;
+ALTER table tasks ALTER report_time TYPE timestamp;

db/003_bots_add_last_error.sql

@@ -0,0 +1 @@
+ALTER TABLE bots ADD COLUMN last_error TEXT;

db/004_bots_add_family.sql

@@ -0,0 +1,2 @@
+ALTER TABLE bots ADD COLUMN family TEXT;
+UPDATE bots b SET family = (SELECT family FROM trackers t WHERE b.tracker_id=t.tracker_id);

db/005_results.sql

@@ -0,0 +1,13 @@
+CREATE TYPE result_type AS ENUM ('config', 'blob', 'binary');
+
+CREATE TABLE results (
+    result_id       SERIAL PRIMARY KEY,
+    task_id         INT REFERENCES tasks(task_id),
+    type            result_type,
+    name            TEXT,
+    sha256          CHAR(64),
+    tags            TEXT [],
+    upload_time     TIMESTAMP
+);
+
+ALTER TABLE tasks DROP COLUMN results;

db/006_add_proxy.sql

@@ -0,0 +1,6 @@
+CREATE TABLE proxies (
+    proxy_id        SERIAL PRIMARY KEY,
+    host            TEXT,
+    port            INTEGER,
+    country         CHAR(2)
+);

db/007_alter_proxy_add_username_password.sql

@@ -0,0 +1,2 @@
+ALTER TABLE proxies ADD COLUMN username TEXT;
+ALTER TABLE proxies ADD COLUMN password TEXT;

deploy/docker/scheduler.Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.8
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+COPY src ./src
+COPY MANIFEST.in setup.py .
+RUN pip install .
+
+CMD ["python3", "-m", "mtracker.scheduler"]

deploy/docker/web.Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.8
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+COPY src ./src
+COPY MANIFEST.in setup.py .
+RUN pip install .
+
+CMD ["python3", "-m", "mtracker.server"]

deploy/docker/worker.Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.8
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+COPY src ./src
+COPY MANIFEST.in setup.py .
+RUN pip install .
+
+CMD ["python3", "-m", "mtracker.worker", "/usr/src/app/src/modules"]