diff --git a/.flake8 b/.flake8 index bbe8b7581..a6f057338 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,4 @@ select = F,E722 ignore = F403,F405,F541 per-file-ignores = - */__init__.py:F401,F403 \ No newline at end of file + */__init__.py:F401,F403 diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml new file mode 100644 index 000000000..e9eac869a --- /dev/null +++ b/.github/workflows/distro_tests.yml @@ -0,0 +1,63 @@ +name: Linux Distro Tests +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test-distros: + runs-on: ubuntu-latest + container: + image: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/security"] + steps: + - uses: actions/checkout@v4 + - name: Install Python and Poetry + run: | + if [ -f /etc/os-release ]; then + . /etc/os-release + if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kali" ] || [ "$ID" = "parrotsec" ]; then + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get -y install curl git bash build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev + elif [ "$ID" = "alpine" ]; then + apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git make openssl-dev bzip2-dev zlib-dev xz-dev sqlite-dev + elif [ "$ID" = "arch" ]; then + pacman -Syu --noconfirm curl git bash base-devel + elif [ "$ID" = "fedora" ]; then + dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel + elif [ "$ID" = "gentoo" ]; then + emerge-webrsync + emerge --update --newuse dev-vcs/git curl bash + fi + fi + + # Re-run the script with bash + exec bash -c " + curl https://pyenv.run | bash + export PATH=\"$HOME/.pyenv/bin:\$PATH\" + export PATH=\"$HOME/.local/bin:\$PATH\" + eval \"\$(pyenv init --path)\" + eval \"\$(pyenv init -)\" + eval \"\$(pyenv virtualenv-init -)\" + pyenv install 3.11 + pyenv global 3.11 + pyenv rehash + python3.11 -m pip install --user pipx + python3.11 -m pipx ensurepath + pipx install poetry + " + - name: Run tests + run: | + export PATH="$HOME/.local/bin:$PATH" + export PATH="$HOME/.pyenv/bin:$PATH" + export PATH="$HOME/.pyenv/shims:$PATH" + export BBOT_DISTRO_TESTS=true + poetry env use python3.11 + poetry install + poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG . diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 799c0101e..3a9aecd52 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,12 +14,12 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable with: options: "--check" - name: Install Python 3 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" - name: Install dependencies @@ -27,7 +27,7 @@ jobs: pip install flake8 - name: flake8 run: | - flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403" + flake8 test: needs: lint runs-on: ubuntu-latest @@ -37,9 +37,9 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -48,95 +48,24 @@ jobs: poetry install - name: Run tests run: | - poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . + poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . - name: Upload Code Coverage uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./cov.xml verbose: true - update_docs: - needs: test - runs-on: ubuntu-latest - if: github.event_name == 'push' && (github.ref != 'refs/heads/dev' && github.ref != 'refs/heads/stable') - steps: - - uses: actions/checkout@v3 - with: - token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - name: Install dependencies - run: | - pip install poetry - poetry install - - name: Generate docs - run: | - poetry run bbot/scripts/docs.py - - name: Commit docs - uses: EndBug/add-and-commit@v9 - continue-on-error: true - with: - add: '["*.md", "docs/data/chord_graph/*.json"]' - author_name: "BBOT Docs Autopublish" - author_email: info@blacklanternsecurity.com - message: "Refresh module docs" - publish_docs: - needs: test - runs-on: ubuntu-latest - if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') - steps: - - uses: actions/checkout@v3 - with: - token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} - - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - - uses: actions/cache@v3 - with: - key: mkdocs-material-${{ env.cache_id }} - path: .cache - restore-keys: | - mkdocs-material- - - name: Install dependencies - run: | - pip install poetry - poetry install --only=docs - - name: Configure Git - run: | - git config user.name github-actions - git config user.email github-actions@github.com - git fetch origin gh-pages:refs/remotes/origin/gh-pages - if git show-ref --verify --quiet refs/heads/gh-pages; then - git branch -f gh-pages origin/gh-pages - else - git branch --track gh-pages origin/gh-pages - fi - - name: Generate docs (stable branch) - if: github.ref == 'refs/heads/stable' - run: | - poetry run mike deploy Stable - - name: Generate docs (dev branch) - if: github.ref == 'refs/heads/dev' - run: | - poetry run mike deploy Dev - - name: Publish docs - run: | - git switch gh-pages - git push publish_code: needs: test runs-on: ubuntu-latest if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable') continue-on-error: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" - name: Install dependencies @@ -185,7 +114,7 @@ jobs: # runs-on: ubuntu-latest # if: github.event_name == 'push' && github.ref == 'refs/heads/stable' # steps: - # - uses: actions/checkout@v3 + # - uses: actions/checkout@v4 # with: # ref: ${{ github.head_ref }} # fetch-depth: 0 # Fetch all history for all tags and branches diff --git a/bbot/test/run_tests.sh b/bbot/test/run_tests.sh index 3490f5b65..39458dbf9 100755 --- a/bbot/test/run_tests.sh +++ b/bbot/test/run_tests.sh @@ -10,7 +10,7 @@ echo echo "[+] Linting with flake8" echo "=======================" -flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403" "$bbot_dir" || exit 1 +flake8 "$bbot_dir" || exit 1 echo if [ "${1}x" != "x" ] ; then diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index f5d4255f6..bb63b57e5 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -89,6 +89,10 @@ def module(self): async def module_test( self, httpx_mock, bbot_httpserver, bbot_httpserver_ssl, monkeypatch, request, caplog, capsys ): + # Skip dastardly test if we're in the distro tests (because dastardly uses docker) + if os.getenv("BBOT_DISTRO_TESTS") and self.name == "dastardly": + pytest.skip("Skipping module_test for dastardly module due to BBOT_DISTRO_TESTS environment variable") + self.log.info(f"Starting {self.name} module test") module_test = self.ModuleTest( self, httpx_mock, bbot_httpserver, bbot_httpserver_ssl, monkeypatch, request, caplog, capsys