Unstructured dependencies for multiple os's

.flake8

@@ -1,5 +1,5 @@
 [flake8]
 select = F,E722
 ignore = F403,F405,F541
-per-file-ignores =
+per-file-ignores = 
     */__init__.py:F401,F403

.github/workflows/tests.yml

@@ -14,20 +14,20 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
         with:
           options: "--check"
       - name: Install Python 3
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install dependencies
         run: |
           pip install flake8
       - name: flake8
         run: |
-          flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403"
+          flake8
   test:
     needs: lint
     runs-on: ubuntu-latest
@@ -37,9 +37,9 @@ jobs:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -48,23 +48,42 @@ jobs:
           poetry install
       - name: Run tests
         run: |
-          poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
+          poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
       - name: Upload Code Coverage
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./cov.xml
           verbose: true
+  test-distros:
+    needs: lint
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, debian, archlinux, fedora-latest, gentoo, alpine]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install
+      - name: Run tests
+        run: |
+          poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
   update_docs:
     needs: test
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install dependencies
@@ -87,8 +106,8 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && (github.ref == 'refs/heads/stable')
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
@@ -109,11 +128,11 @@ jobs:
     if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')
     continue-on-error: true
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install dependencies
@@ -162,7 +181,7 @@ jobs:
   #   runs-on: ubuntu-latest
   #   if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
   #   steps:
-  #     - uses: actions/checkout@v3
+  #     - uses: actions/checkout@v4
   #       with:
   #         ref: ${{ github.head_ref }}
   #         fetch-depth: 0 # Fetch all history for all tags and branches

bbot/modules/unstructured.py

@@ -66,7 +66,32 @@ class unstructured(BaseModule):
         "ignore_folders": "Subfolders to ignore when crawling downloaded folders",
     }
 
-    deps_apt = ["libmagic-dev", "poppler-utils", "tesseract-ocr", "libreoffice", "pandoc"]
+    deps_ansible = [
+        {
+            "name": "Install Deps (Debian/Ubuntu)",
+            "package": {
+                "name": ["libmagic-dev", "poppler-utils", "tesseract-ocr", "libreoffice", "pandoc"],
+                "state": "present",
+            },
+            "become": True,
+            "when": "ansible_facts['os_family'] == 'Debian'",
+        },
+        {
+            "name": "Install Deps (Arch)",
+            "package": {"name": ["file", "poppler", "tesseract", "libreoffice-fresh", "pandoc"], "state": "present"},
+            "become": True,
+            "when": "ansible_facts['os_family'] == 'Archlinux'",
+        },
+        {
+            "name": "Install Deps (Fedora)",
+            "package": {
+                "name": ["file-devel", "poppler-utils", "tesseract", "libreoffice", "pandoc"],
+                "state": "present",
+            },
+            "become": True,
+            "when": "ansible_facts['os_family'] == 'Fedora'",
+        },
+    ]
     deps_pip = ["unstructured[all-docs]"]
 
     async def setup(self):

bbot/test/run_tests.sh

@@ -10,7 +10,7 @@ echo
 
 echo "[+] Linting with flake8"
 echo "======================="
-flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403" "$bbot_dir" || exit 1
+flake8 "$bbot_dir" || exit 1
 echo
 
 echo "[+] Testing with pytest"