⬆️ Update hoarder

roles/hoarder/defaults/main.yml

@@ -13,24 +13,16 @@ hoarder_network_name: "hoarder"
 
 # specs
 hoarder_web_memory: 1g
-hoarder_workers_memory: 1g
 hoarder_chrome_memory: 1g
 hoarder_search_memory: 1g
-hoarder_redis_memory: 1g
 
 # docker
 hoarder_web_container_name: hoarder-web
-hoarder_web_image_name: "ghcr.io/hoarder-app/hoarder-web"
+hoarder_web_image_name: "ghcr.io/hoarder-app/hoarder"
 hoarder_web_image_version: "release"
-hoarder_workers_container_name: hoarder-workers
-hoarder_workers_image_name: "ghcr.io/hoarder-app/hoarder-workers"
-hoarder_workers_image_version: "release"
-hoarder_redis_container_name: hoarder-redis
-hoarder_redis_image_name: "redis"
-hoarder_redis_image_version: "7.2-alpine"
 hoarder_search_container_name: hoarder-search
 hoarder_search_image_name: "getmeili/meilisearch"
-hoarder_search_image_version: "v1.6"
+hoarder_search_image_version: "v1.11.1"
 hoarder_chrome_container_name: hoarder-chrome
 hoarder_chrome_image_name: "gcr.io/zenika-hub/alpine-chrome"
 hoarder_chrome_image_version: "123"
@@ -41,22 +33,29 @@ hoarder_group_id: "1000"
 hoarder_data_dir: "/data"
 hoarder_nextauth_url: "https://{{ hoarder_hostname }}.{{ ansible_nas_domain }}"
 hoarder_nextauth_secret: "iVfxDV9JZlcBCRjsk4f+TZlMf6GelJ0Q1KVQCG46m4xPTdi4"
-hoarder_redis_host: "{{ hoarder_redis_container_name }}"
-hoarder_redis_port: "6379"
-hoarder_redis_db_idx: "0"
-hoarder_redis_password: "redis"
 hoarder_meili_addr: "http://{{ hoarder_search_container_name }}:7700"
 hoarder_meili_master_key: "Ot5yRmb6WVnKhwnAdMGEtgBjd0bu6PD5z+PtavENImfsr1ml"
 hoarder_meili_no_analytics: "true"
 hoarder_disable_signups: "false"
+hoarder_disable_password_auth: "false"
+hoarder_oauth_wellknown_url: ""
+hoarder_oauth_client_secret: ""
+hoarder_oauth_client_id: ""
+hoarder_oauth_scope: "openid email profile"
+hoarder_oauth_provider_name: "Custom Provider"
+hoarder_oauth_allow_dangerous_emailo_account_linking: "false"
 hoarder_max_asset_size_mb: "4"
 hoarder_disable_new_release_check: "false"
 hoarder_openai_api_key: ""
 hoarder_openai_base_url: ""
 hoarder_ollama_base_url: ""
-hoarder_inference_text_model: "gpt-3.5-turbo-0125"
-hoarder_inference_image_model: "gpt-4o-2024-05-13"
+hoarder_ollama_keep_alive: ""
+hoarder_inference_text_model: "gpt-4o-mini"
+hoarder_inference_image_model: "gpt-4o-mini"
 hoarder_inference_lang: "english"
+hoarder_embedding_text_model: "text-embedding-3-small"
+hoarder_inference_context_length: "2048"
+hoarder_inference_job_timeout_sec: "30"
 hoarder_crawler_num_workers: "1"
 hoarder_browser_web_url: "http://{{ hoarder_chrome_container_name }}:9222"
 hoarder_browser_websocket_url: ""
@@ -67,27 +66,41 @@ hoarder_crawler_full_page_screenshot: "false"
 hoarder_crawler_full_page_archive: "false"
 hoarder_crawler_job_timeout_sec: "60"
 hoarder_crawler_navigate_timeout_sec: "30"
+hoarder_crawler_video_downaload: "false"
+hoarder_crawler_video_downaload_max_size: "50"
+hoarder_crawler_video_downaload_timeout_sec: "600"
+hoarder_crawler_enable_adblocker: "true"
+hoarder_ocr_cache_dir: "$TEMP_DIR"
+hoarder_ocr_langs: "eng"
+hoarder_ocr_confidence_threshold: "50"
 
 hoarder_env:
   DATA_DIR: "{{ hoarder_data_dir }}"
   NEXTAUTH_URL: "{{ hoarder_nextauth_url }}"
   NEXTAUTH_SECRET: "{{ hoarder_nextauth_secret }}"
-  REDIS_HOST: "{{ hoarder_redis_host }}"
-  REDIS_PORT: "{{ hoarder_redis_port }}"
-  REDIS_DB_IDX: "{{ hoarder_redis_db_idx }}"
-  REDIS_PASSWORD: "{{ hoarder_redis_password }}"
   MEILI_ADDR: "{{ hoarder_meili_addr }}"
   MEILI_MASTER_KEY: "{{ hoarder_meili_master_key }}"
   MEILI_NO_ANALYTICS: "{{ hoarder_meili_no_analytics }}"
   DISABLE_SIGNUPS: "{{ hoarder_disable_signups }}"
+  DISABLE_PASSWORD_AUTH: "{{ hoarder_disable_password_auth }}"
+  # OAUTH_WELLKNOWN_URL: "{{ hoarder_oauth_wellknown_url }}"
+  # OAUTH_CLIENT_SECRET: "{{ hoarder_oauth_client_secret }}"
+  # OAUTH_CLIENT_ID: "{{ hoarder_oauth_client_id }}"
+  # OAUTH_SCOPE: "{{ hoarder_oauth_scope }}"
+  # OAUTH_PROVIDER_NAME: "{{ hoarder_oauth_provider_name }}"
+  OAUTH_ALLOW_DANGEROUS_EMAIL_ACCOUNT_LINKING: "{{ hoarder_oauth_allow_dangerous_emailo_account_linking }}"
   MAX_ASSET_SIZE_MB: "{{ hoarder_max_asset_size_mb }}"
   DISABLE_NEW_RELEASE_CHECK: "{{ hoarder_disable_new_release_check }}"
   OPENAI_API_KEY: "{{ hoarder_openai_api_key }}"
   # OPENAI_BASE_URL: "{{ hoarder_openai_base_url }}"
   # OLLAMA_BASE_URL: "{{ hoarder_ollama_base_url }}"
+  # OLLAMA_KEEP_ALIVE: "{{ hoarder_ollama_keep_alive }}"
   INFERENCE_TEXT_MODEL: "{{ hoarder_inference_text_model }}"
   INFERENCE_IMAGE_MODEL: "{{ hoarder_inference_image_model }}"
   INFERENCE_LANG: "{{ hoarder_inference_lang }}"
+  INFERENCE_CONTEXT_LENGTH: "{{ hoarder_inference_context_length }}"
+  INFERENCE_JOB_TIMEOUT_SEC: "{{ hoarder_inference_job_timeout_sec }}"
+  EMBEDDING_TEXT_MODEL: "{{ hoarder_embedding_text_model }}"
   CRAWLER_NUM_WORKERS: "{{ hoarder_crawler_num_workers }}"
   BROWSER_WEB_URL: "{{ hoarder_browser_web_url }}"
   # BROWSER_WEBSOCKET_URL: "{{ hoarder_browser_websocket_url }}"
@@ -98,3 +111,10 @@ hoarder_env:
   CRAWLER_FULL_PAGE_ARCHIVE: "{{ hoarder_crawler_full_page_archive }}"
   CRAWLER_JOB_TIMEOUT_SEC: "{{ hoarder_crawler_job_timeout_sec }}"
   CRAWLER_NAVIGATE_TIMEOUT_SEC: "{{ hoarder_crawler_navigate_timeout_sec }}"
+  CRAWLER_VIDEO_DOWNLOAD: "{{ hoarder_crawler_video_downaload }}"
+  CRAWLER_VIDEO_DOWNLOAD_MAX_SIZE: "{{ hoarder_crawler_video_downaload_max_size }}"
+  CRAWLER_VIDEO_DOWNLOAD_TIMEOUT_SEC: "{{ hoarder_crawler_video_downaload_timeout_sec }}"
+  CRAWLER_ENABLE_ADBLOCKER: "{{ hoarder_crawler_enable_adblocker }}"
+  OCR_CACHE_DIR: "{{ hoarder_ocr_cache_dir }}"
+  OCR_LANGS: "{{ hoarder_ocr_langs }}"
+  OCR_CONFIDENCE_THRESHOLD: "{{ hoarder_ocr_confidence_threshold }}"

roles/hoarder/molecule/default/verify.yml

@@ -12,11 +12,6 @@
         name: "{{ hoarder_search_container_name }}"
       register: result_search
 
-    - name: Get hoarder redis container state
-      community.docker.docker_container:
-        name: "{{ hoarder_redis_container_name }}"
-      register: result_redis
-
     - name: Get hoarder chrome container state
       community.docker.docker_container:
         name: "{{ hoarder_chrome_container_name }}"
@@ -27,21 +22,12 @@
         name: "{{ hoarder_web_container_name }}"
       register: result_web
 
-    - name: Get hoarder workers container state
-      community.docker.docker_container:
-        name: "{{ hoarder_workers_container_name }}"
-      register: result_workers
-
     - name: Check if hoarder containers are running
       ansible.builtin.assert:
         that:
           - result_search.container['State']['Status'] == "running"
           - result_search.container['State']['Restarting'] == false
-          - result_redis.container['State']['Status'] == "running"
-          - result_redis.container['State']['Restarting'] == false
           - result_chrome.container['State']['Status'] == "running"
           - result_chrome.container['State']['Restarting'] == false
           - result_web.container['State']['Status'] == "running"
           - result_web.container['State']['Restarting'] == false
-          - result_workers.container['State']['Status'] == "running"
-          - result_workers.container['State']['Restarting'] == false

roles/hoarder/molecule/default/verify_stopped.yml

@@ -13,12 +13,6 @@
         state: absent
       register: result_search
 
-    - name: Try and stop and remove hoarder redis
-      community.docker.docker_container:
-        name: "{{ hoarder_redis_container_name }}"
-        state: absent
-      register: result_redis
-
     - name: Try and stop and remove hoarder chrome
       community.docker.docker_container:
         name: "{{ hoarder_chrome_container_name }}"
@@ -31,17 +25,9 @@
         state: absent
       register: result_web
 
-    - name: Try and stop and remove hoarder workers
-      community.docker.docker_container:
-        name: "{{ hoarder_workers_container_name }}"
-        state: absent
-      register: result_workers
-
     - name: Check if hoarder is stopped
       ansible.builtin.assert:
         that:
           - not result_search.changed
-          - not result_redis.changed
           - not result_chrome.changed
           - not result_web.changed
-          - not result_workers.changed

roles/hoarder/tasks/main.yml

@@ -45,28 +45,6 @@
       retries: 20
       delay: 30
 
-    - name: Create Hoarder Redis Docker Container
-      community.docker.docker_container:
-        container_default_behavior: no_defaults
-        name: "{{ hoarder_redis_container_name }}"
-        image: "{{ hoarder_redis_image_name }}:{{ hoarder_redis_image_version }}"
-        pull: true
-        volumes:
-          - "{{ hoarder_data_directory }}/redis:/data:rw"
-        networks:
-          - name: "{{ hoarder_network_name }}"
-        network_mode: "{{ hoarder_network_name }}"
-        labels:
-          traefik.enable: "false"
-        restart_policy: always
-        memory: "{{ hoarder_redis_memory }}"
-        command: ['redis-server', '--requirepass', "{{ hoarder_redis_password }}"]
-        healthcheck:
-          test: ['CMD-SHELL', 'redis-cli --pass {{ hoarder_redis_password }} ping | grep PONG']
-          interval: 10s
-          timeout: 5s
-          retries: 5
-
     - name: Create Hoarder Chrome Container
       community.docker.docker_container:
         container_default_behavior: no_defaults
@@ -111,23 +89,6 @@
           traefik.http.routers.hoarder.tls.domains[0].main: "{{ ansible_nas_domain }}"
           traefik.http.routers.hoarder.tls.domains[0].sans: "*.{{ ansible_nas_domain }}"
           traefik.http.services.hoarder.loadbalancer.server.port: "3000"
-
-    - name: Create Hoarder Workers Docker Container
-      community.docker.docker_container:
-        container_default_behavior: no_defaults
-        name: "{{ hoarder_workers_container_name }}"
-        image: "{{ hoarder_workers_image_name }}:{{ hoarder_workers_image_version }}"
-        pull: true
-        volumes:
-          - "{{ hoarder_data_directory }}/data:/data"
-        networks:
-          - name: "{{ hoarder_network_name }}"
-        network_mode: "{{ hoarder_network_name }}"
-        env: "{{ hoarder_env }}"
-        restart_policy: unless-stopped
-        memory: "{{ hoarder_workers_memory }}"
-        labels:
-          traefik.enable: "false"
   when: hoarder_enabled is true
 
 - name: Stop Hoarder
@@ -137,11 +98,6 @@
         name: "{{ hoarder_search_container_name }}"
         state: absent
 
-    - name: Stop Hoarder redis
-      community.docker.docker_container:
-        name: "{{ hoarder_redis_container_name }}"
-        state: absent
-
     - name: Stop Hoarder chrome
       community.docker.docker_container:
         name: "{{ hoarder_chrome_container_name }}"
@@ -152,8 +108,4 @@
         name: "{{ hoarder_web_container_name }}"
         state: absent
 
-    - name: Stop Hoarder workers
-      community.docker.docker_container:
-        name: "{{ hoarder_workers_container_name }}"
-        state: absent
   when: hoarder_enabled is false