diff --git a/roles/hoarder/defaults/main.yml b/roles/hoarder/defaults/main.yml index 0976057ac..19849650e 100644 --- a/roles/hoarder/defaults/main.yml +++ b/roles/hoarder/defaults/main.yml @@ -13,24 +13,16 @@ hoarder_network_name: "hoarder" # specs hoarder_web_memory: 1g -hoarder_workers_memory: 1g hoarder_chrome_memory: 1g hoarder_search_memory: 1g -hoarder_redis_memory: 1g # docker hoarder_web_container_name: hoarder-web -hoarder_web_image_name: "ghcr.io/hoarder-app/hoarder-web" +hoarder_web_image_name: "ghcr.io/hoarder-app/hoarder" hoarder_web_image_version: "release" -hoarder_workers_container_name: hoarder-workers -hoarder_workers_image_name: "ghcr.io/hoarder-app/hoarder-workers" -hoarder_workers_image_version: "release" -hoarder_redis_container_name: hoarder-redis -hoarder_redis_image_name: "redis" -hoarder_redis_image_version: "7.2-alpine" hoarder_search_container_name: hoarder-search hoarder_search_image_name: "getmeili/meilisearch" -hoarder_search_image_version: "v1.6" +hoarder_search_image_version: "v1.11.1" hoarder_chrome_container_name: hoarder-chrome hoarder_chrome_image_name: "gcr.io/zenika-hub/alpine-chrome" hoarder_chrome_image_version: "123" @@ -41,22 +33,29 @@ hoarder_group_id: "1000" hoarder_data_dir: "/data" hoarder_nextauth_url: "https://{{ hoarder_hostname }}.{{ ansible_nas_domain }}" hoarder_nextauth_secret: "iVfxDV9JZlcBCRjsk4f+TZlMf6GelJ0Q1KVQCG46m4xPTdi4" -hoarder_redis_host: "{{ hoarder_redis_container_name }}" -hoarder_redis_port: "6379" -hoarder_redis_db_idx: "0" -hoarder_redis_password: "redis" hoarder_meili_addr: "http://{{ hoarder_search_container_name }}:7700" hoarder_meili_master_key: "Ot5yRmb6WVnKhwnAdMGEtgBjd0bu6PD5z+PtavENImfsr1ml" hoarder_meili_no_analytics: "true" hoarder_disable_signups: "false" +hoarder_disable_password_auth: "false" +hoarder_oauth_wellknown_url: "" +hoarder_oauth_client_secret: "" +hoarder_oauth_client_id: "" +hoarder_oauth_scope: "openid email profile" +hoarder_oauth_provider_name: "Custom Provider" +hoarder_oauth_allow_dangerous_emailo_account_linking: "false" hoarder_max_asset_size_mb: "4" hoarder_disable_new_release_check: "false" hoarder_openai_api_key: "" hoarder_openai_base_url: "" hoarder_ollama_base_url: "" -hoarder_inference_text_model: "gpt-3.5-turbo-0125" -hoarder_inference_image_model: "gpt-4o-2024-05-13" +hoarder_ollama_keep_alive: "" +hoarder_inference_text_model: "gpt-4o-mini" +hoarder_inference_image_model: "gpt-4o-mini" hoarder_inference_lang: "english" +hoarder_embedding_text_model: "text-embedding-3-small" +hoarder_inference_context_length: "2048" +hoarder_inference_job_timeout_sec: "30" hoarder_crawler_num_workers: "1" hoarder_browser_web_url: "http://{{ hoarder_chrome_container_name }}:9222" hoarder_browser_websocket_url: "" @@ -67,27 +66,41 @@ hoarder_crawler_full_page_screenshot: "false" hoarder_crawler_full_page_archive: "false" hoarder_crawler_job_timeout_sec: "60" hoarder_crawler_navigate_timeout_sec: "30" +hoarder_crawler_video_downaload: "false" +hoarder_crawler_video_downaload_max_size: "50" +hoarder_crawler_video_downaload_timeout_sec: "600" +hoarder_crawler_enable_adblocker: "true" +hoarder_ocr_cache_dir: "$TEMP_DIR" +hoarder_ocr_langs: "eng" +hoarder_ocr_confidence_threshold: "50" hoarder_env: DATA_DIR: "{{ hoarder_data_dir }}" NEXTAUTH_URL: "{{ hoarder_nextauth_url }}" NEXTAUTH_SECRET: "{{ hoarder_nextauth_secret }}" - REDIS_HOST: "{{ hoarder_redis_host }}" - REDIS_PORT: "{{ hoarder_redis_port }}" - REDIS_DB_IDX: "{{ hoarder_redis_db_idx }}" - REDIS_PASSWORD: "{{ hoarder_redis_password }}" MEILI_ADDR: "{{ hoarder_meili_addr }}" MEILI_MASTER_KEY: "{{ hoarder_meili_master_key }}" MEILI_NO_ANALYTICS: "{{ hoarder_meili_no_analytics }}" DISABLE_SIGNUPS: "{{ hoarder_disable_signups }}" + DISABLE_PASSWORD_AUTH: "{{ hoarder_disable_password_auth }}" + # OAUTH_WELLKNOWN_URL: "{{ hoarder_oauth_wellknown_url }}" + # OAUTH_CLIENT_SECRET: "{{ hoarder_oauth_client_secret }}" + # OAUTH_CLIENT_ID: "{{ hoarder_oauth_client_id }}" + # OAUTH_SCOPE: "{{ hoarder_oauth_scope }}" + # OAUTH_PROVIDER_NAME: "{{ hoarder_oauth_provider_name }}" + OAUTH_ALLOW_DANGEROUS_EMAIL_ACCOUNT_LINKING: "{{ hoarder_oauth_allow_dangerous_emailo_account_linking }}" MAX_ASSET_SIZE_MB: "{{ hoarder_max_asset_size_mb }}" DISABLE_NEW_RELEASE_CHECK: "{{ hoarder_disable_new_release_check }}" OPENAI_API_KEY: "{{ hoarder_openai_api_key }}" # OPENAI_BASE_URL: "{{ hoarder_openai_base_url }}" # OLLAMA_BASE_URL: "{{ hoarder_ollama_base_url }}" + # OLLAMA_KEEP_ALIVE: "{{ hoarder_ollama_keep_alive }}" INFERENCE_TEXT_MODEL: "{{ hoarder_inference_text_model }}" INFERENCE_IMAGE_MODEL: "{{ hoarder_inference_image_model }}" INFERENCE_LANG: "{{ hoarder_inference_lang }}" + INFERENCE_CONTEXT_LENGTH: "{{ hoarder_inference_context_length }}" + INFERENCE_JOB_TIMEOUT_SEC: "{{ hoarder_inference_job_timeout_sec }}" + EMBEDDING_TEXT_MODEL: "{{ hoarder_embedding_text_model }}" CRAWLER_NUM_WORKERS: "{{ hoarder_crawler_num_workers }}" BROWSER_WEB_URL: "{{ hoarder_browser_web_url }}" # BROWSER_WEBSOCKET_URL: "{{ hoarder_browser_websocket_url }}" @@ -98,3 +111,10 @@ hoarder_env: CRAWLER_FULL_PAGE_ARCHIVE: "{{ hoarder_crawler_full_page_archive }}" CRAWLER_JOB_TIMEOUT_SEC: "{{ hoarder_crawler_job_timeout_sec }}" CRAWLER_NAVIGATE_TIMEOUT_SEC: "{{ hoarder_crawler_navigate_timeout_sec }}" + CRAWLER_VIDEO_DOWNLOAD: "{{ hoarder_crawler_video_downaload }}" + CRAWLER_VIDEO_DOWNLOAD_MAX_SIZE: "{{ hoarder_crawler_video_downaload_max_size }}" + CRAWLER_VIDEO_DOWNLOAD_TIMEOUT_SEC: "{{ hoarder_crawler_video_downaload_timeout_sec }}" + CRAWLER_ENABLE_ADBLOCKER: "{{ hoarder_crawler_enable_adblocker }}" + OCR_CACHE_DIR: "{{ hoarder_ocr_cache_dir }}" + OCR_LANGS: "{{ hoarder_ocr_langs }}" + OCR_CONFIDENCE_THRESHOLD: "{{ hoarder_ocr_confidence_threshold }}" diff --git a/roles/hoarder/molecule/default/verify.yml b/roles/hoarder/molecule/default/verify.yml index 415b48ce8..d0e2a687f 100644 --- a/roles/hoarder/molecule/default/verify.yml +++ b/roles/hoarder/molecule/default/verify.yml @@ -12,11 +12,6 @@ name: "{{ hoarder_search_container_name }}" register: result_search - - name: Get hoarder redis container state - community.docker.docker_container: - name: "{{ hoarder_redis_container_name }}" - register: result_redis - - name: Get hoarder chrome container state community.docker.docker_container: name: "{{ hoarder_chrome_container_name }}" @@ -27,21 +22,12 @@ name: "{{ hoarder_web_container_name }}" register: result_web - - name: Get hoarder workers container state - community.docker.docker_container: - name: "{{ hoarder_workers_container_name }}" - register: result_workers - - name: Check if hoarder containers are running ansible.builtin.assert: that: - result_search.container['State']['Status'] == "running" - result_search.container['State']['Restarting'] == false - - result_redis.container['State']['Status'] == "running" - - result_redis.container['State']['Restarting'] == false - result_chrome.container['State']['Status'] == "running" - result_chrome.container['State']['Restarting'] == false - result_web.container['State']['Status'] == "running" - result_web.container['State']['Restarting'] == false - - result_workers.container['State']['Status'] == "running" - - result_workers.container['State']['Restarting'] == false diff --git a/roles/hoarder/molecule/default/verify_stopped.yml b/roles/hoarder/molecule/default/verify_stopped.yml index 4f82c144b..d02b241ec 100644 --- a/roles/hoarder/molecule/default/verify_stopped.yml +++ b/roles/hoarder/molecule/default/verify_stopped.yml @@ -13,12 +13,6 @@ state: absent register: result_search - - name: Try and stop and remove hoarder redis - community.docker.docker_container: - name: "{{ hoarder_redis_container_name }}" - state: absent - register: result_redis - - name: Try and stop and remove hoarder chrome community.docker.docker_container: name: "{{ hoarder_chrome_container_name }}" @@ -31,17 +25,9 @@ state: absent register: result_web - - name: Try and stop and remove hoarder workers - community.docker.docker_container: - name: "{{ hoarder_workers_container_name }}" - state: absent - register: result_workers - - name: Check if hoarder is stopped ansible.builtin.assert: that: - not result_search.changed - - not result_redis.changed - not result_chrome.changed - not result_web.changed - - not result_workers.changed diff --git a/roles/hoarder/tasks/main.yml b/roles/hoarder/tasks/main.yml index b8f72f437..e35f7d20d 100644 --- a/roles/hoarder/tasks/main.yml +++ b/roles/hoarder/tasks/main.yml @@ -45,28 +45,6 @@ retries: 20 delay: 30 - - name: Create Hoarder Redis Docker Container - community.docker.docker_container: - container_default_behavior: no_defaults - name: "{{ hoarder_redis_container_name }}" - image: "{{ hoarder_redis_image_name }}:{{ hoarder_redis_image_version }}" - pull: true - volumes: - - "{{ hoarder_data_directory }}/redis:/data:rw" - networks: - - name: "{{ hoarder_network_name }}" - network_mode: "{{ hoarder_network_name }}" - labels: - traefik.enable: "false" - restart_policy: always - memory: "{{ hoarder_redis_memory }}" - command: ['redis-server', '--requirepass', "{{ hoarder_redis_password }}"] - healthcheck: - test: ['CMD-SHELL', 'redis-cli --pass {{ hoarder_redis_password }} ping | grep PONG'] - interval: 10s - timeout: 5s - retries: 5 - - name: Create Hoarder Chrome Container community.docker.docker_container: container_default_behavior: no_defaults @@ -111,23 +89,6 @@ traefik.http.routers.hoarder.tls.domains[0].main: "{{ ansible_nas_domain }}" traefik.http.routers.hoarder.tls.domains[0].sans: "*.{{ ansible_nas_domain }}" traefik.http.services.hoarder.loadbalancer.server.port: "3000" - - - name: Create Hoarder Workers Docker Container - community.docker.docker_container: - container_default_behavior: no_defaults - name: "{{ hoarder_workers_container_name }}" - image: "{{ hoarder_workers_image_name }}:{{ hoarder_workers_image_version }}" - pull: true - volumes: - - "{{ hoarder_data_directory }}/data:/data" - networks: - - name: "{{ hoarder_network_name }}" - network_mode: "{{ hoarder_network_name }}" - env: "{{ hoarder_env }}" - restart_policy: unless-stopped - memory: "{{ hoarder_workers_memory }}" - labels: - traefik.enable: "false" when: hoarder_enabled is true - name: Stop Hoarder @@ -137,11 +98,6 @@ name: "{{ hoarder_search_container_name }}" state: absent - - name: Stop Hoarder redis - community.docker.docker_container: - name: "{{ hoarder_redis_container_name }}" - state: absent - - name: Stop Hoarder chrome community.docker.docker_container: name: "{{ hoarder_chrome_container_name }}" @@ -152,8 +108,4 @@ name: "{{ hoarder_web_container_name }}" state: absent - - name: Stop Hoarder workers - community.docker.docker_container: - name: "{{ hoarder_workers_container_name }}" - state: absent when: hoarder_enabled is false