minor adjustment to llama.cpp server call

KatherLab · Nov 11, 2024 · 12ae811 · 12ae811
1 parent 8fc7c88
commit 12ae811
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/.gitignore b/.gitignore
@@ -5,4 +5,5 @@ logs*
 __pycache__
 
 test_*.py
+test_*
 output.pdf
diff --git a/webapp/llm_processing/routes.py b/webapp/llm_processing/routes.py
@@ -227,7 +227,9 @@ def extract_from_report(
                 "-b",
                 "2048",
                 "-ub",
-                "2048",
+                "512",
+                "-t",
+                "16",
             ] + (["--verbose"] if verbose_llama else []) + (["--mlock"] if mlock else []) +
             (["-ctk", kv_cache_type, "-ctv", kv_cache_type] if kv_cache_type != "" else []) + 
             (["-sm", "none", "-mg", str(gpu)] if gpu not in ["all", "ALL", "mps", ""] else [])+