Merge pull request #15 from TianyiQ/main

fix(evaluations): increase memory util in inference

src/abstractions/backends.py

@@ -316,7 +316,7 @@ def vllm_process_batch(
             )
 
         with open(os.devnull, "w") as devnull:
-            frac_static = 0.8 if purpose == "responses" else 0.4
+            frac_static = 0.8 if purpose == "responses" else 0.7
             prefill_size = 8192 if purpose == "responses" else 1024
 
             model_size = get_model_size(model_repoid_or_path)