From 3c5a7f521e0a62279011b7141bbaccd105b1dd21 Mon Sep 17 00:00:00 2001 From: Robert Shaw Date: Mon, 24 Jun 2024 01:08:40 +0000 Subject: [PATCH] remove flaky gptq models --- neuralmagic/lm-eval/full-small-models.yaml | 12 ------------ neuralmagic/lm-eval/smoke-small-models.yaml | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/neuralmagic/lm-eval/full-small-models.yaml b/neuralmagic/lm-eval/full-small-models.yaml index 129ea4c5bf99c..adb00ba65c1f1 100644 --- a/neuralmagic/lm-eval/full-small-models.yaml +++ b/neuralmagic/lm-eval/full-small-models.yaml @@ -9,15 +9,3 @@ value: 0.74 limit: 250 num_fewshot: 5 - -# ./nm-run-lm-eval-gsm-hf-baseline -m TechxGenus/Meta-Llama-3-8B-Instruct-GPTQ -b 32 -d cuda -l 250 -f 5 -- model_name: "TechxGenus/Meta-Llama-3-8B-Instruct-GPTQ" - tasks: - - name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.684 - - name: "exact_match,flexible-extract" - value: 0.688 - limit: 250 - num_fewshot: 5 diff --git a/neuralmagic/lm-eval/smoke-small-models.yaml b/neuralmagic/lm-eval/smoke-small-models.yaml index c9ecd35bf793d..546a221872af8 100644 --- a/neuralmagic/lm-eval/smoke-small-models.yaml +++ b/neuralmagic/lm-eval/smoke-small-models.yaml @@ -9,15 +9,3 @@ value: 0.74 limit: 250 num_fewshot: 5 - -# ./nm-run-lm-eval-gsm-hf-baseline.sh -m TechxGenus/Meta-Llama-3-8B-Instruct-GPTQ -b 32 -d cuda -l 250 -f 5 -- model_name: "TechxGenus/Meta-Llama-3-8B-Instruct-GPTQ" - tasks: - - name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.684 - - name: "exact_match,flexible-extract" - value: 0.688 - limit: 250 - num_fewshot: 5