From d4cecf9fd093edec3e5a0724ea26d820538df0f5 Mon Sep 17 00:00:00 2001 From: itlackey Date: Sun, 29 Sep 2024 18:58:57 -0500 Subject: [PATCH 1/9] - Added codegeex4 --- aider/website/_data/edit_leaderboard.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/aider/website/_data/edit_leaderboard.yml b/aider/website/_data/edit_leaderboard.yml index d4bcd8786e5..338812739f4 100644 --- a/aider/website/_data/edit_leaderboard.yml +++ b/aider/website/_data/edit_leaderboard.yml @@ -1315,4 +1315,27 @@ date: 2024-09-28 versions: 0.57.2.dev seconds_per_case: 67.2 + total_cost: 0.0000 + +- dirname: 2024-09-29-17-51-11--codegeex4-whole-2 + test_cases: 133 + model: ollama/codegeex4 + edit_format: whole + commit_hash: 228ae24 + pass_rate_1: 28.6 + pass_rate_2: 32.3 + percent_cases_well_formed: 97.0 + error_outputs: 20 + num_malformed_responses: 20 + num_with_malformed_responses: 4 + user_asks: 56 + lazy_comments: 5 + syntax_errors: 5 + indentation_errors: 0 + exhausted_context_windows: 0 + test_timeouts: 4 + command: aider --model ollama/codegeex4 + date: 2024-09-29 + versions: 0.57.2.dev + seconds_per_case: 128.1 total_cost: 0.0000 \ No newline at end of file From e2d5f15aff965f275500590fae58e509bc1cf54c Mon Sep 17 00:00:00 2001 From: itlackey Date: Sun, 29 Sep 2024 22:47:38 -0500 Subject: [PATCH 2/9] - Added wojtek/opencodeinterpreter:6.7b --- aider/website/_data/edit_leaderboard.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/aider/website/_data/edit_leaderboard.yml b/aider/website/_data/edit_leaderboard.yml index 338812739f4..5ab9ce7cee3 100644 --- a/aider/website/_data/edit_leaderboard.yml +++ b/aider/website/_data/edit_leaderboard.yml @@ -1338,4 +1338,27 @@ date: 2024-09-29 versions: 0.57.2.dev seconds_per_case: 128.1 + total_cost: 0.0000 + +- dirname: 2024-09-30-00-09-00--wojtek-opencodeinterpreter-6.7b-whole-2 + test_cases: 133 + model: ollama/wojtek/opencodeinterpreter:6.7b + edit_format: whole + commit_hash: 6d586fd + pass_rate_1: 26.3 + pass_rate_2: 30.1 + percent_cases_well_formed: 91.0 + error_outputs: 18 + num_malformed_responses: 18 + num_with_malformed_responses: 12 + user_asks: 79 + lazy_comments: 7 + syntax_errors: 0 + indentation_errors: 1 + exhausted_context_windows: 0 + test_timeouts: 6 + command: aider --model ollama/wojtek/opencodeinterpreter:6.7b + date: 2024-09-30 + versions: 0.58.1.dev + seconds_per_case: 59.3 total_cost: 0.0000 \ No newline at end of file From e381260932ec60dbdb2148f71c367034e30a5107 Mon Sep 17 00:00:00 2001 From: itlackey Date: Mon, 30 Sep 2024 01:26:55 -0500 Subject: [PATCH 3/9] - Added mistral-nemo:12b-instruct-2407-q4_K_M --- aider/website/_data/edit_leaderboard.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/aider/website/_data/edit_leaderboard.yml b/aider/website/_data/edit_leaderboard.yml index 5ab9ce7cee3..473681500e8 100644 --- a/aider/website/_data/edit_leaderboard.yml +++ b/aider/website/_data/edit_leaderboard.yml @@ -1361,4 +1361,27 @@ date: 2024-09-30 versions: 0.58.1.dev seconds_per_case: 59.3 + total_cost: 0.0000 + +- dirname: 2024-09-30-03-49-01--mistral-nemo-12b-instruct-2407-q4_K_M-whole-1 + test_cases: 133 + model: ollama/mistral-nemo:12b-instruct-2407-q4_K_M + edit_format: whole + commit_hash: ba4dec8 + pass_rate_1: 22.6 + pass_rate_2: 33.1 + percent_cases_well_formed: 100.0 + error_outputs: 0 + num_malformed_responses: 0 + num_with_malformed_responses: 0 + user_asks: 53 + lazy_comments: 37 + syntax_errors: 2 + indentation_errors: 2 + exhausted_context_windows: 0 + test_timeouts: 2 + command: aider --model ollama/mistral-nemo:12b-instruct-2407-q4_K_M + date: 2024-09-30 + versions: 0.58.1.dev + seconds_per_case: 34.7 total_cost: 0.0000 \ No newline at end of file From fb34af436221b462ab770bf07874d2239411525c Mon Sep 17 00:00:00 2001 From: itlackey Date: Mon, 30 Sep 2024 14:18:21 -0500 Subject: [PATCH 4/9] - Added qwen2.5:32b --- aider/website/_data/edit_leaderboard.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/aider/website/_data/edit_leaderboard.yml b/aider/website/_data/edit_leaderboard.yml index 473681500e8..ff6093a67ef 100644 --- a/aider/website/_data/edit_leaderboard.yml +++ b/aider/website/_data/edit_leaderboard.yml @@ -1384,4 +1384,27 @@ date: 2024-09-30 versions: 0.58.1.dev seconds_per_case: 34.7 + total_cost: 0.0000 + +- dirname: 2024-09-30-14-09-43--qwen2.5-32b-whole-2 + test_cases: 133 + model: ollama/qwen2.5:32b + edit_format: whole + commit_hash: 765c4cb + pass_rate_1: 44.4 + pass_rate_2: 54.1 + percent_cases_well_formed: 100.0 + error_outputs: 0 + num_malformed_responses: 0 + num_with_malformed_responses: 0 + user_asks: 9 + lazy_comments: 0 + syntax_errors: 0 + indentation_errors: 0 + exhausted_context_windows: 0 + test_timeouts: 3 + command: aider --model ollama/qwen2.5:32b + date: 2024-09-30 + versions: 0.58.1.dev + seconds_per_case: 134.9 total_cost: 0.0000 \ No newline at end of file From 8d4dd5c9de00c554ba6e4e0281b950147e82e3c3 Mon Sep 17 00:00:00 2001 From: itlackey Date: Mon, 30 Sep 2024 17:24:27 -0500 Subject: [PATCH 5/9] - Added llama3.2:3b-instruct-fp16 --- aider/website/_data/edit_leaderboard.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/aider/website/_data/edit_leaderboard.yml b/aider/website/_data/edit_leaderboard.yml index ff6093a67ef..c7a18d0d96e 100644 --- a/aider/website/_data/edit_leaderboard.yml +++ b/aider/website/_data/edit_leaderboard.yml @@ -1407,4 +1407,27 @@ date: 2024-09-30 versions: 0.58.1.dev seconds_per_case: 134.9 + total_cost: 0.0000 + +- dirname: 2024-09-30-19-35-40--llama3.2-3b-instruct-fp16-whole-1 + test_cases: 133 + model: ollama/llama3.2:3b-instruct-fp16 + edit_format: whole + commit_hash: 3f12290 + pass_rate_1: 20.3 + pass_rate_2: 26.3 + percent_cases_well_formed: 97.0 + error_outputs: 21 + num_malformed_responses: 21 + num_with_malformed_responses: 4 + user_asks: 73 + lazy_comments: 11 + syntax_errors: 1 + indentation_errors: 3 + exhausted_context_windows: 0 + test_timeouts: 1 + command: aider --model ollama/llama3.2:3b-instruct-fp16 + date: 2024-09-30 + versions: 0.58.1.dev + seconds_per_case: 66.6 total_cost: 0.0000 \ No newline at end of file From 268331d5c0412577659e1a4976707643639bbadc Mon Sep 17 00:00:00 2001 From: itlackey Date: Mon, 30 Sep 2024 20:45:13 -0500 Subject: [PATCH 6/9] - Added hermes3:8b-llama3.1-fp16 --- aider/website/_data/edit_leaderboard.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/aider/website/_data/edit_leaderboard.yml b/aider/website/_data/edit_leaderboard.yml index c7a18d0d96e..b397136f5d7 100644 --- a/aider/website/_data/edit_leaderboard.yml +++ b/aider/website/_data/edit_leaderboard.yml @@ -1430,4 +1430,20 @@ date: 2024-09-30 versions: 0.58.1.dev seconds_per_case: 66.6 - total_cost: 0.0000 \ No newline at end of file + total_cost: 0.0000 +- dirname: 2024-09-30-23-01-24--hermes3-8b-llama3.1-fp16-whole-2 + test_cases: 133 + model: ollama/hermes3:8b-llama3.1-fp16 + edit_format: whole + commit_hash: c5ba4f7 + pass_rate_1: 24.1 + pass_rate_2: 30.1 + percent_cases_well_formed: 98.5 + syntax_errors: 0 + exhausted_context_windows: 0 + command: aider --model ollama/hermes3:8b-llama3.1-fp16 + date: 2024-09-30 + versions: 0.58.1.dev + seconds_per_case: 64.7 + total_cost: 0.0000 + From aa911a847deee61188a2f76a3906f40315252fd6 Mon Sep 17 00:00:00 2001 From: itlackey Date: Tue, 1 Oct 2024 00:51:07 -0500 Subject: [PATCH 7/9] - Added mistral-small --- aider/website/_data/edit_leaderboard.yml | 47 ++++++++++++++++-------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/aider/website/_data/edit_leaderboard.yml b/aider/website/_data/edit_leaderboard.yml index b397136f5d7..7cc683b538d 100644 --- a/aider/website/_data/edit_leaderboard.yml +++ b/aider/website/_data/edit_leaderboard.yml @@ -1431,19 +1431,36 @@ versions: 0.58.1.dev seconds_per_case: 66.6 total_cost: 0.0000 -- dirname: 2024-09-30-23-01-24--hermes3-8b-llama3.1-fp16-whole-2 - test_cases: 133 - model: ollama/hermes3:8b-llama3.1-fp16 - edit_format: whole - commit_hash: c5ba4f7 - pass_rate_1: 24.1 - pass_rate_2: 30.1 - percent_cases_well_formed: 98.5 - syntax_errors: 0 - exhausted_context_windows: 0 - command: aider --model ollama/hermes3:8b-llama3.1-fp16 - date: 2024-09-30 - versions: 0.58.1.dev - seconds_per_case: 64.7 - total_cost: 0.0000 +- dirname: 2024-09-30-23-01-24--hermes3-8b-llama3.1-fp16-whole-2 + test_cases: 133 + model: ollama/hermes3:8b-llama3.1-fp16 + edit_format: whole + commit_hash: c5ba4f7 + pass_rate_1: 24.1 + pass_rate_2: 30.1 + percent_cases_well_formed: 98.5 + syntax_errors: 0 + exhausted_context_windows: 0 + command: aider --model ollama/hermes3:8b-llama3.1-fp16 + date: 2024-09-30 + versions: 0.58.1.dev + seconds_per_case: 64.7 + total_cost: 0.0000 +- dirname: 2024-10-01-02-33-11--mistral-small-whole-1 + test_cases: 133 + model: ollama/mistral-small + edit_format: whole + commit_hash: 8a908fa + pass_rate_1: 30.1 + pass_rate_2: 38.3 + percent_cases_well_formed: 99.2 + lazy_comments: 0 + syntax_errors: 0 + indentation_errors: 0 + exhausted_context_windows: 0 + command: aider --model ollama/mistral-small + date: 2024-10-01 + versions: 0.58.1.dev + seconds_per_case: 84.6 + total_cost: 0.0000 From 61bc130464a4864dab4c058fcc839fc760afc2f2 Mon Sep 17 00:00:00 2001 From: itlackey Date: Tue, 1 Oct 2024 08:55:47 -0500 Subject: [PATCH 8/9] - Added yi-coder:9b-chat-fp16 --- aider/website/_data/edit_leaderboard.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/aider/website/_data/edit_leaderboard.yml b/aider/website/_data/edit_leaderboard.yml index 7cc683b538d..8e275681fea 100644 --- a/aider/website/_data/edit_leaderboard.yml +++ b/aider/website/_data/edit_leaderboard.yml @@ -1431,6 +1431,7 @@ versions: 0.58.1.dev seconds_per_case: 66.6 total_cost: 0.0000 + - dirname: 2024-09-30-23-01-24--hermes3-8b-llama3.1-fp16-whole-2 test_cases: 133 model: ollama/hermes3:8b-llama3.1-fp16 @@ -1464,3 +1465,20 @@ versions: 0.58.1.dev seconds_per_case: 84.6 total_cost: 0.0000 + +- dirname: 2024-10-01-07-05-40--yi-coder-9b-chat-fp16-whole-1 + test_cases: 133 + model: ollama/yi-coder:9b-chat-fp16 + edit_format: whole + commit_hash: 52c6632-dirty + pass_rate_1: 39.8 + pass_rate_2: 43.6 + percent_cases_well_formed: 99.2 + lazy_comments: 0 + indentation_errors: 0 + exhausted_context_windows: 0 + command: aider --model ollama/yi-coder:9b-chat-fp16 + date: 2024-10-01 + versions: 0.58.1.dev + seconds_per_case: 63.7 + total_cost: 0.0000 \ No newline at end of file From d621d16255a2a1eb73dc592bb03536c33093a0b6 Mon Sep 17 00:00:00 2001 From: itlackey Date: Tue, 1 Oct 2024 15:53:57 -0500 Subject: [PATCH 9/9] - Added hermes3 --- aider/website/_data/edit_leaderboard.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/aider/website/_data/edit_leaderboard.yml b/aider/website/_data/edit_leaderboard.yml index 8e275681fea..2952f7a47fe 100644 --- a/aider/website/_data/edit_leaderboard.yml +++ b/aider/website/_data/edit_leaderboard.yml @@ -1481,4 +1481,19 @@ date: 2024-10-01 versions: 0.58.1.dev seconds_per_case: 63.7 + total_cost: 0.0000 + +- dirname: 2024-10-01-16-50-09--hermes3-whole-4 + test_cases: 133 + model: ollama/hermes3 + edit_format: whole + commit_hash: 415e898 + pass_rate_1: 21.1 + pass_rate_2: 22.6 + percent_cases_well_formed: 98.5 + exhausted_context_windows: 0 + command: aider --model ollama/hermes3 + date: 2024-10-01 + versions: 0.58.1.dev + seconds_per_case: 24.8 total_cost: 0.0000 \ No newline at end of file