From 55ba5e508aa43881a139b6d55885f2e5cf1d7005 Mon Sep 17 00:00:00 2001 From: Patrick Hammer Date: Wed, 23 Aug 2023 07:19:56 +0200 Subject: [PATCH] Update: paths corrected, less tolerance for term replacement --- .../1_GenerateTestOutput.py | 13 +++++++------ .../2_EvaluateTestOutput.py | 6 +++--- .../3_CheckDifferencesToLastRun.py | 0 {Evaluation => Evaluation_INT_inf}/Correct.json | 0 .../INT_Inf_benchmarkTest.csv | 0 {Evaluation => Evaluation_INT_inf}/Incorrect.json | 0 {Evaluation => Evaluation_INT_inf}/Scores.json | 0 {Evaluation => Evaluation_INT_inf}/TestOutput.json | 0 {Evaluation => Evaluation_INT_inf}/evaluation.sh | 2 +- Evaluation_babI_qa16/1_GenerateTestOutput.py | 9 +++++---- Evaluation_babI_qa16/2_EvaluateTestOutput.py | 6 +++--- NarsGPT.py | 4 ++-- OpenNARS-for-Applications | 2 +- 13 files changed, 22 insertions(+), 20 deletions(-) rename {Evaluation => Evaluation_INT_inf}/1_GenerateTestOutput.py (84%) rename {Evaluation => Evaluation_INT_inf}/2_EvaluateTestOutput.py (87%) rename {Evaluation => Evaluation_INT_inf}/3_CheckDifferencesToLastRun.py (100%) rename {Evaluation => Evaluation_INT_inf}/Correct.json (100%) rename {Evaluation => Evaluation_INT_inf}/INT_Inf_benchmarkTest.csv (100%) rename {Evaluation => Evaluation_INT_inf}/Incorrect.json (100%) rename {Evaluation => Evaluation_INT_inf}/Scores.json (100%) rename {Evaluation => Evaluation_INT_inf}/TestOutput.json (100%) rename {Evaluation => Evaluation_INT_inf}/evaluation.sh (74%) diff --git a/Evaluation/1_GenerateTestOutput.py b/Evaluation_INT_inf/1_GenerateTestOutput.py similarity index 84% rename from Evaluation/1_GenerateTestOutput.py rename to Evaluation_INT_inf/1_GenerateTestOutput.py index d0ca269..8d26e7b 100644 --- a/Evaluation/1_GenerateTestOutput.py +++ b/Evaluation_INT_inf/1_GenerateTestOutput.py @@ -1,5 +1,7 @@ import sys import os +with open("INT_Inf_benchmarkTest.csv") as f: + lines = f.read().split("\n") cwd = os.getcwd() sys.path.append(cwd + "/../") os.chdir(cwd + "/../") @@ -9,13 +11,12 @@ Line_Input_Output_ExpectedOutput = [] Line = 1 -while True: - try: - line = input() - except EOFError: - exit(0) +for line in lines: parts = ",".join(line.split(",")[1:]).split(",,,,,,") - Input, expectedOutput = parts + try: + Input, expectedOutput = parts + except: + continue Input = Input.strip() expectedOutput = expectedOutput.strip() if expectedOutput != "": diff --git a/Evaluation/2_EvaluateTestOutput.py b/Evaluation_INT_inf/2_EvaluateTestOutput.py similarity index 87% rename from Evaluation/2_EvaluateTestOutput.py rename to Evaluation_INT_inf/2_EvaluateTestOutput.py index e39af37..54f16a0 100644 --- a/Evaluation/2_EvaluateTestOutput.py +++ b/Evaluation_INT_inf/2_EvaluateTestOutput.py @@ -7,7 +7,7 @@ if x.startswith("API_KEY="): openai.api_key = x.split("API_KEY=")[1] -with open("TestOutput.json") as json_file: +with open("../TestOutput.json") as json_file: ListOfDicts, _ = json.load(json_file) # {"Line": Line, "Input": Input, "actualOutput": actualOutput, "expectedOutput": expectedOutput} @@ -36,8 +36,8 @@ send_prompt = PROMPT.replace("_QUESTION_", Input).replace("_ACTUAL_OUTPUT_",actualOutput).replace("_EXPECTED_OUTPUT_",expectedOutput) print(send_prompt) while True: - try: - response = openai.ChatCompletion.create(model='gpt-4', messages=[ {"role": "user", "content": send_prompt}], max_tokens=200, temperature=0) + try: #'gpt-4' + response = openai.ChatCompletion.create(model='gpt-3.5-turbo', messages=[ {"role": "user", "content": send_prompt}], max_tokens=200, temperature=0) ret = response['choices'][0]['message']['content'] except: print("Error: API call failed, will try repeating it in 10 seconds!") diff --git a/Evaluation/3_CheckDifferencesToLastRun.py b/Evaluation_INT_inf/3_CheckDifferencesToLastRun.py similarity index 100% rename from Evaluation/3_CheckDifferencesToLastRun.py rename to Evaluation_INT_inf/3_CheckDifferencesToLastRun.py diff --git a/Evaluation/Correct.json b/Evaluation_INT_inf/Correct.json similarity index 100% rename from Evaluation/Correct.json rename to Evaluation_INT_inf/Correct.json diff --git a/Evaluation/INT_Inf_benchmarkTest.csv b/Evaluation_INT_inf/INT_Inf_benchmarkTest.csv similarity index 100% rename from Evaluation/INT_Inf_benchmarkTest.csv rename to Evaluation_INT_inf/INT_Inf_benchmarkTest.csv diff --git a/Evaluation/Incorrect.json b/Evaluation_INT_inf/Incorrect.json similarity index 100% rename from Evaluation/Incorrect.json rename to Evaluation_INT_inf/Incorrect.json diff --git a/Evaluation/Scores.json b/Evaluation_INT_inf/Scores.json similarity index 100% rename from Evaluation/Scores.json rename to Evaluation_INT_inf/Scores.json diff --git a/Evaluation/TestOutput.json b/Evaluation_INT_inf/TestOutput.json similarity index 100% rename from Evaluation/TestOutput.json rename to Evaluation_INT_inf/TestOutput.json diff --git a/Evaluation/evaluation.sh b/Evaluation_INT_inf/evaluation.sh similarity index 74% rename from Evaluation/evaluation.sh rename to Evaluation_INT_inf/evaluation.sh index d31346c..7564759 100644 --- a/Evaluation/evaluation.sh +++ b/Evaluation_INT_inf/evaluation.sh @@ -1,6 +1,6 @@ move Correct.json CorrectBefore.json move Incorrect.json IncorrectBefore.json -python3 1_GenerateTestOutput.py API_KEY=YOUR_KEY < INT_Inf_benchmarkTest.csv +python3 1_GenerateTestOutput.py API_KEY=YOUR_KEY python3 2_EvaluateTestOutput.py API_KEY=YOUR_KEY python3 3_CheckDifferencesToLastRun.py > differences.txt cat differences.txt diff --git a/Evaluation_babI_qa16/1_GenerateTestOutput.py b/Evaluation_babI_qa16/1_GenerateTestOutput.py index b5200a7..3e8c830 100644 --- a/Evaluation_babI_qa16/1_GenerateTestOutput.py +++ b/Evaluation_babI_qa16/1_GenerateTestOutput.py @@ -1,5 +1,10 @@ import sys import os + +lines = ""#"../../babI/task_16_v1-2/en-valid/qa16_test.txt" +with open("../../babI/task_16_v1-2/en-valid/qa16_train.txt") as f: + lines = f.read().split("\n") + cwd = os.getcwd() sys.path.append(cwd + "/../") os.chdir(cwd + "/../") @@ -7,10 +12,6 @@ os.chdir(cwd) import json -lines = "" -with open("/home/tc/babI/tasks_1-20_v1-2/en-valid/qa16_train.txt") as f: - lines = f.read().split("\n") - lastnum = -1 examples = [] example_cur = [] diff --git a/Evaluation_babI_qa16/2_EvaluateTestOutput.py b/Evaluation_babI_qa16/2_EvaluateTestOutput.py index fc53beb..e8da028 100644 --- a/Evaluation_babI_qa16/2_EvaluateTestOutput.py +++ b/Evaluation_babI_qa16/2_EvaluateTestOutput.py @@ -7,7 +7,7 @@ if x.startswith("API_KEY="): openai.api_key = x.split("API_KEY=")[1] -with open("TestOutput_.json") as json_file: +with open("../TestOutput.json") as json_file: ListOfDicts = json.load(json_file) # {"Line": Line, "Input": Input, "actualOutput": actualOutput, "expectedOutput": expectedOutput} @@ -36,8 +36,8 @@ send_prompt = PROMPT.replace("_QUESTION_", Input).replace("_ACTUAL_OUTPUT_",actualOutput).replace("_EXPECTED_OUTPUT_",expectedOutput) print(send_prompt) while True: - try: - response = openai.ChatCompletion.create(model='gpt-4', messages=[ {"role": "user", "content": send_prompt}], max_tokens=200, temperature=0) + try: #'gpt-4' + response = openai.ChatCompletion.create(model='gpt-3.5-turbo', messages=[ {"role": "user", "content": send_prompt}], max_tokens=200, temperature=0) ret = response['choices'][0]['message']['content'] except: print("Error: API call failed, will try repeating it in 10 seconds!") diff --git a/NarsGPT.py b/NarsGPT.py index 76072cd..e38579c 100644 --- a/NarsGPT.py +++ b/NarsGPT.py @@ -31,11 +31,11 @@ import time openai.api_key = "YOUR_KEY" -usedModel = "gpt-4" +usedModel = "gpt-4" #'gpt-3.5-turbo' relevantViewSize = 30 #how many relevant (judged by statement embedding) ONA memory items GPT can see recentViewSize = 10 #how many recent (judged by lastUsed) ONA memory items GPT can see eternalizationDistance = 3 #how long items are treated as events before contributing to generic belief evidence in long-term memory -atomCreationThreshold = 0.88 #how different a new word needs to be to existing of same type to become a new atom +atomCreationThreshold = 0.95 #how different a new word needs to be to existing of same type to become a new atom filename = "mem.json" #the system's memory file IYouExchange = True or "NoIYouExchange" in sys.argv #whether I and you, my and your is exchanged in communication ConsiderGPTKnowledge = False or "ConsiderGPTKnowledge" in sys.argv #Whether it should be allowed to consider GPT's knowledge too for answering a question diff --git a/OpenNARS-for-Applications b/OpenNARS-for-Applications index 6507eed..db9d173 160000 --- a/OpenNARS-for-Applications +++ b/OpenNARS-for-Applications @@ -1 +1 @@ -Subproject commit 6507eed48a2c5172a26c451d500c06955b8581a1 +Subproject commit db9d173ba985c6cb1c5810e57acc6bd8ef148988