Update: paths corrected, less tolerance for term replacement

opennars · Aug 23, 2023 · 55ba5e5 · 55ba5e5
1 parent ffe7fa3
commit 55ba5e5
Show file tree

Hide file tree

Showing 13 changed files with 22 additions and 20 deletions.
diff --git a/Evaluation/1_GenerateTestOutput.py → Evaluation_INT_inf/1_GenerateTestOutput.py b/Evaluation/1_GenerateTestOutput.py → Evaluation_INT_inf/1_GenerateTestOutput.py
@@ -1,5 +1,7 @@
 import sys
 import os
+with open("INT_Inf_benchmarkTest.csv") as f:
+    lines = f.read().split("\n")
 cwd = os.getcwd()
 sys.path.append(cwd + "/../")
 os.chdir(cwd + "/../")
@@ -9,13 +11,12 @@
 
 Line_Input_Output_ExpectedOutput = []
 Line = 1
-while True:
-    try:
-        line = input()
-    except EOFError:
-        exit(0)
+for line in lines:
     parts = ",".join(line.split(",")[1:]).split(",,,,,,")
-    Input, expectedOutput = parts
+    try:
+        Input, expectedOutput = parts
+    except:
+        continue
     Input = Input.strip()
     expectedOutput = expectedOutput.strip()
     if expectedOutput != "":

diff --git a/Evaluation/2_EvaluateTestOutput.py → Evaluation_INT_inf/2_EvaluateTestOutput.py b/Evaluation/2_EvaluateTestOutput.py → Evaluation_INT_inf/2_EvaluateTestOutput.py
@@ -7,7 +7,7 @@
     if x.startswith("API_KEY="):
         openai.api_key = x.split("API_KEY=")[1]
 
-with open("TestOutput.json") as json_file:
+with open("../TestOutput.json") as json_file:
     ListOfDicts, _ = json.load(json_file)
 
 # {"Line": Line, "Input": Input, "actualOutput": actualOutput, "expectedOutput": expectedOutput}
@@ -36,8 +36,8 @@
     send_prompt = PROMPT.replace("_QUESTION_", Input).replace("_ACTUAL_OUTPUT_",actualOutput).replace("_EXPECTED_OUTPUT_",expectedOutput)
     print(send_prompt)
     while True:
-        try:
-            response = openai.ChatCompletion.create(model='gpt-4', messages=[ {"role": "user", "content": send_prompt}], max_tokens=200, temperature=0)
+        try:                                            #'gpt-4'
+            response = openai.ChatCompletion.create(model='gpt-3.5-turbo', messages=[ {"role": "user", "content": send_prompt}], max_tokens=200, temperature=0)
             ret = response['choices'][0]['message']['content']
         except:
             print("Error: API call failed, will try repeating it in 10 seconds!")

diff --git a/Evaluation/3_CheckDifferencesToLastRun.py → ...on_INT_inf/3_CheckDifferencesToLastRun.py b/Evaluation/3_CheckDifferencesToLastRun.py → ...on_INT_inf/3_CheckDifferencesToLastRun.py
diff --git a/Evaluation/Correct.json → Evaluation_INT_inf/Correct.json b/Evaluation/Correct.json → Evaluation_INT_inf/Correct.json
diff --git a/Evaluation/INT_Inf_benchmarkTest.csv → Evaluation_INT_inf/INT_Inf_benchmarkTest.csv b/Evaluation/INT_Inf_benchmarkTest.csv → Evaluation_INT_inf/INT_Inf_benchmarkTest.csv
diff --git a/Evaluation/Incorrect.json → Evaluation_INT_inf/Incorrect.json b/Evaluation/Incorrect.json → Evaluation_INT_inf/Incorrect.json
diff --git a/Evaluation/Scores.json → Evaluation_INT_inf/Scores.json b/Evaluation/Scores.json → Evaluation_INT_inf/Scores.json
diff --git a/Evaluation/TestOutput.json → Evaluation_INT_inf/TestOutput.json b/Evaluation/TestOutput.json → Evaluation_INT_inf/TestOutput.json
diff --git a/Evaluation/evaluation.sh → Evaluation_INT_inf/evaluation.sh b/Evaluation/evaluation.sh → Evaluation_INT_inf/evaluation.sh
@@ -1,6 +1,6 @@
 move Correct.json CorrectBefore.json
 move Incorrect.json IncorrectBefore.json
-python3 1_GenerateTestOutput.py API_KEY=YOUR_KEY < INT_Inf_benchmarkTest.csv
+python3 1_GenerateTestOutput.py API_KEY=YOUR_KEY
 python3 2_EvaluateTestOutput.py API_KEY=YOUR_KEY
 python3 3_CheckDifferencesToLastRun.py > differences.txt
 cat differences.txt

diff --git a/Evaluation_babI_qa16/1_GenerateTestOutput.py b/Evaluation_babI_qa16/1_GenerateTestOutput.py
@@ -1,16 +1,17 @@
 import sys
 import os
+
+lines = ""#"../../babI/task_16_v1-2/en-valid/qa16_test.txt"
+with open("../../babI/task_16_v1-2/en-valid/qa16_train.txt") as f:
+    lines = f.read().split("\n")
+
 cwd = os.getcwd()
 sys.path.append(cwd + "/../")
 os.chdir(cwd + "/../")
 from NarsGPT import *
 os.chdir(cwd)
 import json
 
-lines = ""
-with open("/home/tc/babI/tasks_1-20_v1-2/en-valid/qa16_train.txt") as f:
-    lines = f.read().split("\n")
-
 lastnum = -1
 examples = []
 example_cur = []

diff --git a/Evaluation_babI_qa16/2_EvaluateTestOutput.py b/Evaluation_babI_qa16/2_EvaluateTestOutput.py
@@ -7,7 +7,7 @@
     if x.startswith("API_KEY="):
         openai.api_key = x.split("API_KEY=")[1]
 
-with open("TestOutput_.json") as json_file:
+with open("../TestOutput.json") as json_file:
     ListOfDicts = json.load(json_file)
 
 # {"Line": Line, "Input": Input, "actualOutput": actualOutput, "expectedOutput": expectedOutput}
@@ -36,8 +36,8 @@
     send_prompt = PROMPT.replace("_QUESTION_", Input).replace("_ACTUAL_OUTPUT_",actualOutput).replace("_EXPECTED_OUTPUT_",expectedOutput)
     print(send_prompt)
     while True:
-        try:
-            response = openai.ChatCompletion.create(model='gpt-4', messages=[ {"role": "user", "content": send_prompt}], max_tokens=200, temperature=0)
+        try:                                            #'gpt-4'
+            response = openai.ChatCompletion.create(model='gpt-3.5-turbo', messages=[ {"role": "user", "content": send_prompt}], max_tokens=200, temperature=0)
             ret = response['choices'][0]['message']['content']
         except:
             print("Error: API call failed, will try repeating it in 10 seconds!")

diff --git a/NarsGPT.py b/NarsGPT.py
@@ -31,11 +31,11 @@
 import time
 
 openai.api_key = "YOUR_KEY"
-usedModel = "gpt-4"
+usedModel = "gpt-4" #'gpt-3.5-turbo'
 relevantViewSize = 30      #how many relevant (judged by statement embedding) ONA memory items GPT can see
 recentViewSize = 10        #how many recent (judged by lastUsed) ONA memory items GPT can see
 eternalizationDistance = 3  #how long items are treated as events before contributing to generic belief evidence in long-term memory
-atomCreationThreshold = 0.88 #how different a new word needs to be to existing of same type to become a new atom
+atomCreationThreshold = 0.95 #how different a new word needs to be to existing of same type to become a new atom
 filename = "mem.json" #the system's memory file
 IYouExchange = True or "NoIYouExchange" in sys.argv #whether I and you, my and your is exchanged in communication
 ConsiderGPTKnowledge = False or "ConsiderGPTKnowledge" in sys.argv #Whether it should be allowed to consider GPT's knowledge too for answering a question

diff --git a/OpenNARS-for-Applications b/OpenNARS-for-Applications