refactor: reorg datastructures

unit-mesh · Jan 2, 2024 · e1ab743 · e1ab743
1 parent 4f62d1e
commit e1ab743
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 1 deletion.
diff --git a/config/custom-connection.yml b/config/custom-connection.yml
@@ -0,0 +1,6 @@
+name: open_ai_connection
+type: AzureOpenAI
+configs:
+  api-host: https://phodal-18lyqn5spad8.gear-c1.openbayes.net/api/chat # if you are using aios.chat
+secrets:
+  api-key: sk-xxx # your api key
diff --git a/config/fixtures/code-completion.jsonl b/config/fixtures/code-completion.jsonl
@@ -0,0 +1,5 @@
+{"language":"java","beforeCursor":"package com.afs.unittest;\n\nimport com.afs.unittest.exception.UnexpectedProjectTypeException;\nimport com.afs.unittest.expense.ExpenseType;\nimport com.afs.unittest.Project.Project;\nimport com.afs.unittest.Project.ProjectType;\n\nimport static com.afs.unittest.expense.ExpenseType.*;\n\nclass ExpenseService {\n    ExpenseType getExpenseCodeByProjectTypeAndName(Project project) {","relatedCode":"// class UnexpectedProjectTypeException {\n// \n//  }\n// \n// class ExpenseType {\n// \n//  }\n// \n// class Project {\n//    : ProjectType\n//    : String\n// \n//    'getter/setter: getProjectName\n// \n//  }\n// \n// class ProjectType {\n// \n//  }\n// ","output":"        ProjectType projectType = project.getProjectType();\n        if(projectType == ProjectType.INTERNAL){\n            return INTERNAL_PROJECT_EXPENSE;\n        }else{\n            if(projectType == ProjectType.EXTERNAL){\n                String projectName = project.getProjectName();\n                if(projectName.equals(\"Project A\")){\n                    return EXPENSE_TYPE_A;\n                }\n\n                if(projectName.equals(\"Project B\")){\n                    return EXPENSE_TYPE_B;\n                }\n\n                return OTHER_EXPENSE;\n            }\n\n            throw new UnexpectedProjectTypeException(\"You enter invalid project type\");\n        }\n    }"}
+{"language":"java","beforeCursor":"package com.afs.unittest;\n\nimport com.afs.unittest.Project.Project;\nimport com.afs.unittest.Project.ProjectType;\nimport com.afs.unittest.exception.UnexpectedProjectTypeException;\nimport com.afs.unittest.expense.ExpenseType;\nimport org.junit.jupiter.api.Test;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\nclass ExpenseServiceTest {\n    @Test\n    void should_return_internal_expense_type_if_project_is_internal() {","relatedCode":"// class Project {\n//    : ProjectType\n//    : String\n// \n//    'getter/setter: getProjectName\n// \n//  }\n// \n// class ProjectType {\n// \n//  }\n// \n// class UnexpectedProjectTypeException {\n// \n//  }\n// \n// class ExpenseType {\n// \n//  }\n// ","output":"        // given\n        Project project = new Project(ProjectType.INTERNAL, \"Project\");\n        ExpenseService expenseService = new ExpenseService();\n\n        // when\n        ExpenseType expenseCode = expenseService.getExpenseCodeByProjectTypeAndName(project);\n\n        // then\n        assertEquals(ExpenseType.INTERNAL_PROJECT_EXPENSE, expenseCode);\n    }"}
+{"language":"java","beforeCursor":"package com.afs.unittest;\n\nimport com.afs.unittest.Project.Project;\nimport com.afs.unittest.Project.ProjectType;\nimport com.afs.unittest.exception.UnexpectedProjectTypeException;\nimport com.afs.unittest.expense.ExpenseType;\nimport org.junit.jupiter.api.Test;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\nclass ExpenseServiceTest {\n    @Test\n    void should_return_internal_expense_type_if_project_is_internal() {\n        // given\n        Project project = new Project(ProjectType.INTERNAL, \"Project\");\n        ExpenseService expenseService = new ExpenseService();\n\n        // when\n        ExpenseType expenseCode = expenseService.getExpenseCodeByProjectTypeAndName(project);\n\n        // then\n        assertEquals(ExpenseType.INTERNAL_PROJECT_EXPENSE, expenseCode);\n    }\n\n    @Test\n    void should_return_expense_type_A_if_project_is_external_and_name_is_project_A() {","relatedCode":"// class Project {\n//    : ProjectType\n//    : String\n// \n//    'getter/setter: getProjectName\n// \n//  }\n// \n// class ProjectType {\n// \n//  }\n// \n// class UnexpectedProjectTypeException {\n// \n//  }\n// \n// class ExpenseType {\n// \n//  }\n// ","output":"        // given\n        Project project = new Project(ProjectType.EXTERNAL, \"Project A\");\n        ExpenseService expenseService = new ExpenseService();\n\n        // when\n        ExpenseType expenseCode = expenseService.getExpenseCodeByProjectTypeAndName(project);\n\n        // then\n        assertEquals(ExpenseType.EXPENSE_TYPE_A, expenseCode);\n    }"}
+{"language":"java","beforeCursor":"package com.afs.unittest;\n\nimport com.afs.unittest.Project.Project;\nimport com.afs.unittest.Project.ProjectType;\nimport com.afs.unittest.exception.UnexpectedProjectTypeException;\nimport com.afs.unittest.expense.ExpenseType;\nimport org.junit.jupiter.api.Test;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\nclass ExpenseServiceTest {\n    @Test\n    void should_return_internal_expense_type_if_project_is_internal() {\n        // given\n        Project project = new Project(ProjectType.INTERNAL, \"Project\");\n        ExpenseService expenseService = new ExpenseService();\n\n        // when\n        ExpenseType expenseCode = expenseService.getExpenseCodeByProjectTypeAndName(project);\n\n        // then\n        assertEquals(ExpenseType.INTERNAL_PROJECT_EXPENSE, expenseCode);\n    }\n\n    @Test\n    void should_return_expense_type_A_if_project_is_external_and_name_is_project_A() {\n        // given\n        Project project = new Project(ProjectType.EXTERNAL, \"Project A\");\n        ExpenseService expenseService = new ExpenseService();\n\n        // when\n        ExpenseType expenseCode = expenseService.getExpenseCodeByProjectTypeAndName(project);\n\n        // then\n        assertEquals(ExpenseType.EXPENSE_TYPE_A, expenseCode);\n    }\n\n    @Test\n    void should_return_expense_type_B_if_project_is_external_and_name_is_project_B() {","relatedCode":"// class Project {\n//    : ProjectType\n//    : String\n// \n//    'getter/setter: getProjectName\n// \n//  }\n// \n// class ProjectType {\n// \n//  }\n// \n// class UnexpectedProjectTypeException {\n// \n//  }\n// \n// class ExpenseType {\n// \n//  }\n// ","output":"        // given\n        Project project = new Project(ProjectType.EXTERNAL, \"Project B\");\n        ExpenseService expenseService = new ExpenseService();\n\n        // when\n        ExpenseType expenseCode = expenseService.getExpenseCodeByProjectTypeAndName(project);\n\n        // then\n\n        assertEquals(ExpenseType.EXPENSE_TYPE_B, expenseCode);\n\n    }"}
+{"language":"java","beforeCursor":"package com.afs.unittest;\n\nimport com.afs.unittest.Project.Project;\nimport com.afs.unittest.Project.ProjectType;\nimport com.afs.unittest.exception.UnexpectedProjectTypeException;\nimport com.afs.unittest.expense.ExpenseType;\nimport org.junit.jupiter.api.Test;\n\nimport static org.junit.jupiter.api.Assertions.*;\n\nclass ExpenseServiceTest {\n    @Test\n    void should_return_internal_expense_type_if_project_is_internal() {\n        // given\n        Project project = new Project(ProjectType.INTERNAL, \"Project\");\n        ExpenseService expenseService = new ExpenseService();\n\n        // when\n        ExpenseType expenseCode = expenseService.getExpenseCodeByProjectTypeAndName(project);\n\n        // then\n        assertEquals(ExpenseType.INTERNAL_PROJECT_EXPENSE, expenseCode);\n    }\n\n    @Test\n    void should_return_expense_type_A_if_project_is_external_and_name_is_project_A() {\n        // given\n        Project project = new Project(ProjectType.EXTERNAL, \"Project A\");\n        ExpenseService expenseService = new ExpenseService();\n\n        // when\n        ExpenseType expenseCode = expenseService.getExpenseCodeByProjectTypeAndName(project);\n\n        // then\n        assertEquals(ExpenseType.EXPENSE_TYPE_A, expenseCode);\n    }\n\n    @Test\n    void should_return_expense_type_B_if_project_is_external_and_name_is_project_B() {\n        // given\n        Project project = new Project(ProjectType.EXTERNAL, \"Project B\");\n        ExpenseService expenseService = new ExpenseService();\n\n        // when\n        ExpenseType expenseCode = expenseService.getExpenseCodeByProjectTypeAndName(project);\n\n        // then\n\n        assertEquals(ExpenseType.EXPENSE_TYPE_B, expenseCode);\n\n    }\n\n    @Test\n    void should_return_other_expense_type_if_project_is_external_and_has_other_name() {","relatedCode":"// class Project {\n//    : ProjectType\n//    : String\n// \n//    'getter/setter: getProjectName\n// \n//  }\n// \n// class ProjectType {\n// \n//  }\n// \n// class UnexpectedProjectTypeException {\n// \n//  }\n// \n// class ExpenseType {\n// \n//  }\n// ","output":"        // given\n        Project project = new Project(ProjectType.EXTERNAL, \"Project C\");\n        ExpenseService expenseService = new ExpenseService();\n\n        // when\n        ExpenseType expenseCode = expenseService.getExpenseCodeByProjectTypeAndName(project);\n\n        // then\n        assertEquals(ExpenseType.OTHER_EXPENSE, expenseCode);\n    }"}
diff --git a/config/fixtures/code-completion.vm b/config/fixtures/code-completion.vm
@@ -0,0 +1,10 @@
+Complete ${language} code, return rest code, no explaining
+
+```${language}
+${relatedCode}
+```
+
+Code:
+```${language}
+${beforeCursor}
+```
diff --git a/config/unit-eval.yml b/config/unit-eval.yml
@@ -0,0 +1,18 @@
+name: "Eval Units"
+description: "A sample of UnitGen"
+
+jobs:
+  prompt-evaluate:
+    description: "Evaluate prompt with different parameters"
+    connection: custom-connection.yml # 根据你的实际情况修改
+    template: fixtures/code-completion.vm
+    log-path: logs/
+    template-datasource:
+      - type: file
+        value: fixtures/code-completion.jsonl
+
+    strategy:
+      - type: datasource-collection
+        value:
+          - temperature: 0.3
+            max_tokens: 1000
diff --git a/src/main/kotlin/cc/unitmesh/eval/Eval.kt b/src/main/kotlin/cc/unitmesh/eval/Eval.kt
@@ -8,7 +8,7 @@ import org.slf4j.Logger
 import java.io.File
 
 class EvalCommand : CliktCommand() {
-    private val input by option(help = "prompt script config file").default("unit-eval.yml")
+    private val input by option(help = "prompt script config file").default("config" + File.separator + "unit-eval.yml")
 
     override fun run() {
         // check is yaml file