improve fa signature

extra/prompt_tuning/config/optimizer/copro.yaml

@@ -1,6 +1,6 @@
 name: COPRO
 params:
-  breadth: 3
-  depth: 10
-  init_temperature: 1.4
+  breadth: 4
+  depth: 15
+  init_temperature: 1.5
 compile:

extra/prompt_tuning/config/prompt/program/coth.yaml

@@ -0,0 +1 @@
+id: CoTH

extra/prompt_tuning/evaluate.py

@@ -61,8 +61,9 @@ async def evaluate(config: DictConfig) -> None:
         run = neptune.init_run()
         run["sys/tags"].add(
             [
-                config.program.type,
-                config.program.name,
+                config.prompt.type.id,
+                config.prompt.signature.id,
+                config.prompt.program.id,
                 *config.data.db_ids,
                 *config.data.difficulties,
             ]

extra/prompt_tuning/tuning/programs/__init__.py

@@ -1,8 +1,15 @@
-from .iql import AggregationAssessorCoT, AggregationAssessorPredict, FilteringAssessorCoT, FilteringAssessorPredict
+from .iql import (
+    AggregationAssessorCoT,
+    AggregationAssessorPredict,
+    FilteringAssessorCoT,
+    FilteringAssessorCoTH,
+    FilteringAssessorPredict,
+)
 
 PROGRAMS = {
     FilteringAssessorPredict.__name__: FilteringAssessorPredict,
     FilteringAssessorCoT.__name__: FilteringAssessorCoT,
+    FilteringAssessorCoTH.__name__: FilteringAssessorCoTH,
     AggregationAssessorPredict.__name__: AggregationAssessorPredict,
     AggregationAssessorCoT.__name__: AggregationAssessorCoT,
 }
@@ -13,4 +20,5 @@
     "AggregationAssessorCoT",
     "FilteringAssessorPredict",
     "FilteringAssessorCoT",
+    "FilteringAssessorCoTH",
 ]

extra/prompt_tuning/tuning/programs/iql.py

@@ -1,6 +1,6 @@
 from typing import Type
 
-from dspy import ChainOfThought, Module, Predict, Prediction
+from dspy import ChainOfThought, ChainOfThoughtWithHint, Module, Predict, Prediction
 
 from ..signatures.iql import AggregationAssessor, FilteringAssessor
 
@@ -51,6 +51,32 @@ def forward(self, question: str) -> Prediction:
         return Prediction(decision=decision.lower() == "true")
 
 
+class FilteringAssessorCoTH(Module):
+    """
+    Program that assesses whether a question requires filtering.
+    """
+
+    def __init__(self, signature: Type[FilteringAssessor]) -> None:
+        super().__init__()
+        self.decide = ChainOfThoughtWithHint(signature)
+
+    def forward(self, question: str) -> Prediction:
+        """
+        Assess whether a question requires filtering.
+
+        Args:
+            question: The question to assess.
+
+        Returns:
+            The prediction.
+        """
+        decision = self.decide(
+            question=question,
+            hint="Look for words indicating data specific features.",
+        ).decision
+        return Prediction(decision=decision.lower() == "true")
+
+
 class AggregationAssessorPredict(Module):
     """
     Program that assesses whether a question requires aggregation.

extra/prompt_tuning/tuning/signatures/iql.py

@@ -32,8 +32,8 @@ class FilteringAssessorBaseline(FilteringAssessor):
 class FilteringAssessorOptimized(FilteringAssessor):
     """
     Given a question, determine whether the answer requires initial data filtering in order to compute it.
-    Initial data filtering is a process in which the result set is reduced to only include the rows that
-    meet certain criteria specified in the question.
+    Initial data filtering is a process in which the result set is filtered based on the specific features
+    stated in the question.
     """
 
 
@@ -61,6 +61,11 @@ class AggregationAssessorBaseline(AggregationAssessor):
 
 class AggregationAssessorOptimized(AggregationAssessor):
     """
-    Given a question, determine whether the answer requires data aggregation in order to compute it.
-    Data aggregation is a process in which we calculate a single values for a group of rows in the result set.
+    Look at the dependencies between the elements in the question and distinguish whether a single value can be obtained
+    for a groupof entities in the data table by aggregating necessary values.
     """
+
+    decision = OutputField(
+        prefix="Instructions to identify aggregated computations given a question, analyze dependencies -> ",
+        desc="indicates whether the answer to the question requires data aggregation. (Respond with True or False)",
+    )