diff --git a/benchmarks/sql/bench.py b/benchmarks/sql/bench.py index a86c6ee4..0257b52d 100644 --- a/benchmarks/sql/bench.py +++ b/benchmarks/sql/bench.py @@ -8,10 +8,13 @@ from bench.evaluator import Evaluator from bench.loaders import CollectionDataLoader, IQLViewDataLoader, SQLViewDataLoader from bench.metrics import ( + AggregationAccuracy, ExecutionAccuracy, FilteringAccuracy, FilteringPrecision, FilteringRecall, + IQLAggregationCorrectness, + IQLAggregationParseability, IQLFiltersAccuracy, IQLFiltersCorrectness, IQLFiltersParseability, @@ -57,9 +60,12 @@ class EvaluationType(Enum): EVALUATION_METRICS = { EvaluationType.IQL.value: MetricSet( + AggregationAccuracy, FilteringAccuracy, FilteringPrecision, FilteringRecall, + IQLAggregationParseability, + IQLAggregationCorrectness, IQLFiltersAccuracy, IQLFiltersPrecision, IQLFiltersRecall, @@ -72,9 +78,12 @@ class EvaluationType(Enum): ExecutionAccuracy, ), EvaluationType.E2E.value: MetricSet( + AggregationAccuracy, FilteringAccuracy, FilteringPrecision, FilteringRecall, + IQLAggregationParseability, + IQLAggregationCorrectness, IQLFiltersAccuracy, IQLFiltersPrecision, IQLFiltersRecall, diff --git a/benchmarks/sql/bench/metrics/__init__.py b/benchmarks/sql/bench/metrics/__init__.py index 3684635f..ccd32b1c 100644 --- a/benchmarks/sql/bench/metrics/__init__.py +++ b/benchmarks/sql/bench/metrics/__init__.py @@ -1,5 +1,6 @@ from .base import Metric, MetricSet from .iql import ( + AggregationAccuracy, FilteringAccuracy, FilteringPrecision, FilteringRecall, @@ -17,6 +18,7 @@ __all__ = [ "Metric", "MetricSet", + "AggregationAccuracy", "FilteringAccuracy", "FilteringPrecision", "FilteringRecall", diff --git a/benchmarks/sql/bench/metrics/iql.py b/benchmarks/sql/bench/metrics/iql.py index 118c73d7..5cee9c1a 100644 --- a/benchmarks/sql/bench/metrics/iql.py +++ b/benchmarks/sql/bench/metrics/iql.py @@ -5,20 +5,23 @@ from .base import Metric -class FilteringAccuracy(Metric): +class AssessingAccuracy(Metric, ABC): """ - Filtering accuracy is proportion of correct decisions (to filter or not) out of all decisions made. + Assessing accuracy is proportion of correct decisions out of all decisions made. """ + prefix: str + iql: str + def compute(self, results: List[EvaluationResult]) -> Dict[str, Any]: """ - Computes the filtering accuracy. + Computes the assessing accuracy. Args: results: List of evaluation results. Returns: - Filtering accuracy. + Assessing accuracy. """ results = [ result @@ -27,14 +30,20 @@ def compute(self, results: List[EvaluationResult]) -> Dict[str, Any]: and result.prediction.iql and result.reference.view_name and result.prediction.view_name - and result.reference.iql.filters.generated - and result.prediction.iql.filters.generated + and getattr(result.reference.iql, self.iql).generated + and getattr(result.prediction.iql, self.iql).generated ] return { - "DM/FLT/ACC": ( + f"DM/{self.prefix}/ACC": ( sum( - (result.reference.iql.filters.source is not None or result.reference.iql.filters.unsupported) - == (result.prediction.iql.filters.source is not None or result.prediction.iql.filters.unsupported) + ( + getattr(result.reference.iql, self.iql).source is not None + or getattr(result.reference.iql, self.iql).unsupported + ) + == ( + getattr(result.prediction.iql, self.iql).source is not None + or getattr(result.prediction.iql, self.iql).unsupported + ) for result in results ) / len(results) @@ -44,6 +53,24 @@ def compute(self, results: List[EvaluationResult]) -> Dict[str, Any]: } +class FilteringAccuracy(AssessingAccuracy): + """ + Filtering accuracy is proportion of correct decisions (to filter or not) out of all decisions made. + """ + + prefix: str = "FLT" + iql: str = "filters" + + +class AggregationAccuracy(AssessingAccuracy): + """ + Aggregation accuracy is proportion of correct decisions (to aggregate or not) out of all decisions made. + """ + + prefix: str = "AGG" + iql: str = "aggregation" + + class FilteringPrecision(Metric): """ Filtering precision is proportion of correct decisions to filter out of all decisions to filter. diff --git a/benchmarks/sql/bench/views/structured/superhero.py b/benchmarks/sql/bench/views/structured/superhero.py index 75a86ca7..73d4f3da 100644 --- a/benchmarks/sql/bench/views/structured/superhero.py +++ b/benchmarks/sql/bench/views/structured/superhero.py @@ -76,12 +76,6 @@ class SuperheroFilterMixin: def filter_by_superhero_id(self, superhero_id: int) -> ColumnElement: """ Filters the view by the superhero id. - - Args: - superhero_id: The id of the superhero. - - Returns: - The filter condition. """ return Superhero.id == superhero_id @@ -89,12 +83,6 @@ def filter_by_superhero_id(self, superhero_id: int) -> ColumnElement: def filter_by_superhero_name(self, superhero_name: str) -> ColumnElement: """ Filters the view by the superhero nick or handle. - - Args: - superhero_name: The abstract nick or handle of the superhero. - - Returns: - The filter condition. """ return Superhero.superhero_name == superhero_name @@ -102,9 +90,6 @@ def filter_by_superhero_name(self, superhero_name: str) -> ColumnElement: def filter_by_missing_superhero_full_name(self) -> ColumnElement: """ Filters the view by the missing full name of the superhero. - - Returns: - The filter condition. """ return Superhero.full_name == None @@ -112,12 +97,6 @@ def filter_by_missing_superhero_full_name(self) -> ColumnElement: def filter_by_superhero_full_name(self, superhero_full_name: str) -> ColumnElement: """ Filters the view by the full name of the superhero. - - Args: - superhero_full_name: The human name of the superhero. - - Returns: - The filter condition. """ return Superhero.full_name == superhero_full_name @@ -125,12 +104,6 @@ def filter_by_superhero_full_name(self, superhero_full_name: str) -> ColumnEleme def filter_by_superhero_first_name(self, superhero_first_name: str) -> ColumnElement: """ Filters the view by the simmilar full name of the superhero. - - Args: - superhero_first_name: The first name of the superhero. - - Returns: - The filter condition. """ return Superhero.full_name.like(f"{superhero_first_name}%") @@ -138,12 +111,6 @@ def filter_by_superhero_first_name(self, superhero_first_name: str) -> ColumnEle def filter_by_height_cm(self, height_cm: float) -> ColumnElement: """ Filters the view by the height of the superhero. - - Args: - height_cm: The height of the superhero. - - Returns: - The filter condition. """ return Superhero.height_cm == height_cm @@ -151,12 +118,6 @@ def filter_by_height_cm(self, height_cm: float) -> ColumnElement: def filter_by_height_cm_less_than(self, height_cm: float) -> ColumnElement: """ Filters the view by the height of the superhero. - - Args: - height_cm: The height of the superhero. - - Returns: - The filter condition. """ return Superhero.height_cm < height_cm @@ -164,12 +125,6 @@ def filter_by_height_cm_less_than(self, height_cm: float) -> ColumnElement: def filter_by_height_cm_greater_than(self, height_cm: float) -> ColumnElement: """ Filters the view by the height of the superhero. - - Args: - height_cm: The height of the superhero. - - Returns: - The filter condition. """ return Superhero.height_cm > height_cm @@ -177,13 +132,6 @@ def filter_by_height_cm_greater_than(self, height_cm: float) -> ColumnElement: def filter_by_height_cm_between(self, begin_height_cm: float, end_height_cm: float) -> ColumnElement: """ Filters the view by the height of the superhero. - - Args: - begin_height_cm: The begin height of the superhero. - end_height_cm: The end height of the superhero. - - Returns: - The filter condition. """ return Superhero.height_cm.between(begin_height_cm, end_height_cm) @@ -191,9 +139,6 @@ def filter_by_height_cm_between(self, begin_height_cm: float, end_height_cm: flo def filter_by_the_tallest(self) -> ColumnElement: """ Filter the view by the tallest superhero. - - Returns: - The filter condition. """ return Superhero.height_cm == select(func.max(Superhero.height_cm)).scalar_subquery() @@ -201,9 +146,6 @@ def filter_by_the_tallest(self) -> ColumnElement: def filter_by_missing_weight(self) -> ColumnElement: """ Filters the view by the missing weight of the superhero. - - Returns: - The filter condition. """ return Superhero.weight_kg == 0 or Superhero.weight_kg == None @@ -224,12 +166,6 @@ def filter_by_weight_kg(self, weight_kg: int) -> ColumnElement: def filter_by_weight_kg_greater_than(self, weight_kg: int) -> ColumnElement: """ Filters the view by the weight of the superhero. - - Args: - weight_kg: The weight of the superhero. - - Returns: - The filter condition. """ return Superhero.weight_kg > weight_kg @@ -237,12 +173,6 @@ def filter_by_weight_kg_greater_than(self, weight_kg: int) -> ColumnElement: def filter_by_weight_kg_less_than(self, weight_kg: int) -> ColumnElement: """ Filters the view by the weight of the superhero. - - Args: - weight_kg: The weight of the superhero. - - Returns: - The filter condition. """ return Superhero.weight_kg < weight_kg @@ -250,12 +180,6 @@ def filter_by_weight_kg_less_than(self, weight_kg: int) -> ColumnElement: def filter_by_weight_greater_than_percentage_of_average(self, average_percentage: int) -> ColumnElement: """ Filters the view by the weight greater than the percentage of average of superheroes. - - Args: - average_percentage: The percentage of the average weight. - - Returns: - The filter condition. """ return Superhero.weight_kg * 100 > select(func.avg(Superhero.weight_kg)).scalar_subquery() * average_percentage @@ -263,9 +187,6 @@ def filter_by_weight_greater_than_percentage_of_average(self, average_percentage def filter_by_the_heaviest(self) -> ColumnElement: """ Filters the view by the heaviest superhero. - - Returns: - The filter condition. """ return Superhero.weight_kg == select(func.max(Superhero.weight_kg)).scalar_subquery() @@ -273,9 +194,6 @@ def filter_by_the_heaviest(self) -> ColumnElement: def filter_by_missing_publisher(self) -> ColumnElement: """ Filters the view by the missing publisher of the superhero. - - Returns: - The filter condition. """ return Superhero.publisher_id == None @@ -289,9 +207,6 @@ class SuperheroAggregationMixin: def count_superheroes(self) -> Select: """ Counts the number of superheros. - - Returns: - The superheros count. """ return self.select.with_only_columns(func.count(Superhero.id).label("count_superheroes")).group_by(Superhero.id) @@ -299,9 +214,6 @@ def count_superheroes(self) -> Select: def average_height(self) -> Select: """ Averages the height of the superheros. - - Returns: - The superheros average height. """ return self.select.with_only_columns(func.avg(Superhero.height_cm).label("average_height")).group_by( Superhero.id @@ -317,12 +229,6 @@ class SuperheroColourFilterMixin: def filter_by_eye_colour(self, eye_colour: str) -> ColumnElement: """ Filters the view by the superhero eye colour. - - Args: - eye_colour: The eye colour of the superhero. - - Returns: - The filter condition. """ return self.eye_colour.colour == eye_colour @@ -330,12 +236,6 @@ def filter_by_eye_colour(self, eye_colour: str) -> ColumnElement: def filter_by_hair_colour(self, hair_colour: str) -> ColumnElement: """ Filters the view by the superhero hair colour. - - Args: - hair_colour: The hair colour of the superhero. - - Returns: - The filter condition. """ return self.hair_colour.colour == hair_colour @@ -343,12 +243,6 @@ def filter_by_hair_colour(self, hair_colour: str) -> ColumnElement: def filter_by_skin_colour(self, skin_colour: str) -> ColumnElement: """ Filters the view by the superhero skin colour. - - Args: - skin_colour: The skin colour of the superhero. - - Returns: - The filter condition. """ return self.skin_colour.colour == skin_colour @@ -356,9 +250,6 @@ def filter_by_skin_colour(self, skin_colour: str) -> ColumnElement: def filter_by_same_hair_and_eye_colour(self) -> ColumnElement: """ Filters the view by the superhero with the same hair and eye colour. - - Returns: - The filter condition. """ return self.eye_colour.colour == self.hair_colour.colour @@ -366,9 +257,6 @@ def filter_by_same_hair_and_eye_colour(self) -> ColumnElement: def filter_by_same_hair_and_skin_colour(self) -> ColumnElement: """ Filters the view by the superhero with the same hair and skin colour. - - Returns: - The filter condition. """ return self.hair_colour.colour == self.skin_colour.colour @@ -382,12 +270,6 @@ class SuperheroColourAggregationMixin: def percentage_of_eye_colour(self, eye_colour: str) -> Select: """ Calculates the percentage of objects with eye colour. - - Args: - eye_colour: The eye colour of the object. - - Returns: - The percentage of objects with eye colour. """ return self.select.with_only_columns( ( @@ -407,12 +289,6 @@ class PublisherFilterMixin: def filter_by_publisher_name(self, publisher_name: str) -> ColumnElement: """ Filters the view by the publisher name. - - Args: - publisher_name: The name of the publisher. - - Returns: - The filter condition. """ return Publisher.publisher_name == publisher_name @@ -426,12 +302,6 @@ class PublisherAggregationMixin: def percentage_of_publisher(self, publisher_name: str) -> Select: """ Calculates the percentage of objects with publisher. - - Args: - publisher_name: The name of the publisher. - - Returns: - The percentage of objects with publisher. """ return self.select.with_only_columns( ( @@ -451,12 +321,6 @@ class AlignmentFilterMixin: def filter_by_alignment(self, alignment: Literal["Good", "Bad", "Neutral", "N/A"]) -> ColumnElement: """ Filters the view by the superhero alignment. - - Args: - alignment: The alignment of the superhero. - - Returns: - The filter condition. """ return Alignment.alignment == alignment @@ -470,12 +334,6 @@ class AlignmentAggregationMixin: def percentage_of_alignment(self, alignment: Literal["Good", "Bad", "Neutral", "N/A"]) -> Select: """ Calculates the percentage of objects with alignment. - - Args: - alignment: The alignment of the object. - - Returns: - The percentage of objects with alignment. """ return self.select.with_only_columns( ( @@ -495,12 +353,6 @@ class GenderFilterMixin: def filter_by_gender(self, gender: Literal["Male", "Female", "N/A"]) -> ColumnElement: """ Filters the view by the object gender. - - Args: - gender: The gender of the object. - - Returns: - The filter condition. """ return Gender.gender == gender @@ -514,12 +366,6 @@ class GenderAggregationMixin: def percentage_of_gender(self, gender: Literal["Male", "Female", "N/A"]) -> Select: """ Calculates the percentage of objects with gender. - - Args: - gender: The gender of the object. - - Returns: - The percentage of objects with gender. """ return self.select.with_only_columns( ( @@ -539,12 +385,6 @@ class RaceFilterMixin: def filter_by_race(self, race: str) -> ColumnElement: """ Filters the view by the object race. - - Args: - race: The race of the object. - - Returns: - The filter condition. """ return Race.race == race