DSI 0.4.0 and Saved Query Exports (#8950)

.changes/unreleased/Dependencies-20231031-131954.yaml

@@ -0,0 +1,6 @@
+kind: Dependencies
+body: Begin using DSI 0.4.x
+time: 2023-10-31T13:19:54.750009-07:00
+custom:
+  Author: QMalcolm peterallenwebb
+  PR: "8892"

.changes/unreleased/Features-20231031-132022.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add exports to SavedQuery spec
+time: 2023-10-31T13:20:22.448158-07:00
+custom:
+  Author: QMalcolm peterallenwebb
+  Issue: "8892"

core/dbt/contracts/graph/nodes.py

@@ -12,6 +12,7 @@
 
 from dbt.clients.system import write_file
 from dbt.contracts.files import FileHash
+from dbt.contracts.graph.saved_queries import Export, QueryParams
 from dbt.contracts.graph.semantic_models import (
     Defaults,
     Dimension,
@@ -36,6 +37,7 @@
     UnparsedColumn,
 )
 from dbt.contracts.graph.node_args import ModelNodeArgs
+from dbt.contracts.graph.semantic_layer_common import WhereFilterIntersection
 from dbt.contracts.util import Replaceable, AdditionalPropertiesMixin
 from dbt.events.functions import warn_or_error
 from dbt.exceptions import ParsingError, ContractBreakingChangeError
@@ -49,7 +51,6 @@
 from dbt.events.contextvars import set_log_contextvars
 from dbt.flags import get_flags
 from dbt.node_types import ModelLanguage, NodeType, AccessType
-from dbt_semantic_interfaces.call_parameter_sets import FilterCallParameterSets
 from dbt_semantic_interfaces.references import (
     EntityReference,
     MeasureReference,
@@ -59,7 +60,6 @@
 )
 from dbt_semantic_interfaces.references import MetricReference as DSIMetricReference
 from dbt_semantic_interfaces.type_enums import MetricType, TimeGranularity
-from dbt_semantic_interfaces.parsing.where_filter.where_filter_parser import WhereFilterParser
 
 from .model_config import (
     NodeConfig,
@@ -1399,24 +1399,6 @@ def group(self):
 # ====================================
 
 
-@dataclass
-class WhereFilter(dbtClassMixin):
-    where_sql_template: str
-
-    @property
-    def call_parameter_sets(self) -> FilterCallParameterSets:
-        return WhereFilterParser.parse_call_parameter_sets(self.where_sql_template)
-
-
-@dataclass
-class WhereFilterIntersection(dbtClassMixin):
-    where_filters: List[WhereFilter]
-
-    @property
-    def filter_expression_parameter_sets(self) -> Sequence[Tuple[str, FilterCallParameterSets]]:
-        raise NotImplementedError
-
-
 @dataclass
 class MetricInputMeasure(dbtClassMixin):
     name: str
@@ -1745,9 +1727,8 @@ def same_contents(self, old: Optional["SemanticModel"]) -> bool:
 
 @dataclass
 class SavedQueryMandatory(GraphNode):
-    metrics: List[str]
-    group_bys: List[str]
-    where: Optional[WhereFilterIntersection]
+    query_params: QueryParams
+    exports: List[Export]
 
 
 @dataclass
@@ -1762,21 +1743,25 @@ class SavedQuery(NodeInfoMixin, SavedQueryMandatory):
     created_at: float = field(default_factory=lambda: time.time())
     refs: List[RefArgs] = field(default_factory=list)
 
+    @property
+    def metrics(self) -> List[str]:
+        return self.query_params.metrics
+
     @property
     def depends_on_nodes(self):
         return self.depends_on.nodes
 
     def same_metrics(self, old: "SavedQuery") -> bool:
-        return self.metrics == old.metrics
+        return self.query_params.metrics == old.query_params.metrics
 
-    def same_group_bys(self, old: "SavedQuery") -> bool:
-        return self.group_bys == old.group_bys
+    def same_group_by(self, old: "SavedQuery") -> bool:
+        return self.query_params.group_by == old.query_params.group_by
 
     def same_description(self, old: "SavedQuery") -> bool:
         return self.description == old.description
 
     def same_where(self, old: "SavedQuery") -> bool:
-        return self.where == old.where
+        return self.query_params.where == old.query_params.where
 
     def same_label(self, old: "SavedQuery") -> bool:
         return self.label == old.label
@@ -1787,6 +1772,17 @@ def same_config(self, old: "SavedQuery") -> bool:
     def same_group(self, old: "SavedQuery") -> bool:
         return self.group == old.group
 
+    def same_exports(self, old: "SavedQuery") -> bool:
+        if len(self.exports) != len(old.exports):
+            return False
+
+        # exports should be in the same order, so we zip them for easy iteration
+        for (old_export, new_export) in zip(old.exports, self.exports):
+            if not new_export.same_contents(old_export):
+                return False
+
+        return True
+
     def same_contents(self, old: Optional["SavedQuery"]) -> bool:
         # existing when it didn't before is a change!
         # metadata/tags changes are not "changes"
@@ -1795,7 +1791,7 @@ def same_contents(self, old: Optional["SavedQuery"]) -> bool:
 
         return (
             self.same_metrics(old)
-            and self.same_group_bys(old)
+            and self.same_group_by(old)
             and self.same_description(old)
             and self.same_where(old)
             and self.same_label(old)

core/dbt/contracts/graph/saved_queries.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dbt.contracts.graph.semantic_layer_common import WhereFilterIntersection
+from dbt.dataclass_schema import dbtClassMixin
+from dbt_semantic_interfaces.type_enums.export_destination_type import ExportDestinationType
+from typing import List, Optional
+
+
+@dataclass
+class ExportConfig(dbtClassMixin):
+    """Nested configuration attributes for exports."""
+
+    export_as: ExportDestinationType
+    schema_name: Optional[str] = None
+    alias: Optional[str] = None
+
+
+@dataclass
+class Export(dbtClassMixin):
+    """Configuration for writing query results to a table."""
+
+    name: str
+    config: ExportConfig
+
+    def same_name(self, old: Export) -> bool:
+        return self.name == old.name
+
+    def same_export_as(self, old: Export) -> bool:
+        return self.config.export_as == old.config.export_as
+
+    def same_schema_name(self, old: Export) -> bool:
+        return self.config.schema_name == old.config.schema_name
+
+    def same_alias(self, old: Export) -> bool:
+        return self.config.alias == old.config.alias
+
+    def same_contents(self, old: Export) -> bool:
+        return (
+            self.same_name(old)
+            and self.same_export_as(old)
+            and self.same_schema_name(old)
+            and self.same_alias(old)
+        )
+
+
+@dataclass
+class QueryParams(dbtClassMixin):
+    """The query parameters for the saved query"""
+
+    metrics: List[str]
+    group_by: List[str]
+    where: Optional[WhereFilterIntersection]

core/dbt/contracts/graph/semantic_layer_common.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from dbt.dataclass_schema import dbtClassMixin
+from dbt_semantic_interfaces.call_parameter_sets import FilterCallParameterSets
+from dbt_semantic_interfaces.parsing.where_filter.where_filter_parser import WhereFilterParser
+from typing import List, Sequence, Tuple
+
+
+@dataclass
+class WhereFilter(dbtClassMixin):
+    where_sql_template: str
+
+    @property
+    def call_parameter_sets(self) -> FilterCallParameterSets:
+        return WhereFilterParser.parse_call_parameter_sets(self.where_sql_template)
+
+
+@dataclass
+class WhereFilterIntersection(dbtClassMixin):
+    where_filters: List[WhereFilter]
+
+    @property
+    def filter_expression_parameter_sets(self) -> Sequence[Tuple[str, FilterCallParameterSets]]:
+        raise NotImplementedError

core/dbt/contracts/graph/unparsed.py

@@ -19,6 +19,7 @@
 from dbt.exceptions import CompilationError, ParsingError, DbtInternalError
 
 from dbt.dataclass_schema import dbtClassMixin, StrEnum, ExtensibleDbtClassMixin, ValidationError
+from dbt_semantic_interfaces.type_enums.export_destination_type import ExportDestinationType
 
 from dataclasses import dataclass, field
 from datetime import timedelta
@@ -721,14 +722,37 @@ class UnparsedSemanticModel(dbtClassMixin):
     primary_entity: Optional[str] = None
 
 
+@dataclass
+class UnparsedQueryParams(dbtClassMixin):
+    metrics: List[str] = field(default_factory=list)
+    group_by: List[str] = field(default_factory=list)
+    where: Optional[Union[str, List[str]]] = None
+
+
+@dataclass
+class UnparsedExportConfig(dbtClassMixin):
+    """Nested configuration attributes for exports."""
+
+    export_as: ExportDestinationType
+    schema: Optional[str] = None
+    alias: Optional[str] = None
+
+
+@dataclass
+class UnparsedExport(dbtClassMixin):
+    """Configuration for writing query results to a table."""
+
+    name: str
+    config: UnparsedExportConfig
+
+
 @dataclass
 class UnparsedSavedQuery(dbtClassMixin):
     name: str
+    query_params: UnparsedQueryParams
     description: Optional[str] = None
     label: Optional[str] = None
-    metrics: List[str] = field(default_factory=list)
-    group_bys: List[str] = field(default_factory=list)
-    where: Optional[Union[str, List[str]]] = None
+    exports: List[UnparsedExport] = field(default_factory=list)
     config: Dict[str, Any] = field(default_factory=dict)
 
 

core/dbt/parser/manifest.py

@@ -1113,7 +1113,7 @@ def process_saved_queries(self, config: RuntimeConfig):
         for saved_query in self.manifest.saved_queries.values():
             # TODO:
             # 1. process `where` of SavedQuery for `depends_on`s
-            # 2. process `group_bys` of SavedQuery for `depends_on``
+            # 2. process `group_by` of SavedQuery for `depends_on``
             _process_metrics_for_node(self.manifest, current_project, saved_query)
 
     def update_semantic_model(self, semantic_model) -> None:

core/dbt/parser/schema_renderer.py

@@ -79,7 +79,7 @@ def should_render_keypath(self, keypath: Keypath) -> bool:
             elif self._is_norender_key(keypath[0:]):
                 return False
         elif self.key == "saved_queries":
-            if keypath[0] == "where":
+            if keypath[0] == "query_params" and len(keypath) > 1 and keypath[1] == "where":
                 return False
             elif self._is_norender_key(keypath[0:]):
                 return False

core/dbt/parser/schema_yaml_readers.py

@@ -5,6 +5,8 @@
     UnparsedDimension,
     UnparsedDimensionTypeParams,
     UnparsedEntity,
+    UnparsedExport,
+    UnparsedExportConfig,
     UnparsedExposure,
     UnparsedGroup,
     UnparsedMeasure,
@@ -13,6 +15,7 @@
     UnparsedMetricInputMeasure,
     UnparsedMetricTypeParams,
     UnparsedNonAdditiveDimension,
+    UnparsedQueryParams,
     UnparsedSavedQuery,
     UnparsedSemanticModel,
 )
@@ -26,9 +29,9 @@
     MetricTypeParams,
     SemanticModel,
     SavedQuery,
-    WhereFilter,
-    WhereFilterIntersection,
 )
+from dbt.contracts.graph.saved_queries import Export, ExportConfig, QueryParams
+from dbt.contracts.graph.semantic_layer_common import WhereFilter, WhereFilterIntersection
 from dbt.contracts.graph.semantic_models import (
     Dimension,
     DimensionTypeParams,
@@ -666,6 +669,23 @@ def _generate_saved_query_config(
 
         return config
 
+    def _get_export_config(self, unparsed: UnparsedExportConfig) -> ExportConfig:
+        return ExportConfig(
+            export_as=unparsed.export_as,
+            schema_name=unparsed.schema,
+            alias=unparsed.alias,
+        )
+
+    def _get_export(self, unparsed: UnparsedExport) -> Export:
+        return Export(name=unparsed.name, config=self._get_export_config(unparsed.config))
+
+    def _get_query_params(self, unparsed: UnparsedQueryParams) -> QueryParams:
+        return QueryParams(
+            group_by=unparsed.group_by,
+            metrics=unparsed.metrics,
+            where=parse_where_filter(unparsed.where),
+        )
+
     def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None:
         package_name = self.project.project_name
         unique_id = f"{NodeType.SavedQuery}.{package_name}.{unparsed.name}"
@@ -694,15 +714,14 @@ def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None:
             description=unparsed.description,
             label=unparsed.label,
             fqn=fqn,
-            group_bys=unparsed.group_bys,
-            metrics=unparsed.metrics,
             name=unparsed.name,
             original_file_path=self.yaml.path.original_file_path,
             package_name=package_name,
             path=path,
             resource_type=NodeType.SavedQuery,
             unique_id=unique_id,
-            where=parse_where_filter(unparsed.where),
+            query_params=self._get_query_params(unparsed.query_params),
+            exports=[self._get_export(export) for export in unparsed.exports],
             config=config,
             unrendered_config=unrendered_config,
             group=config.group,

core/setup.py

@@ -74,7 +74,7 @@
         "dbt-extractor~=0.5.0",
         "minimal-snowplow-tracker~=0.0.2",
         # DSI is under active development, so we're pinning to specific dev versions for now.
-        "dbt-semantic-interfaces~=0.3.0",
+        "dbt-semantic-interfaces~=0.4.0",
         # ----
         # Expect compatibility with all new versions of these packages, so lower bounds only.
         "jsonschema>=3.0",