From f29836fcf35b10ec3a7aa3fc1096139177ed32cf Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 11 Feb 2025 16:01:16 +0000 Subject: [PATCH] Round 2: Add doc_blocks to manifest for nodes and columns (#11294) * Reapply "Add `doc_blocks` to manifest for nodes and columns (#11224)" (#11283) This reverts commit 55e0df181f89241f1d222425f928459f3453ea81. * Expand doc_blocks backcompat test * Refactor to method, add docstring --- .../unreleased/Features-20250122-170328.yaml | 6 + core/dbt/artifacts/resources/v1/components.py | 25 ++ .../resources/v1/source_definition.py | 1 + core/dbt/parser/manifest.py | 65 ++++- schemas/dbt/manifest/v12.json | 234 ++++++++++++++++++ .../functional/artifacts/expected_manifest.py | 50 ++++ .../configs/test_contract_configs.py | 71 +++++- .../docs/test_doc_blocks_backcompat.py | 38 +++ .../functional/docs/test_good_docs_blocks.py | 44 +++- tests/unit/contracts/graph/test_manifest.py | 1 + tests/unit/contracts/graph/test_nodes.py | 2 + .../unit/contracts/graph/test_nodes_parsed.py | 16 ++ 12 files changed, 537 insertions(+), 16 deletions(-) create mode 100644 .changes/unreleased/Features-20250122-170328.yaml create mode 100644 tests/functional/docs/test_doc_blocks_backcompat.py diff --git a/.changes/unreleased/Features-20250122-170328.yaml b/.changes/unreleased/Features-20250122-170328.yaml new file mode 100644 index 00000000000..ce95d99402c --- /dev/null +++ b/.changes/unreleased/Features-20250122-170328.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add doc_blocks to manifest for nodes and columns +time: 2025-01-22T17:03:28.866522Z +custom: + Author: aranke + Issue: 11000 11001 diff --git a/core/dbt/artifacts/resources/v1/components.py b/core/dbt/artifacts/resources/v1/components.py index 8eb43f35d8e..ec2c6cc828c 100644 --- a/core/dbt/artifacts/resources/v1/components.py +++ b/core/dbt/artifacts/resources/v1/components.py @@ -15,6 +15,20 @@ NodeVersion = Union[str, float] +def _backcompat_doc_blocks(doc_blocks: Any) -> List[str]: + """ + Make doc_blocks backwards-compatible for scenarios where a user specifies `doc_blocks` on a model or column. + Mashumaro will raise a serialization error if the specified `doc_blocks` isn't a list of strings. + In such a scenario, this method returns an empty list to avoid a serialization error. + Further along, `_get_doc_blocks` in `manifest.py` populates the correct `doc_blocks` for the happy path. + """ + + if isinstance(doc_blocks, list) and all(isinstance(x, str) for x in doc_blocks): + return doc_blocks + + return [] + + @dataclass class MacroDependsOn(dbtClassMixin): macros: List[str] = field(default_factory=list) @@ -68,6 +82,12 @@ class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin): tags: List[str] = field(default_factory=list) _extra: Dict[str, Any] = field(default_factory=dict) granularity: Optional[TimeGranularity] = None + doc_blocks: List[str] = field(default_factory=list) + + def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None) -> dict: + dct = super().__post_serialize__(dct, context) + dct["doc_blocks"] = _backcompat_doc_blocks(dct["doc_blocks"]) + return dct @dataclass @@ -197,13 +217,18 @@ class ParsedResource(ParsedResourceMandatory): unrendered_config_call_dict: Dict[str, Any] = field(default_factory=dict) relation_name: Optional[str] = None raw_code: str = "" + doc_blocks: List[str] = field(default_factory=list) def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) + if context and context.get("artifact") and "config_call_dict" in dct: del dct["config_call_dict"] if context and context.get("artifact") and "unrendered_config_call_dict" in dct: del dct["unrendered_config_call_dict"] + + dct["doc_blocks"] = _backcompat_doc_blocks(dct["doc_blocks"]) + return dct diff --git a/core/dbt/artifacts/resources/v1/source_definition.py b/core/dbt/artifacts/resources/v1/source_definition.py index e09095fa0af..f827d114942 100644 --- a/core/dbt/artifacts/resources/v1/source_definition.py +++ b/core/dbt/artifacts/resources/v1/source_definition.py @@ -74,3 +74,4 @@ class SourceDefinition(ParsedSourceMandatory): created_at: float = field(default_factory=lambda: time.time()) unrendered_database: Optional[str] = None unrendered_schema: Optional[str] = None + doc_blocks: List[str] = field(default_factory=list) diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 023c5db9300..627ad034c2f 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -10,6 +10,7 @@ from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Tuple, Type, Union import msgpack +from jinja2.nodes import Call import dbt.deprecations import dbt.exceptions @@ -115,6 +116,7 @@ from dbt.parser.sources import SourcePatcher from dbt.parser.unit_tests import process_models_for_unit_test from dbt.version import __version__ +from dbt_common.clients.jinja import parse from dbt_common.clients.system import make_directory, path_exists, read_json, write_file from dbt_common.constants import SECRET_ENV_PREFIX from dbt_common.dataclass_schema import StrEnum, dbtClassMixin @@ -1240,7 +1242,7 @@ def process_docs(self, config: RuntimeConfig): self.manifest, config.project_name, ) - _process_docs_for_node(ctx, node) + _process_docs_for_node(ctx, node, self.manifest) for source in self.manifest.sources.values(): if source.created_at < self.started_at: continue @@ -1250,7 +1252,7 @@ def process_docs(self, config: RuntimeConfig): self.manifest, config.project_name, ) - _process_docs_for_source(ctx, source) + _process_docs_for_source(ctx, source, self.manifest) for macro in self.manifest.macros.values(): if macro.created_at < self.started_at: continue @@ -1657,13 +1659,54 @@ def _check_manifest(manifest: Manifest, config: RuntimeConfig) -> None: DocsContextCallback = Callable[[ResultNode], Dict[str, Any]] +def _get_doc_blocks(description: str, manifest: Manifest, node_package: str) -> List[str]: + ast = parse(description) + doc_blocks: List[str] = [] + + if not hasattr(ast, "body"): + return doc_blocks + + for statement in ast.body: + for node in statement.nodes: + if ( + isinstance(node, Call) + and hasattr(node, "node") + and hasattr(node, "args") + and node.node.name == "doc" + ): + doc_args = [arg.value for arg in node.args] + + if len(doc_args) == 1: + package, name = None, doc_args[0] + elif len(doc_args) == 2: + package, name = doc_args + else: + continue + + if not manifest.metadata.project_name: + continue + + resolved_doc = manifest.resolve_doc( + name, package, manifest.metadata.project_name, node_package + ) + + if resolved_doc: + doc_blocks.append(resolved_doc.unique_id) + + return doc_blocks + + # node and column descriptions def _process_docs_for_node( context: Dict[str, Any], node: ManifestNode, + manifest: Manifest, ): + node.doc_blocks = _get_doc_blocks(node.description, manifest, node.package_name) node.description = get_rendered(node.description, context) + for column_name, column in node.columns.items(): + column.doc_blocks = _get_doc_blocks(column.description, manifest, node.package_name) column.description = get_rendered(column.description, context) @@ -1671,18 +1714,16 @@ def _process_docs_for_node( def _process_docs_for_source( context: Dict[str, Any], source: SourceDefinition, + manifest: Manifest, ): - table_description = source.description - source_description = source.source_description - table_description = get_rendered(table_description, context) - source_description = get_rendered(source_description, context) - source.description = table_description - source.source_description = source_description + source.doc_blocks = _get_doc_blocks(source.description, manifest, source.package_name) + source.description = get_rendered(source.description, context) + + source.source_description = get_rendered(source.source_description, context) for column in source.columns.values(): - column_desc = column.description - column_desc = get_rendered(column_desc, context) - column.description = column_desc + column.doc_blocks = _get_doc_blocks(column.description, manifest, source.package_name) + column.description = get_rendered(column.description, context) # macro argument descriptions @@ -2040,7 +2081,7 @@ def process_node(config: RuntimeConfig, manifest: Manifest, node: ManifestNode): _process_sources_for_node(manifest, config.project_name, node) _process_refs(manifest, config.project_name, node, config.dependencies) ctx = generate_runtime_docs_context(config, node, manifest, config.project_name) - _process_docs_for_node(ctx, node) + _process_docs_for_node(ctx, node, manifest) def write_semantic_manifest(manifest: Manifest, target_path: str) -> None: diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index c19549cc0f4..0be0a978588 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -622,6 +622,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -730,6 +736,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "root_path": { "anyOf": [ { @@ -1667,6 +1679,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -1775,6 +1793,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -2321,6 +2345,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -2429,6 +2459,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -3115,6 +3151,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -3223,6 +3265,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -3928,6 +3976,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -4036,6 +4090,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -5336,6 +5396,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -5444,6 +5510,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -5990,6 +6062,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -6098,6 +6176,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -6979,6 +7063,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -7087,6 +7177,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -8172,6 +8268,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -8273,6 +8375,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -8546,6 +8654,12 @@ { "type": "string" }, + { + "type": "array", + "items": { + "type": "string" + } + }, { "type": "null" } @@ -9903,6 +10017,12 @@ { "type": "string" }, + { + "type": "array", + "items": { + "type": "string" + } + }, { "type": "null" } @@ -10471,6 +10591,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -10579,6 +10705,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "root_path": { "anyOf": [ { @@ -11516,6 +11648,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -11624,6 +11762,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -12170,6 +12314,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -12278,6 +12428,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -12964,6 +13120,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -13072,6 +13234,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -13777,6 +13945,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -13885,6 +14059,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -15185,6 +15365,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -15293,6 +15479,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -15839,6 +16031,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -15947,6 +16145,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -16828,6 +17032,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -16936,6 +17146,12 @@ "type": "string", "default": "" }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } + }, "language": { "type": "string", "default": "sql" @@ -18012,6 +18228,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": true, @@ -18113,6 +18335,12 @@ } ], "default": null + }, + "doc_blocks": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -18184,6 +18412,12 @@ { "type": "string" }, + { + "type": "array", + "items": { + "type": "string" + } + }, { "type": "null" } diff --git a/tests/functional/artifacts/expected_manifest.py b/tests/functional/artifacts/expected_manifest.py index 02db1905443..8ef9cd3c9be 100644 --- a/tests/functional/artifacts/expected_manifest.py +++ b/tests/functional/artifacts/expected_manifest.py @@ -316,6 +316,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "first_name": { "name": "first_name", @@ -326,6 +327,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "email": { "name": "email", @@ -336,6 +338,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "ip_address": { "name": "ip_address", @@ -346,6 +349,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "updated_at": { "name": "updated_at", @@ -356,6 +360,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, }, "contract": {"checksum": None, "enforced": False, "alias_types": True}, @@ -373,6 +378,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "latest_version": None, "time_spine": None, "freshness": None, + "doc_blocks": [], }, "model.test.second_model": { "compiled_path": os.path.join(compiled_model_path, "second_model.sql"), @@ -416,6 +422,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "first_name": { "name": "first_name", @@ -426,6 +433,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "email": { "name": "email", @@ -436,6 +444,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "ip_address": { "name": "ip_address", @@ -446,6 +455,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "updated_at": { "name": "updated_at", @@ -456,6 +466,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, }, "contract": {"checksum": None, "enforced": False, "alias_types": True}, @@ -473,6 +484,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "latest_version": None, "time_spine": None, "freshness": None, + "doc_blocks": [], }, "seed.test.seed": { "build_path": None, @@ -506,6 +518,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "first_name": { "name": "first_name", @@ -516,6 +529,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "email": { "name": "email", @@ -526,6 +540,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "ip_address": { "name": "ip_address", @@ -536,6 +551,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "updated_at": { "name": "updated_at", @@ -546,6 +562,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, }, "docs": {"node_color": None, "show": True}, @@ -554,6 +571,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "relation_name": relation_name_node_format.format( project.database, my_schema_name, "seed" ), + "doc_blocks": [], }, "test.test.not_null_model_id.d01cc630e6": { "alias": "not_null_model_id", @@ -607,6 +625,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, "contract": {"checksum": None, "enforced": False, "alias_types": True}, + "doc_blocks": [], }, "snapshot.test.snapshot_seed": { "alias": "snapshot_seed", @@ -653,6 +672,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "unique_id": "snapshot.test.snapshot_seed", "unrendered_config": unrendered_snapshot_config, + "doc_blocks": [], }, "test.test.test_nothing_model_.5d38568946": { "alias": "test_nothing_model_", @@ -705,6 +725,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "doc_blocks": [], }, "test.test.unique_model_id.67b76558ff": { "alias": "unique_model_id", @@ -758,6 +779,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "doc_blocks": [], }, }, "sources": { @@ -773,6 +795,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], } }, "config": { @@ -817,6 +840,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "unrendered_config": {}, "unrendered_database": None, "unrendered_schema": "{{ var('test_schema') }}", + "doc_blocks": [], }, }, "exposures": { @@ -1006,6 +1030,7 @@ def expected_references_manifest(project): "constraints": [], "time_spine": None, "freshness": None, + "doc_blocks": [], }, "model.test.ephemeral_summary": { "alias": "ephemeral_summary", @@ -1022,6 +1047,7 @@ def expected_references_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": ["doc.test.summary_first_name"], }, "ct": { "description": "The number of instances of the first name", @@ -1032,6 +1058,7 @@ def expected_references_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": ["doc.test.summary_count"], }, }, "config": get_rendered_model_config(materialized="table", group="test_group"), @@ -1079,6 +1106,7 @@ def expected_references_manifest(project): "constraints": [], "time_spine": None, "freshness": None, + "doc_blocks": ["doc.test.ephemeral_summary"], }, "model.test.view_summary": { "alias": "view_summary", @@ -1095,6 +1123,7 @@ def expected_references_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": ["doc.test.summary_first_name"], }, "ct": { "description": "The number of instances of the first name", @@ -1105,6 +1134,7 @@ def expected_references_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": ["doc.test.summary_count"], }, }, "config": get_rendered_model_config(), @@ -1148,6 +1178,7 @@ def expected_references_manifest(project): "constraints": [], "time_spine": None, "freshness": None, + "doc_blocks": ["doc.test.view_summary"], }, "seed.test.seed": { "alias": "seed", @@ -1163,6 +1194,7 @@ def expected_references_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "first_name": { "name": "first_name", @@ -1173,6 +1205,7 @@ def expected_references_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "email": { "name": "email", @@ -1183,6 +1216,7 @@ def expected_references_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "ip_address": { "name": "ip_address", @@ -1193,6 +1227,7 @@ def expected_references_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "updated_at": { "name": "updated_at", @@ -1203,6 +1238,7 @@ def expected_references_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, }, "config": get_rendered_seed_config(), @@ -1227,6 +1263,7 @@ def expected_references_manifest(project): "checksum": checksum_file(seed_path), "unrendered_config": get_unrendered_seed_config(), "relation_name": '"{0}"."{1}".seed'.format(project.database, my_schema_name), + "doc_blocks": [], }, "snapshot.test.snapshot_seed": { "alias": "snapshot_seed", @@ -1268,6 +1305,7 @@ def expected_references_manifest(project): "unrendered_config": get_unrendered_snapshot_config( target_schema=alternate_schema ), + "doc_blocks": [], }, }, "sources": { @@ -1282,6 +1320,7 @@ def expected_references_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": ["doc.test.column_info"], } }, "config": { @@ -1325,6 +1364,7 @@ def expected_references_manifest(project): "unrendered_config": {}, "unrendered_database": None, "unrendered_schema": "{{ var('test_schema') }}", + "doc_blocks": ["doc.test.table_info"], }, }, "exposures": { @@ -1555,6 +1595,7 @@ def expected_versions_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "ct": { "description": "The number of instances of the first name", @@ -1565,6 +1606,7 @@ def expected_versions_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, }, "config": get_rendered_model_config( @@ -1615,6 +1657,7 @@ def expected_versions_manifest(project): "latest_version": 2, "time_spine": None, "freshness": None, + "doc_blocks": [], }, "model.test.versioned_model.v2": { "alias": "versioned_model_v2", @@ -1631,6 +1674,7 @@ def expected_versions_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, "extra": { "description": "", @@ -1641,6 +1685,7 @@ def expected_versions_manifest(project): "tags": [], "constraints": [], "granularity": None, + "doc_blocks": [], }, }, "config": get_rendered_model_config( @@ -1687,6 +1732,7 @@ def expected_versions_manifest(project): "latest_version": 2, "time_spine": None, "freshness": None, + "doc_blocks": [], }, "model.test.ref_versioned_model": { "alias": "ref_versioned_model", @@ -1746,6 +1792,7 @@ def expected_versions_manifest(project): "latest_version": None, "time_spine": None, "freshness": None, + "doc_blocks": [], }, "test.test.unique_versioned_model_v1_first_name.6138195dec": { "alias": "unique_versioned_model_v1_first_name", @@ -1799,6 +1846,7 @@ def expected_versions_manifest(project): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "doc_blocks": [], }, "test.test.unique_versioned_model_v1_count.0b4c0b688a": { "alias": "unique_versioned_model_v1_count", @@ -1852,6 +1900,7 @@ def expected_versions_manifest(project): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "doc_blocks": [], }, "test.test.unique_versioned_model_v2_first_name.998430d28e": { "alias": "unique_versioned_model_v2_first_name", @@ -1905,6 +1954,7 @@ def expected_versions_manifest(project): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, + "doc_blocks": [], }, }, "exposures": { diff --git a/tests/functional/configs/test_contract_configs.py b/tests/functional/configs/test_contract_configs.py index 179b0058a8d..86649015e6a 100644 --- a/tests/functional/configs/test_contract_configs.py +++ b/tests/functional/configs/test_contract_configs.py @@ -2,6 +2,7 @@ import pytest +from dbt.artifacts.resources.v1.components import ColumnInfo from dbt.exceptions import ParsingError, ValidationError from dbt.tests.util import ( get_artifact, @@ -10,6 +11,7 @@ run_dbt_and_capture, write_file, ) +from dbt_common.contracts.constraints import ColumnLevelConstraint, ConstraintType my_model_sql = """ {{ @@ -331,9 +333,74 @@ def test__model_contract_true(self, project): assert contract_actual_config.enforced is True - expected_columns = "{'id': ColumnInfo(name='id', description='hello', meta={}, data_type='integer', constraints=[ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[]), ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[]), ColumnLevelConstraint(type=, name=None, expression='(id > 0)', warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[])], quote=True, tags=[], _extra={}, granularity=None), 'color': ColumnInfo(name='color', description='', meta={}, data_type='string', constraints=[], quote=None, tags=[], _extra={}, granularity=None), 'date_day': ColumnInfo(name='date_day', description='', meta={}, data_type='date', constraints=[], quote=None, tags=[], _extra={}, granularity=None)}" + expected_columns = { + "id": ColumnInfo( + name="id", + description="hello", + meta={}, + data_type="integer", + doc_blocks=[], + constraints=[ + ColumnLevelConstraint( + type=ConstraintType.not_null, + name=None, + expression=None, + warn_unenforced=True, + warn_unsupported=True, + to=None, + to_columns=[], + ), + ColumnLevelConstraint( + type=ConstraintType.primary_key, + name=None, + expression=None, + warn_unenforced=True, + warn_unsupported=True, + to=None, + to_columns=[], + ), + ColumnLevelConstraint( + type=ConstraintType.check, + name=None, + expression="(id > 0)", + warn_unenforced=True, + warn_unsupported=True, + to=None, + to_columns=[], + ), + ], + quote=True, + tags=[], + _extra={}, + granularity=None, + ), + "color": ColumnInfo( + name="color", + description="", + doc_blocks=[], + meta={}, + data_type="string", + constraints=[], + quote=None, + tags=[], + _extra={}, + granularity=None, + ), + "date_day": ColumnInfo( + name="date_day", + description="", + doc_blocks=[], + meta={}, + data_type="date", + constraints=[], + quote=None, + tags=[], + _extra={}, + granularity=None, + ), + } - assert expected_columns == str(my_model_columns) + assert expected_columns == my_model_columns # compiled fields aren't in the manifest above because it only has parsed fields manifest_json = get_artifact(project.project_root, "target", "manifest.json") diff --git a/tests/functional/docs/test_doc_blocks_backcompat.py b/tests/functional/docs/test_doc_blocks_backcompat.py new file mode 100644 index 00000000000..9623331c60a --- /dev/null +++ b/tests/functional/docs/test_doc_blocks_backcompat.py @@ -0,0 +1,38 @@ +import json +import os + +import pytest + +from dbt.tests.util import run_dbt + +schema_yml = """ +models: + - name: my_colors + doc_blocks: 2 + columns: + - name: id + doc_blocks: 2 + - name: color + doc_blocks: ["hello", 2, "world"] +""" + + +class TestDocBlocksBackCompat: + @pytest.fixture(scope="class") + def models(self): + return { + "my_colors.sql": "select 1 as id, 'blue' as color", + "schema.yml": schema_yml, + } + + def test_doc_blocks_back_compat(self, project): + run_dbt(["parse"]) + + assert os.path.exists("./target/manifest.json") + + with open("./target/manifest.json") as fp: + manifest = json.load(fp) + + model_data = manifest["nodes"]["model.test.my_colors"] + assert model_data["doc_blocks"] == [] + assert all(column["doc_blocks"] == [] for column in model_data["columns"].values()) diff --git a/tests/functional/docs/test_good_docs_blocks.py b/tests/functional/docs/test_good_docs_blocks.py index e1ed96c5eb7..60f39b6e35e 100644 --- a/tests/functional/docs/test_good_docs_blocks.py +++ b/tests/functional/docs/test_good_docs_blocks.py @@ -58,6 +58,8 @@ description: The user's first name - name: last_name description: "{{ doc('test', 'my_model_doc__last_name') }}" + - name: tricky + description: "{{ doc('my_model_doc__id') }} The user's first name {{ doc('test', 'my_model_doc__last_name') }}" """ @@ -82,6 +84,7 @@ def test_valid_doc_ref(self, project): model_data = manifest["nodes"]["model.test.model"] assert model_data["description"] == "My model is just a copy of the seed" + assert model_data["doc_blocks"] == ["doc.test.my_model_doc"] assert { "name": "id", @@ -92,6 +95,7 @@ def test_valid_doc_ref(self, project): "quote": None, "tags": [], "granularity": None, + "doc_blocks": ["doc.test.my_model_doc__id"], } == model_data["columns"]["id"] assert { @@ -103,6 +107,7 @@ def test_valid_doc_ref(self, project): "quote": None, "tags": [], "granularity": None, + "doc_blocks": [], } == model_data["columns"]["first_name"] assert { @@ -114,9 +119,25 @@ def test_valid_doc_ref(self, project): "quote": None, "tags": [], "granularity": None, + "doc_blocks": ["doc.test.my_model_doc__last_name"], } == model_data["columns"]["last_name"] - assert len(model_data["columns"]) == 3 + assert { + "name": "tricky", + "description": "The user ID number The user's first name The user's last name", + "data_type": None, + "constraints": [], + "meta": {}, + "quote": None, + "tags": [], + "granularity": None, + "doc_blocks": [ + "doc.test.my_model_doc__id", + "doc.test.my_model_doc__last_name", + ], + } == model_data["columns"]["tricky"] + + assert len(model_data["columns"]) == 4 class TestGoodDocsBlocksAltPath: @@ -146,6 +167,7 @@ def test_alternative_docs_path(self, project): model_data = manifest["nodes"]["model.test.model"] assert model_data["description"] == "Alt text about the model" + assert model_data["doc_blocks"] == ["doc.test.my_model_doc"] assert { "name": "id", @@ -156,6 +178,7 @@ def test_alternative_docs_path(self, project): "quote": None, "tags": [], "granularity": None, + "doc_blocks": ["doc.test.my_model_doc__id"], } == model_data["columns"]["id"] assert { @@ -167,6 +190,7 @@ def test_alternative_docs_path(self, project): "quote": None, "tags": [], "granularity": None, + "doc_blocks": [], } == model_data["columns"]["first_name"] assert { @@ -178,6 +202,22 @@ def test_alternative_docs_path(self, project): "quote": None, "tags": [], "granularity": None, + "doc_blocks": ["doc.test.my_model_doc__last_name"], } == model_data["columns"]["last_name"] - assert len(model_data["columns"]) == 3 + assert { + "name": "tricky", + "description": "The user ID number with alternative text The user's first name The user's last name in this other file", + "data_type": None, + "constraints": [], + "meta": {}, + "quote": None, + "tags": [], + "granularity": None, + "doc_blocks": [ + "doc.test.my_model_doc__id", + "doc.test.my_model_doc__last_name", + ], + } == model_data["columns"]["tricky"] + + assert len(model_data["columns"]) == 4 diff --git a/tests/unit/contracts/graph/test_manifest.py b/tests/unit/contracts/graph/test_manifest.py index 526f46c7884..9b6c056e802 100644 --- a/tests/unit/contracts/graph/test_manifest.py +++ b/tests/unit/contracts/graph/test_manifest.py @@ -82,6 +82,7 @@ "compiled_path", "patch_path", "docs", + "doc_blocks", "checksum", "unrendered_config", "unrendered_config_call_dict", diff --git a/tests/unit/contracts/graph/test_nodes.py b/tests/unit/contracts/graph/test_nodes.py index 0648fe1174d..665b4dfe573 100644 --- a/tests/unit/contracts/graph/test_nodes.py +++ b/tests/unit/contracts/graph/test_nodes.py @@ -209,6 +209,7 @@ def basic_compiled_dict(): "config_call_dict": {}, "access": "protected", "constraints": [], + "doc_blocks": [], } @@ -529,6 +530,7 @@ def basic_compiled_schema_test_dict(): }, "unrendered_config_call_dict": {}, "config_call_dict": {}, + "doc_blocks": [], } diff --git a/tests/unit/contracts/graph/test_nodes_parsed.py b/tests/unit/contracts/graph/test_nodes_parsed.py index 75d451b9956..0b275fb5ac9 100644 --- a/tests/unit/contracts/graph/test_nodes_parsed.py +++ b/tests/unit/contracts/graph/test_nodes_parsed.py @@ -207,6 +207,7 @@ def base_parsed_model_dict(): "config_call_dict": {}, "access": AccessType.Protected.value, "constraints": [], + "doc_blocks": [], } @@ -315,6 +316,7 @@ def complex_parsed_model_dict(): "meta": {}, "tags": [], "constraints": [], + "doc_blocks": [], }, }, "checksum": { @@ -330,6 +332,7 @@ def complex_parsed_model_dict(): "config_call_dict": {}, "access": AccessType.Protected.value, "constraints": [], + "doc_blocks": [], } @@ -538,6 +541,7 @@ def basic_parsed_seed_dict(): "unrendered_config": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, + "doc_blocks": [], } @@ -632,6 +636,7 @@ def complex_parsed_seed_dict(): "meta": {}, "tags": [], "constraints": [], + "doc_blocks": [], } }, "meta": {"foo": 1000}, @@ -644,6 +649,7 @@ def complex_parsed_seed_dict(): }, "unrendered_config_call_dict": {}, "config_call_dict": {}, + "doc_blocks": [], } @@ -844,6 +850,7 @@ def base_parsed_hook_dict(): "unrendered_config": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, + "doc_blocks": [], } @@ -925,6 +932,7 @@ def complex_parsed_hook_dict(): "meta": {}, "tags": [], "constraints": [], + "doc_blocks": [], }, }, "index": 13, @@ -938,6 +946,7 @@ def complex_parsed_hook_dict(): }, "unrendered_config_call_dict": {}, "config_call_dict": {}, + "doc_blocks": [], } @@ -1083,6 +1092,7 @@ def basic_parsed_schema_test_dict(): "unrendered_config": {}, "unrendered_config_call_dict": {}, "config_call_dict": {}, + "doc_blocks": [], } @@ -1158,6 +1168,7 @@ def complex_parsed_schema_test_dict(): "meta": {}, "tags": [], "constraints": [], + "doc_blocks": [], }, }, "column_name": "id", @@ -1172,6 +1183,7 @@ def complex_parsed_schema_test_dict(): "unrendered_config": {"materialized": "table", "severity": "WARN"}, "unrendered_config_call_dict": {}, "config_call_dict": {}, + "doc_blocks": [], } @@ -1567,6 +1579,7 @@ def basic_timestamp_snapshot_dict(): }, "unrendered_config_call_dict": {}, "config_call_dict": {}, + "doc_blocks": [], } @@ -1672,6 +1685,7 @@ def basic_check_snapshot_dict(): }, "unrendered_config_call_dict": {}, "config_call_dict": {}, + "doc_blocks": [], } @@ -1881,6 +1895,7 @@ def basic_parsed_source_definition_dict(): "enabled": True, }, "unrendered_config": {}, + "doc_blocks": [], } @@ -1913,6 +1928,7 @@ def complex_parsed_source_definition_dict(): "freshness": {"warn_after": {"period": "hour", "count": 1}, "error_after": {}}, "loaded_at_field": "loaded_at", "unrendered_config": {}, + "doc_blocks": [], }