feat(ingestion/lookml): resolve access notation for LookML Constant

metadata-ingestion/docs/sources/looker/looker_recipe.yml

@@ -8,4 +8,18 @@ source:
     client_id: ${LOOKER_CLIENT_ID}
     client_secret: ${LOOKER_CLIENT_SECRET}
 
+    # Liquid variables
+    # liquid_variables:
+    #   _user_attributes:
+    #     looker_env: "dev"
+    #     dev_database_prefix: "employee"
+    #     dev_schema_prefix: "public"
+    #   dw_eff_dt_date:
+    #     _is_selected: true
+    #   source_region: "ap-south-1"
+    #   db: "test-db"
+
+    # LookML Constants
+    # lookml_constants:
+    #   star_award_winner_year: "public.winner_2025"
 # sink configs

metadata-ingestion/docs/sources/looker/lookml_post.md

@@ -1,11 +1,49 @@
-#### Configuration Notes
-
-1. If a view contains a liquid template (e.g. `sql_table_name: {{ user_attributes['db']}}.kafka_streaming.events }}`, with `db=ANALYTICS_PROD`), then you will need to specify the values of those variables in the `liquid_variable` config as shown below:
-    ```yml
-      liquid_variable:
-        user_attributes:
-          db: ANALYTICS_PROD
-    ```
+### Configuration Notes
+
+1. Handling Liquid Templates
+
+   If a view contains a liquid template, for example:
+
+   ```
+   sql_table_name: {{ user_attributes['db'] }}.kafka_streaming.events
+   ```
+
+   where `db=ANALYTICS_PROD`, you need to specify the values of those variables in the liquid_variables configuration as shown below:
+
+   ```yml
+   liquid_variables:
+     user_attributes:
+       db: ANALYTICS_PROD
+   ```
+
+2. Resolving LookML Constants
+
+   If a view contains a LookML constant, for example:
+
+   ```
+   sql_table_name: @{db}.kafka_streaming.events;
+   ```
+
+   Ingestion attempts to resolve it's value by looking at project manifest files
+
+     ```yml
+     manifest.lkml
+       constant: db {
+           value: "ANALYTICS_PROD"
+       }
+     ```
+
+   - If the constant's value is not resolved or incorrectly resolved, you can specify `lookml_constants` configuration in ingestion recipe as shown below. The constant value in recipe takes precedence over constant values resolved from manifest.
+
+     ```yml
+     lookml_constants:
+       db: ANALYTICS_PROD
+     ```
+
+
+**Additional Notes**
+
+Although liquid variables and LookML constants can be used anywhere in LookML code, their values are currently resolved only for LookML views by DataHub LookML ingestion. This behavior is sufficient since LookML ingestion processes only views and their upstream dependencies.
 
 ### Multi-Project LookML (Advanced)
 

metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py

@@ -32,6 +32,12 @@ class LookerField:
     sql: Optional[str]
 
 
+@dataclass
+class LookerConstant:
+    name: str
+    value: str
+
+
 @dataclass
 class LookerModel:
     connection: str
@@ -75,6 +81,7 @@ def from_looker_dict(
             try:
                 parsed = load_and_preprocess_file(
                     path=included_file,
+                    reporter=reporter,
                     source_config=source_config,
                 )
                 included_explores = parsed.get("explores", [])
@@ -217,6 +224,7 @@ def resolve_includes(
                 try:
                     parsed = load_and_preprocess_file(
                         path=included_file,
+                        reporter=reporter,
                         source_config=source_config,
                     )
                     seen_so_far.add(included_file)

metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py

@@ -1,10 +1,13 @@
 import logging
 import pathlib
 from dataclasses import replace
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition
-from datahub.ingestion.source.looker.looker_dataclasses import LookerViewFile
+from datahub.ingestion.source.looker.looker_dataclasses import (
+    LookerConstant,
+    LookerViewFile,
+)
 from datahub.ingestion.source.looker.looker_template_language import (
     load_and_preprocess_file,
 )
@@ -30,12 +33,14 @@ def __init__(
         base_projects_folder: Dict[str, pathlib.Path],
         reporter: LookMLSourceReport,
         source_config: LookMLSourceConfig,
+        manifest_constants: List[LookerConstant] = [],
     ) -> None:
         self.viewfile_cache: Dict[str, Optional[LookerViewFile]] = {}
         self._root_project_name = root_project_name
         self._base_projects_folder = base_projects_folder
         self.reporter = reporter
         self.source_config = source_config
+        self.manifest_constants = manifest_constants
 
     def _load_viewfile(
         self, project_name: str, path: str, reporter: LookMLSourceReport
@@ -71,9 +76,15 @@ def _load_viewfile(
         try:
             logger.debug(f"Loading viewfile {path}")
 
+            # load_and preprocess_file is called multiple times for loading view file from multiple flows.
+            # Flag resolve_constants is a hack to avoid passing around manifest_constants from all of the flows.
+            # This is fine as rest of flows do not need resolution of constants.
             parsed = load_and_preprocess_file(
                 path=path,
+                reporter=self.reporter,
                 source_config=self.source_config,
+                resolve_constants=True,
+                manifest_constants=self.manifest_constants,
             )
 
             looker_viewfile = LookerViewFile.from_looker_dict(