diff --git a/dlt/extract/extractors.py b/dlt/extract/extractors.py index 41d3035a9f..12ae9e5972 100644 --- a/dlt/extract/extractors.py +++ b/dlt/extract/extractors.py @@ -130,7 +130,7 @@ def write_items(self, resource: DltResource, items: TDataItems, meta: Any) -> No # convert to table meta if created table variant so item is assigned to this table if meta.create_table_variant: # name in hints meta must be a string, otherwise merge_hints would fail - meta = TableNameMeta(meta.hints["name"]) # type: ignore[arg-type] + meta = TableNameMeta(meta.hints["table_name"]) # type: ignore[arg-type] self._reset_contracts_cache() if table_name := self._get_static_table_name(resource, meta): diff --git a/dlt/extract/hints.py b/dlt/extract/hints.py index 037ebbddf9..228753cbf0 100644 --- a/dlt/extract/hints.py +++ b/dlt/extract/hints.py @@ -42,23 +42,23 @@ class TResourceHintsBase(TypedDict, total=False): + table_name: Optional[TTableHintTemplate[str]] write_disposition: Optional[TTableHintTemplate[TWriteDispositionConfig]] parent: Optional[TTableHintTemplate[str]] primary_key: Optional[TTableHintTemplate[TColumnNames]] + columns: Optional[TTableHintTemplate[TAnySchemaColumns]] schema_contract: Optional[TTableHintTemplate[TSchemaContract]] table_format: Optional[TTableHintTemplate[TTableFormat]] + file_format: TTableHintTemplate[TFileFormat] merge_key: Optional[TTableHintTemplate[TColumnNames]] + nested_hints: Optional[Dict[TColumnNames, "TResourceHintsBase"]] class TResourceHints(TResourceHintsBase, total=False): - name: TTableHintTemplate[str] # description: TTableHintTemplate[str] - # table_sealed: Optional[bool] - columns: TTableHintTemplate[TTableSchemaColumns] - incremental: Incremental[Any] - file_format: TTableHintTemplate[TFileFormat] + incremental: Optional[Incremental[Any]] validator: ValidateItem - original_columns: TTableHintTemplate[TAnySchemaColumns] + original_columns: Optional[TTableHintTemplate[TAnySchemaColumns]] class HintsMeta: @@ -98,7 +98,7 @@ def make_hints( file_format=file_format, # type: ignore ) if not table_name: - new_template.pop("name") + new_template.pop("table_name") if not write_disposition and "write_disposition" in new_template: new_template.pop("write_disposition") # remember original columns and set template columns @@ -139,7 +139,7 @@ def table_name(self) -> TTableHintTemplate[str]: if self._table_name_hint_fun: return self._table_name_hint_fun # get table name or default name - return self._hints.get("name") or self.name if self._hints else self.name + return self._hints.get("table_name") or self.name if self._hints else self.name @table_name.setter def table_name(self, value: TTableHintTemplate[str]) -> None: @@ -158,7 +158,7 @@ def write_disposition(self, value: TTableHintTemplate[TWriteDispositionConfig]) @property def columns(self) -> TTableHintTemplate[TTableSchemaColumns]: """Gets columns' schema that can be modified in place""" - return None if self._hints is None else self._hints.get("columns") + return None if self._hints is None else self._hints.get("columns") # type: ignore[return-value] @property def schema_contract(self) -> TTableHintTemplate[TSchemaContract]: @@ -187,8 +187,8 @@ def compute_table_schema(self, item: TDataItem = None, meta: Any = None) -> TTab # resolve a copy of a held template table_template = self._clone_hints(table_template) - if "name" not in table_template: - table_template["name"] = self.name + if "table_name" not in table_template: + table_template["table_name"] = self.name # if table template present and has dynamic hints, the data item must be provided. if self._table_name_hint_fun and item is None: @@ -276,9 +276,9 @@ def apply_hints( t = self._clone_hints(t) if table_name is not None: if table_name: - t["name"] = table_name + t["table_name"] = table_name else: - t.pop("name", None) + t.pop("table_name", None) if parent_table_name is not None: if parent_table_name: t["parent"] = parent_table_name @@ -296,6 +296,7 @@ def apply_hints( # normalize columns columns = ensure_table_schema_columns(columns) # this updates all columns with defaults + assert isinstance(t["columns"], dict) t["columns"] = merge_columns(t["columns"], columns, merge_columns=True) else: # set to empty columns @@ -354,7 +355,8 @@ def _set_hints( DltResourceHints.validate_dynamic_hints(hints_template) DltResourceHints.validate_write_disposition_hint(hints_template.get("write_disposition")) if create_table_variant: - table_name: str = hints_template["name"] # type: ignore[assignment] + # for table variants, table name must be a str + table_name: str = hints_template["table_name"] # type: ignore[assignment] # incremental cannot be specified in variant if hints_template.get("incremental"): raise InconsistentTableTemplate( @@ -388,7 +390,7 @@ def merge_hints( self, hints_template: TResourceHints, create_table_variant: bool = False ) -> None: self.apply_hints( - table_name=hints_template.get("name"), + table_name=hints_template.get("table_name"), parent_table_name=hints_template.get("parent"), write_disposition=hints_template.get("write_disposition"), columns=hints_template.get("original_columns"), diff --git a/dlt/sources/rest_api/typing.py b/dlt/sources/rest_api/typing.py index 22a9560433..e51ecd640f 100644 --- a/dlt/sources/rest_api/typing.py +++ b/dlt/sources/rest_api/typing.py @@ -254,9 +254,7 @@ class ProcessingSteps(TypedDict): class ResourceBase(TResourceHintsBase, total=False): """Defines hints that may be passed to `dlt.resource` decorator""" - table_name: Optional[TTableHintTemplate[str]] max_table_nesting: Optional[int] - columns: Optional[TTableHintTemplate[TAnySchemaColumns]] selected: Optional[bool] parallelized: Optional[bool] processing_steps: Optional[List[ProcessingSteps]]