From 9463fad648586c0f29195a7b7877b15338da345d Mon Sep 17 00:00:00 2001 From: salim_moulouel Date: Mon, 26 Feb 2024 13:30:55 +0100 Subject: [PATCH 1/3] support dbt-core between 1.7.0 to 1.7.8 --- .github/workflows/test_cli.yml | 2 +- .github/workflows/test_integration_adapter.yml | 2 +- .github/workflows/test_integration_cli.yml | 4 ++-- projects/adapter/pyproject.toml | 4 ++-- projects/adapter/src/dbt/adapters/fal/__version__.py | 2 +- .../src/dbt/adapters/fal_experimental/__version__.py | 2 +- projects/adapter/src/fal/dbt/cli/fal_runner.py | 4 ++++ projects/adapter/src/fal/dbt/integration/parse.py | 4 ++-- projects/adapter/src/fal/dbt/integration/project.py | 8 +++++++- 9 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_cli.yml b/.github/workflows/test_cli.yml index 478443bbc..202b9c9d9 100644 --- a/.github/workflows/test_cli.yml +++ b/.github/workflows/test_cli.yml @@ -21,7 +21,7 @@ jobs: - "3.10" # - "3.11" dbt: - - "1.5.*" + - "1.7.*" # Run only the latest commit pushed to PR concurrency: diff --git a/.github/workflows/test_integration_adapter.yml b/.github/workflows/test_integration_adapter.yml index 217aefb5b..0e145454b 100644 --- a/.github/workflows/test_integration_adapter.yml +++ b/.github/workflows/test_integration_adapter.yml @@ -33,7 +33,7 @@ jobs: # - duckdb # - sqlserver dbt_version: - - "1.5.0" + - "1.7.0" python: - "3.8" - "3.9" diff --git a/.github/workflows/test_integration_cli.yml b/.github/workflows/test_integration_cli.yml index 655fdd867..cb8fffe1d 100644 --- a/.github/workflows/test_integration_cli.yml +++ b/.github/workflows/test_integration_cli.yml @@ -52,7 +52,7 @@ on: options: - "" - "latest" - - "1.5.*" + - "1.7.*" jobs: matrix-adapter: @@ -163,7 +163,7 @@ jobs: shell: python run: | OPTIONS = [ - "1.5.*", + "1.7.*", ] OUTPUT = OPTIONS diff --git a/projects/adapter/pyproject.toml b/projects/adapter/pyproject.toml index 9e303aa75..edb5fcc3c 100644 --- a/projects/adapter/pyproject.toml +++ b/projects/adapter/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dbt-fal" -version = "1.5.10a0" +version = "1.7.10a0" # name = "fal" # version = "0.9.4a0" description = "Run python scripts from any dbt project." @@ -22,7 +22,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.8" -dbt-core = ">=1.5,<=1.5.5" +dbt-core = ">=1.7,<=1.7.8" pandas = "^1.3.4" posthog = "^1.4.5" "backports.functools_lru_cache" = "^1.6.4" diff --git a/projects/adapter/src/dbt/adapters/fal/__version__.py b/projects/adapter/src/dbt/adapters/fal/__version__.py index 1340d2b53..6477de0c5 100644 --- a/projects/adapter/src/dbt/adapters/fal/__version__.py +++ b/projects/adapter/src/dbt/adapters/fal/__version__.py @@ -1 +1 @@ -version = '1.5.10a0' +version = '1.7.10a0' diff --git a/projects/adapter/src/dbt/adapters/fal_experimental/__version__.py b/projects/adapter/src/dbt/adapters/fal_experimental/__version__.py index 1340d2b53..6477de0c5 100644 --- a/projects/adapter/src/dbt/adapters/fal_experimental/__version__.py +++ b/projects/adapter/src/dbt/adapters/fal_experimental/__version__.py @@ -1 +1 @@ -version = '1.5.10a0' +version = '1.7.10a0' diff --git a/projects/adapter/src/fal/dbt/cli/fal_runner.py b/projects/adapter/src/fal/dbt/cli/fal_runner.py index 63365249a..f8bce7cd6 100644 --- a/projects/adapter/src/fal/dbt/cli/fal_runner.py +++ b/projects/adapter/src/fal/dbt/cli/fal_runner.py @@ -14,8 +14,11 @@ def create_fal_dbt( args: argparse.Namespace, generated_models: Dict[str, Path] = {} ) -> FalDbt: real_state = None + real_defer_state = None if hasattr(args, "state") and args.state is not None: real_state = args.state + if hasattr(args, "real_defer_state") and args.defer_state is not None: + real_defer_state = args.state return FalDbt( args.project_dir, @@ -25,6 +28,7 @@ def create_fal_dbt( args.selector, args.threads, real_state, + real_defer_state, args.target, args.vars, generated_models, diff --git a/projects/adapter/src/fal/dbt/integration/parse.py b/projects/adapter/src/fal/dbt/integration/parse.py index ba211bb04..1c999ac2b 100644 --- a/projects/adapter/src/fal/dbt/integration/parse.py +++ b/projects/adapter/src/fal/dbt/integration/parse.py @@ -5,7 +5,7 @@ from typing import Any, List, Dict, Optional, Union, TYPE_CHECKING from dbt.contracts.project import Project as ProjectContract -from dbt.config import RuntimeConfig, Project +from dbt.config import RuntimeConfig, Project, PartialProject from dbt.config.utils import parse_cli_vars as dbt_parse_cli_vars from dbt.contracts.graph.manifest import Manifest from dbt.contracts.results import RunResultsArtifact, FreshnessExecutionResultArtifact @@ -47,7 +47,7 @@ class RuntimeArgs: def load_dbt_project_contract(project_dir: str) -> ProjectContract: - partial_project = Project.partial_load(project_dir) + partial_project = PartialProject.from_project_root(project_dir) contract = ProjectContract.from_dict(partial_project.project_dict) if not hasattr(contract, "model_paths") or contract.model_paths is None: setattr(contract, "model_paths", contract.source_paths) diff --git a/projects/adapter/src/fal/dbt/integration/project.py b/projects/adapter/src/fal/dbt/integration/project.py index 71d4228c3..38725ed6b 100644 --- a/projects/adapter/src/fal/dbt/integration/project.py +++ b/projects/adapter/src/fal/dbt/integration/project.py @@ -436,6 +436,7 @@ class CompileArgs: models: List[str] exclude: Tuple[str] state: Optional[Path] + defer_state: Optional[Path] single_threaded: Optional[bool] @@ -453,6 +454,7 @@ def __init__( selector: Optional[str] = None, threads: Optional[int] = None, state: Optional[str] = None, + defer_state: Optional[str] = None, profile_target: Optional[str] = None, args_vars: str = "{}", generated_models: Dict[str, Path] = {}, @@ -485,9 +487,13 @@ def __init__( self.profiles_dir = flags.PROFILES_DIR self._state = None + self._defer_state = None if state is not None: self._state = Path(os.path.realpath(os.path.expanduser(state))) + if defer_state is not None: + self._defer_state = Path(os.path.realpath(os.path.expanduser(defer_state))) + self.scripts_dir = parse.get_scripts_dir(self.project_dir, args_vars) @@ -525,7 +531,7 @@ def __init__( # Necessary for manifest loading to not fail # dbt.tracking.initialize_tracking(self.profiles_dir) - args = CompileArgs(selector, select, select, exclude, self._state, None) + args = CompileArgs(selector, select, select, exclude, self._state, self._defer_state, None) self._compile_task = CompileTask(args, self._config, native_manifest) self._compile_task._runtime_initialize() From 8b13aa6a5f56a95a7670048567e76a7ee5d27aec Mon Sep 17 00:00:00 2001 From: salim_moulouel Date: Tue, 27 Feb 2024 18:54:12 +0100 Subject: [PATCH 2/3] managing dbt-core 1.6 also --- projects/adapter/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/adapter/pyproject.toml b/projects/adapter/pyproject.toml index edb5fcc3c..f4018ee82 100644 --- a/projects/adapter/pyproject.toml +++ b/projects/adapter/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.8" -dbt-core = ">=1.7,<=1.7.8" +dbt-core = ">=1.6,<=1.7.8" pandas = "^1.3.4" posthog = "^1.4.5" "backports.functools_lru_cache" = "^1.6.4" From 74e739c2dd664170933781448a6e8ca5f5a1ead0 Mon Sep 17 00:00:00 2001 From: kudryk Date: Thu, 4 Apr 2024 10:13:35 -0600 Subject: [PATCH 3/3] Set source_node argument. --- projects/adapter/src/fal/dbt/integration/project.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/adapter/src/fal/dbt/integration/project.py b/projects/adapter/src/fal/dbt/integration/project.py index 38725ed6b..2b839b787 100644 --- a/projects/adapter/src/fal/dbt/integration/project.py +++ b/projects/adapter/src/fal/dbt/integration/project.py @@ -683,6 +683,7 @@ def _model( ) -> ManifestNode: # HACK: always setting node package as self.project_dir target_model: MaybeNonSource = self._manifest.native_manifest.resolve_ref( + None, target_model_name, target_package_name, None,