Add testing framework, CI, and make post-commands optional (#6)

* Spike out CI workflow yml

* Spike out pytest and fixtures

* Fix duplication of slug logic in copier.yml

* Quote new jinja project_slug in copier.yml

* Resotre README bit that was lost again

* Expand README with CLI recs and learning resources

* Make disclaimer about me working at dbt a footnote

* Fix footnote disclaimer formatting

* Fix wording of disclaimer footnote

* Amended: profiles integration test working; rm'd test-build from git

* Integration tests working for BQ, SF, DDB

* Expand README re command optionality

* Move dev-requirements files to project root, update CI.yml

* Capitalize CI in workflow name

* Expand template exclude list to include dev-reqs

* Compile requirements before installing in CI

* Compile requirements before installing in CI corrected to dev-requirements

* Fix copier.yml from wonky merge

* Ensure tests are running copier state from HEAD

* Simplify CI matrix to py 3.10 macos for now

* Make 3.10 a string in ci.yml

.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+name: CI
+
+on:
+  pull_request:
+    branch: main
+
+jobs:
+  ci:
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+        os: [macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout PR branch
+        uses: actions/[email protected]
+
+      - name: Setup Python
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv and requirements
+        run: |
+          python -m pip install uv
+          uv pip compile dev-requirements.in -o dev-requirements.txt
+          uv pip install -r dev-requirements.txt --system
+
+      - name: Run integration tests
+        run: pytest template-integration-tests

.gitignore

@@ -0,0 +1,4 @@
+.venv
+__pycache__
+template-integration-tests/test-build
+.pytest_cache

copier.yml

@@ -6,13 +6,13 @@ project_name:
 
 project_slug:
   type: str
-  default: "{{ project_name | lower | replace(' ', '_') | replace('-', '_') }}"
+  default: "{{ project_name | lower | replace(' ', '_') | replace('-', '_') | replace('.', '') }}"
   help: The slugified name of your project.
 
 # Warehouse profile connection configs
 profile_name:
   type: str
-  default: "{{ project_name | lower | replace(' ', '_') | replace('-', '_') }}"
+  default: "{{ project_slug }}"
   help: |
     The name of your default dev profile, this is the set of configs for connecting 
     to your warehouse for development.
@@ -122,11 +122,33 @@ group_by_and_order_by_style:
     - implicit
     - explicit
 
+# Task options
+virtual_environment:
+  type: bool
+  default: False
+  help: |
+    Do you want copier to initialize and activate a virtual environment,
+    and install dependencies?
+
+init_repo:
+  type: bool
+  default: False
+  help: |
+    Do you want copier to initialize and make a first commit to a fresh git repo?
+
+move_profile:
+  type: bool
+  default: False
+  help: |
+    Do you want copier to move the contents of your generated `profiles.yml` file
+    to the appropriate place in your home directory (`~/.dbt/profiles.yml`)?
+
 # Python configs
 virtual_environment_name:
   type: str
   default: .venv
   help: What do you want your virtual environment to be called?
+  when: "{{ virtual_environment }}"
 
 # Files to exclude from template
 _exclude:
@@ -135,10 +157,17 @@ _exclude:
   - "~*"
   - "*.py[co]"
   - "__pycache__"
+  - ".pytest_cache"
   - ".git"
+  - ".github"
   - ".DS_STORE"
   - "README.md"
   - ".venv"
+  - ".env"
+  - "venv"
+  - "env"
+  - "dev-requirements.txt"
+  - "dev-requirements.in"
 
 # Pre copy message
 _message_before_copy: |
@@ -171,17 +200,16 @@ _message_after_copy: |
 
 # Tasks
 _tasks:
-  - "python3 -m venv {{ virtual_environment_name }}"
-  - "source {{ virtual_environment_name }}/bin/activate"
-  - "python3 -m pip install uv"
-  - "source {{ virtual_environment_name }}/bin/activate"
-  - "uv pip compile requirements.in -o requirements.txt"
-  - "uv pip install --upgrade pip"
-  - "uv pip install -r requirements.txt"
-  - "mkdir -p ~/.dbt && cat profiles.yml >> ~/.dbt/profiles.yml"
-  - "rm profiles.yml"
-  - "git init"
-  - "git add --all"
-  - "git commit -m 'Initial commit.'"
-  - "source {{ virtual_environment_name }}/bin/activate && pre-commit install"
-
+  - "{% if virtual_environment %} python3 -m venv {{ virtual_environment_name }} {% endif %}"
+  - "{% if virtual_environment %} source {{ virtual_environment_name }}/bin/activate {% endif %}"
+  - "{% if virtual_environment %} python3 -m pip install --upgrade pip {% endif %}"
+  - "{% if virtual_environment %} python3 -m pip install uv {% endif %}"
+  - "{% if virtual_environment %} source {{ virtual_environment_name }}/bin/activate {% endif %}"
+  - "{% if virtual_environment %} uv pip compile requirements.in -o requirements.txt {% endif %}"
+  - "{% if virtual_environment %} uv pip install -r requirements.txt {% endif %}"
+  - "{% if move_profile %} mkdir -p ~/.dbt && cat profiles.yml >> ~/.dbt/profiles.yml {% endif %}"
+  - "{% if move_profile %} rm profiles.yml {% endif %}"
+  - "{% if init_repo %} git init {% endif %}"
+  - "{% if init_repo %} git add --all {% endif %}"
+  - "{% if init_repo %} git commit -m 'Initial commit.' {% endif %}"
+  - "{% if virtual_environment and init_repo %} source {{ virtual_environment_name }}/bin/activate && pre-commit install {% endif %}"

dev-requirements.in

@@ -0,0 +1,3 @@
+pytest
+copier
+deepdiff

dev-requirements.txt

@@ -0,0 +1,62 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile template-integration-tests/dev-requirements.in -o template-integration-tests/dev-requirements.txt
+annotated-types==0.6.0
+    # via pydantic
+colorama==0.4.6
+    # via copier
+copier==9.1.1
+decorator==5.1.1
+    # via copier
+dunamai==1.19.2
+    # via copier
+exceptiongroup==1.2.0
+    # via pytest
+funcy==2.0
+    # via copier
+iniconfig==2.0.0
+    # via pytest
+jinja2==3.1.3
+    # via
+    #   copier
+    #   jinja2-ansible-filters
+jinja2-ansible-filters==1.3.2
+    # via copier
+markupsafe==2.1.5
+    # via jinja2
+packaging==23.2
+    # via
+    #   copier
+    #   dunamai
+    #   pytest
+pathspec==0.12.1
+    # via copier
+pluggy==1.4.0
+    # via pytest
+plumbum==1.8.2
+    # via copier
+prompt-toolkit==3.0.36
+    # via questionary
+pydantic==2.6.3
+    # via copier
+pydantic-core==2.16.3
+    # via pydantic
+pygments==2.17.2
+    # via copier
+pytest==8.1.0
+pyyaml==6.0.1
+    # via
+    #   copier
+    #   jinja2-ansible-filters
+    #   pyyaml-include
+pyyaml-include==1.3.2
+    # via copier
+questionary==2.0.1
+    # via copier
+tomli==2.0.1
+    # via pytest
+typing-extensions==4.10.0
+    # via
+    #   pydantic
+    #   pydantic-core
+wcwidth==0.2.13
+    # via prompt-toolkit

profiles.yml.jinja

@@ -3,71 +3,55 @@
   target: dev
   outputs:
     dev:
-      type: {{ data_warehouse }}
-      account: {{ account_env_var }}
-      user: {{ username }}
+      type: {{ data_warehouse | lower }}
+      account: {{ account_env_var | lower }}
+      user: {{ username | lower }}
       authenticator: externalbrowser
-      database: {{ database }}
-      warehouse: {{ warehouse }}
-      schema: {{ schema }}
-      threads: {{ thread_count }}
+      database: {{ database | lower}}
+      warehouse: {{ warehouse | lower }}
+      schema: {{ schema | lower }}
+      threads: {{ thread_count}}
 {%- endif -%}
-{%- if data_warehouse == "bigquery" -%}
+{% if data_warehouse == "bigquery" %}
 {{ profile_name }}:
   target: dev
   outputs:
     dev:
       type: {{ data_warehouse }}
-      project: {{ database }}
-      dataset: {{ schema }}
+      project: {{ database | lower }}
+      dataset: {{ schema | lower }}
       method: oauth
       threads: {{ thread_count }}
 {%- endif -%}
-{%- if data_warehouse == "redshift" -%}
+{% if data_warehouse == "redshift" %}
 {{ profile_name }}:
   target: dev
   outputs:
     dev:
-      type: {{ data_warehouse }}
-      account: {{ account_env_var }}
-      user: {{ username }}
-      database: {{ database }}
-      schema: {{ schema }}
-      threads: {{ thread_count }}
+      # coming soon
 {%- endif -%}
-{%- if data_warehouse == "duckdb" -%}
+{% if data_warehouse == "duckdb" %}
 {{ profile_name }}:
   target: dev
   outputs:
     dev:
       type: {{ data_warehouse }}
-      path: {{ duckdb_file_path }}
+      path: {{ duckdb_file_path | lower }}
+      database: {{ database | lower }}
+      schema: {{ schema | lower }}
+      threads: {{ thread_count }}
 {%- endif -%}
-{%- if data_warehouse == "postgres" -%}
+{% if data_warehouse == "postgres" %}
 {{ profile_name }}:
   target: dev
   outputs:
     dev:
-      type: {{ data_warehouse }}
-      account: {{ account_env_var }}
-      user: {{ username }}
-      authenticator: externalbrowser
-      database: {{ database }}
-      warehouse: {{ warehouse }}
-      schema: {{ schema }}
-      threads: {{ thread_count }}
+      # coming soon
 {%- endif -%}
-{%- if data_warehouse == "databricks" -%}
+{% if data_warehouse == "databricks" %}
 {{ profile_name }}:
   target: dev
   outputs:
     dev:
-      type: {{ data_warehouse }}
-      account: {{ account_env_var }}
-      user: {{ username }}
-      authenticator: externalbrowser
-      database: {{ database }}
-      warehouse: {{ warehouse }}
-      schema: {{ schema }}
-      threads: {{ thread_count }}
+      # coming soon
 {%- endif -%}

template-integration-tests/test-expectations/bigquery_profile.yml

@@ -0,0 +1,10 @@
+
+legoalas_corp:
+  target: dev
+  outputs:
+    dev:
+      type: bigquery
+      project: mirkwood
+      dataset: archers
+      method: oauth
+      threads: 8

template-integration-tests/test-expectations/duckdb_profile.yml

@@ -0,0 +1,10 @@
+
+lothlorien_enterprises:
+  target: dev
+  outputs:
+    dev:
+      type: duckdb
+      path: ./lothlorien.db
+      database: mallorn
+      schema: flets
+      threads: 8

template-integration-tests/test-expectations/snowflake_profile.yml

@@ -0,0 +1,13 @@
+
+aragorn_inc:
+  target: dev
+  outputs:
+    dev:
+      type: snowflake
+      account: 
+      user: strider
+      authenticator: externalbrowser
+      database: gondor
+      warehouse: narsil
+      schema: rangers
+      threads: 8