[FEATURE] Switch testing to native hatch command (#8)

<!--- Provide a general summary of your changes in the Title above -->

## Description
Switch to `hatch test` command and improve tests.

## Related Issue
#5 

## Motivation and Context
Use native hatch approach

## How Has This Been Tested?
Run all tests

## Types of changes
<!--- What types of changes does your code introduce? Put an `x` in all
the boxes that apply: -->
- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)

## Checklist:
<!--- Go over all the following points, and put an `x` in all the boxes
that apply. -->
<!--- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
- [x] My code follows the code style of this project.
- [ ] My change requires a change to the documentation.
- [ ] I have updated the documentation accordingly.
- [x] I have read the **CONTRIBUTING** document.
- [ ] I have added tests to cover my changes.
- [x] All new and existing tests passed.

---------

Co-authored-by: Danny Meijer <[email protected]>

.github/workflows/test.yaml

@@ -27,18 +27,18 @@ jobs:
         os: [ubuntu-latest]
         # os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ['3.9', '3.10', '3.11', '3.12']
-        pyspark-version: ['33', '34', '35']
-        exclude:
-          - python-version: '3.9'
-            pyspark-version: '35'
-          - python-version: '3.11'
-            pyspark-version: '33'
-          - python-version: '3.11'
-            pyspark-version: '34'
-          - python-version: '3.12'
-            pyspark-version: '33'
-          - python-version: '3.12'
-            pyspark-version: '34'
+        # pyspark-version: ['33', '34', '35']
+        # exclude:
+        #   - python-version: '3.9'
+        #     pyspark-version: '35'
+        #   - python-version: '3.11'
+        #     pyspark-version: '33'
+        #   - python-version: '3.11'
+        #     pyspark-version: '34'
+        #   - python-version: '3.12'
+        #     pyspark-version: '33'
+        #   - python-version: '3.12'
+        #     pyspark-version: '34'
 
     steps:
     - uses: actions/checkout@v4
@@ -52,10 +52,11 @@ jobs:
       run: python -m pip install --upgrade pip
 
     - name: Install hatch
-      run: pip install  hatch
+      run: pip install hatch
 
     - name: Run tests
-      run: hatch run test.py${{ matrix.python-version }}-pyspark${{ matrix.pyspark-version }}:all-tests
+      # run: hatch run test --python=${{ matrix.python-version }} -i pyspark${{ matrix.pyspark-version }}
+      run: hatch test --python=${{ matrix.python-version }}
 
   # https://github.com/marketplace/actions/alls-green#why
   final_check: # This job does nothing and is only used for the branch protection

.gitignore

@@ -126,16 +126,16 @@ spark-warehouse
 out/**
 *.iml
 
-/test/integration/**/task_definition/*.yaml
+/tests/integration/**/task_definition/*.yaml
 /.vscode/settings.json
 /.vscode/launch.json
 
 .databricks
 
 # PYTEST
-/test/data/spark-warehouse
-/test/data/checkpoint
-/test/data/snowflake_staging.parq/
+/tests/_data/spark-warehouse
+/tests/_data/checkpoint
+/tests/_data/snowflake_staging.parq/
 
 # DevContainer
 .devcontainer

Makefile

@@ -105,27 +105,27 @@ coverage: cov
 all-tests:
 	@echo "\033[1mRunning all tests:\033[0m\n\033[35m This will run the full test suite\033[0m"
 	@echo "\033[1;31mWARNING:\033[0;33m This may take upward of 20-30 minutes to complete!\033[0m"
-	@hatch run test:all-tests --no-header --no-summary
+	@hatch test --no-header --no-summary
 .PHONY: spark-tests  ## testing - Run SPARK tests in ALL environments
 spark-tests:
 	@echo "\033[1mRunning Spark tests:\033[0m\n\033[35m This will run the Spark test suite against all specified environments\033[0m"
 	@echo "\033[1;31mWARNING:\033[0;33m This may take upward of 20-30 minutes to complete!\033[0m"
-	@hatch run test:spark-tests --no-header --no-summary
+	@hatch test -m spark --no-header --no-summary
 .PHONY: non-spark-tests  ## testing - Run non-spark tests in ALL environments
 non-spark-tests:
 	@echo "\033[1mRunning non-Spark tests:\033[0m\n\033[35m This will run the non-Spark test suite against all specified environments\033[0m"
-	@hatch run test:non-spark-tests --no-header --no-summary -k "not spark"
+	@hatch test -m "not spark" --no-header --no-summary
 
-.PHONY: pytest ## testing - Run pytest, with all tests in the dev environment
-pytest:
+.PHONY: dev-test ## testing - Run pytest, with all tests in the dev environment
+dev-test:
 	@echo "\033[1mRunning pytest:\033[0m\n\033[35m This will run the full test suite, but only once (in the dev environment)\033[0m"
-	@hatch run dev:all-tests -vv
-.PHONY: pytest-spark  ## testing - Run pytest (just spark) in the dev environment
-pytest-spark:
+	@hatch run dev:test -vv
+.PHONY: dev-test-spark  ## testing - Run pytest (just spark) in the dev environment
+dev-test-spark:
 	@echo "\033[1mRunning pytest for Spark tests:\033[0m\n\033[35m This will run the Spark test suite\033[0m"
 	@hatch run dev:spark-tests -vv
-.PHONY: pytest-non-spark  ## testing - Run pytest without spark in the dev environment
-pytest-non-spark:
+.PHONY: dev-test-non-spark  ## testing - Run pytest without spark in the dev environment
+dev-test-non-spark:
 	@echo "\033[1mRunning pytest for non-Spark tests:\033[0m\n\033[35m This will run the test suite, excluding all spark tests\033[0m"
 	@hatch run dev:non-spark-tests -vv