Merge pull request #9 from joseguerr/minor-SW10-add-dockerfile

Minor sw10 add dockerfile

Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.11-slim
+
+EXPOSE 4040
+
+RUN echo "Installing Packages" && \
+    apt-get update && \
+    apt-get install -y default-jre wget tar make && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN echo "Installing Spark" && \
+    cd /opt && \
+    wget -q https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz && \
+    tar -xf spark-3.5.2-bin-hadoop3.tgz && \
+    rm -f spark-3.5.2-bin-hadoop3.tgz  && \
+    pip install -e /opt/spark-3.5.2-bin-hadoop3/python
+
+ENV HADOOP_CONF_DIR=/opt/spark-3.5.2-bin-hadoop3/conf
+
+COPY . .
+
+RUN make clean && \
+    make setup && \
+    make build
+
+CMD ["/bin/bash"]

Makefile

@@ -24,6 +24,14 @@ build: # Build and package the application and its dependencies to be used throu
 	cp spark_web_events_etl/main.py app_config.yaml spark_web_events_etl/tasks/*/dq_checks_*.yaml deps
 	poetry run python -m zipfile -c deps/libs.zip libs/*
 
+.PHONY: docker-build
+docker-build: # Build the application in a docker container.
+	docker build -t jg-data-engineer-test .
+
+.PHONY: docker-run
+docker-run: # Run the application in a docker container.
+	docker run -it jg-data-engineer-test
+
 .PHONY: run-local
 run-local: # Run a task locally (example: make run-local task=standardise execution-date=2023-04-12).
 	poetry run spark-submit \

README.md

@@ -30,7 +30,7 @@ Configuration is defined in [app_config.yaml](app_config.yaml) and managed by th
 ## Pre-requisites
 - OpenJDK@17
 - [email protected]
-- [email protected].0
+- [email protected].2
 - [email protected]
 
 ## Execution instructions

pyproject.toml

@@ -11,6 +11,7 @@ pyspark = "3.5.2"
 dynaconf = "3.2.1"
 delta-spark = "3.2"
 soda-core-spark-df = "^3.3.5"
+venv-pack = "^0.2.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.2.2"