diff --git a/Makefile b/Makefile index 7d3cffc..05becfa 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ setup-glue-local: chmod +x automation/glue_setup.sh - source automation/glue_setup.sh + . automation/glue_setup.sh $(SOURCE_FILE_PATH) glue-demo-env: cp app/.custom_env .env @@ -19,7 +19,7 @@ lint: test: export KAGGLE_KEY=MOCKKEY export KAGGLE_USERNAME=MOCKUSERNAME - coverage run --source=app -m unittest discover -s tests + coverage run --source=app -m unittest discover -s tests coverage-report: coverage report diff --git a/README.md b/README.md index 5192842..83c03d9 100644 --- a/README.md +++ b/README.md @@ -23,10 +23,24 @@ Example: ```export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_261.jdk/ 2. For Glue-local-setup, run: ```bash - make setup-glue-local + # If default SHELL is zsh use + make setup-glue-local SOURCE_FILE_PATH=~/.zshrc + + # If default SHELL is bash use + make setup-glue-local SOURCE_FILE_PATH=~/.bashrc +``` + +3. Source SHELL profile using: + +```bash + # For zsh + source ~/.zshrc + + # For bash + source ~/.bashrc ``` -3. Install Dependencies: +4. Install Dependencies: ```bash make install ``` diff --git a/automation/glue_setup.sh b/automation/glue_setup.sh index b9604ad..85234ab 100755 --- a/automation/glue_setup.sh +++ b/automation/glue_setup.sh @@ -1,10 +1,15 @@ +# Parameter 1 --> Shell profile path +SOURCE_FILE=$1 +echo $SOURCE_FILE + echo -e "FIRST RUN TIME ESTIMATION: 30-45 MINS\nPlease do NOT exit" export PROJECT_ROOT=$(pwd) -# Doing all the work in separate folder "libs" -cd .. -mkdir libs && cd libs +# Doing all the work in separate folder "glue-libs" +cd ~ +mkdir glue-libs +cd glue-libs # Clone AWS Glue Python Lib git clone https://github.com/awslabs/aws-glue-libs.git @@ -22,19 +27,25 @@ tar -xvf spark-3.1.1-amzn-0-bin-3.2.1-amzn-3.tgz ln -s spark-3.1.1-amzn-0-bin-3.2.1-amzn-3 spark export SPARK_HOME=$(pwd)/spark - -# Export Paths +# Export Path export PATH=$PATH:$SPARK_HOME/bin:$MAVEN_HOME/bin:$AWS_GLUE_HOME/bin export PYTHONPATH=$PROJECT_ROOT # Download Glue ETL .jar files cd $AWS_GLUE_HOME chmod +x bin/glue-setup.sh -bin/glue-setup.sh +./bin/glue-setup.sh mvn install dependency:copy-dependencies cp $AWS_GLUE_HOME/jarsv1/AWSGlue*.jar $SPARK_HOME/jars/ cp $AWS_GLUE_HOME/jarsv1/aws*.jar $SPARK_HOME/jars/ -# Setup Done +echo "export AWS_GLUE_HOME=$AWS_GLUE_HOME +export MAVEN_HOME=$MAVEN_HOME +export SPARK_HOME=$SPARK_HOME +export PATH=$PATH:$SPARK_HOME/bin:$MAVEN_HOME/bin:$AWS_GLUE_HOME/bin +export PYTHONPATH=$PROJECT_ROOT" >> $SOURCE_FILE + + cd $PROJECT_ROOT + echo -e "\nGLUE LOCAL SETUP COMPLETE"