From 03ef42bae1bba709f9b9c3409b761109c3bb7761 Mon Sep 17 00:00:00 2001
From: Daigo Tanaka <daigo.tanaka@gmail.com>
Date: Thu, 6 Aug 2020 02:22:18 -0700
Subject: [PATCH] V0.2.0b0 (#28)

* bump up the version

* fix version format

* remove fgops

* fix handoff source search. fix dependency versions

* attempt to fix broken doc on readthedocs

* add a blankline after each include

* add shell notation at the code blocks

* add shell to begin quote

* fix more quotes
---
 .gitmodules                                   |  3 --
 .travis.yml                                   |  2 +-
 HISTORY.md                                    | 12 ++++-
 bin/cf_create_deploy_group                    |  1 -
 bin/cf_create_deploy_role                     |  1 -
 bin/cf_create_task                            |  1 -
 bin/cf_delete_deploy_group                    |  1 -
 bin/cf_delete_task                            |  1 -
 bin/cf_update_deploy_group                    |  1 -
 bin/cf_update_deploy_role                     |  1 -
 bin/cf_update_task                            |  1 -
 bin/docker_task                               |  1 -
 bin/ecs_run_task                              |  1 -
 bin/events_schedule_create                    |  1 -
 bin/events_schedule_remove                    |  1 -
 bin/iam_assume_role                           |  1 -
 bin/logs_pull_latest_stream                   |  1 -
 bin/mkdevenv                                  |  6 ---
 deploy/fargate                                |  1 -
 docs/01_run_local.md                          | 44 +++++++--------
 docs/02_exchange_rates.md                     | 19 +++----
 docs/03_set_up_aws_account.md                 |  2 +-
 docs/04_run_remote_config.md                  | 54 +++++++++----------
 docs/05_docker.md                             | 14 ++---
 docs/06_fargate.md                            | 20 +++----
 docs/07_schedule.md                           |  6 +--
 docs/08_cleanup.md                            | 51 ++++++------------
 docs/guided_tour.rst                          |  7 +++
 handoff/config.py                             |  2 +-
 handoff/services/container/docker/impl.py     |  6 +--
 .../container/docker/script/install_handoff   |  2 +
 .../aws_get_started/04_run_remote_config      | 15 +++---
 .../scripts/aws_get_started/05_docker         |  2 +-
 .../scripts/aws_get_started/06_fargate        |  8 +--
 .../scripts/aws_get_started/07_schedule       |  2 +-
 handoff/test_projects/scripts/funcs           | 17 +++---
 setup.py                                      | 18 +++----
 37 files changed, 154 insertions(+), 173 deletions(-)
 delete mode 100644 .gitmodules
 delete mode 120000 bin/cf_create_deploy_group
 delete mode 120000 bin/cf_create_deploy_role
 delete mode 120000 bin/cf_create_task
 delete mode 120000 bin/cf_delete_deploy_group
 delete mode 120000 bin/cf_delete_task
 delete mode 120000 bin/cf_update_deploy_group
 delete mode 120000 bin/cf_update_deploy_role
 delete mode 120000 bin/cf_update_task
 delete mode 120000 bin/docker_task
 delete mode 120000 bin/ecs_run_task
 delete mode 120000 bin/events_schedule_create
 delete mode 120000 bin/events_schedule_remove
 delete mode 120000 bin/iam_assume_role
 delete mode 120000 bin/logs_pull_latest_stream
 delete mode 100755 bin/mkdevenv
 delete mode 160000 deploy/fargate

diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 360af17..0000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "deploy/fargate"]
-	path = deploy/fargate
-    url = https://github.com/anelendata/fgops.git
diff --git a/.travis.yml b/.travis.yml
index a0edef0..ae3edaa 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,4 +24,4 @@ script:
   - pytest -s tests/unit
   - pytest -s tests/local_run_no_install
   - pytest -s tests/local_run_install
-  - tests/install_test.sh
+    #  - tests/install_test.sh
diff --git a/HISTORY.md b/HISTORY.md
index 19a079d..a9a04d5 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,7 +1,17 @@
 ## History
 
+### 0.2.0b0 (2020-08-05)
+
+- Beta
+- Covers entire workflow from the local test to Fargate deployment
+- Clean extensible code-design (services and plugins)
+- Unit, integration, and installation tests
+- Interactive tutorial
+- Near-stable CLI version
+
 ### 0.1.2-alpha (2020-07-17)
 
+- Alpha
 - fix docker files to remove project dir
 - fix the issue of env var not passed to subprocesses
 
@@ -11,4 +21,4 @@ Just added a long description to setup.py for the pypi release.
 
 ### 0.1.0-alpha (2020-07-16)
 
-The first package build.
+Pilot. The first package build.
diff --git a/bin/cf_create_deploy_group b/bin/cf_create_deploy_group
deleted file mode 120000
index a4c295a..0000000
--- a/bin/cf_create_deploy_group
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/cf_create_deploy_group
\ No newline at end of file
diff --git a/bin/cf_create_deploy_role b/bin/cf_create_deploy_role
deleted file mode 120000
index 21da57d..0000000
--- a/bin/cf_create_deploy_role
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/cf_create_deploy_role
\ No newline at end of file
diff --git a/bin/cf_create_task b/bin/cf_create_task
deleted file mode 120000
index 62b15bf..0000000
--- a/bin/cf_create_task
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/cf_create_task
\ No newline at end of file
diff --git a/bin/cf_delete_deploy_group b/bin/cf_delete_deploy_group
deleted file mode 120000
index 3f4de40..0000000
--- a/bin/cf_delete_deploy_group
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/cf_delete_deploy_group
\ No newline at end of file
diff --git a/bin/cf_delete_task b/bin/cf_delete_task
deleted file mode 120000
index cacd5bc..0000000
--- a/bin/cf_delete_task
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/cf_delete_task
\ No newline at end of file
diff --git a/bin/cf_update_deploy_group b/bin/cf_update_deploy_group
deleted file mode 120000
index c8aacbb..0000000
--- a/bin/cf_update_deploy_group
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/cf_update_deploy_group
\ No newline at end of file
diff --git a/bin/cf_update_deploy_role b/bin/cf_update_deploy_role
deleted file mode 120000
index 12fe10a..0000000
--- a/bin/cf_update_deploy_role
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/cf_update_deploy_role
\ No newline at end of file
diff --git a/bin/cf_update_task b/bin/cf_update_task
deleted file mode 120000
index 7829faf..0000000
--- a/bin/cf_update_task
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/cf_update_task
\ No newline at end of file
diff --git a/bin/docker_task b/bin/docker_task
deleted file mode 120000
index a5f2c9a..0000000
--- a/bin/docker_task
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/docker_task
\ No newline at end of file
diff --git a/bin/ecs_run_task b/bin/ecs_run_task
deleted file mode 120000
index 284c6b9..0000000
--- a/bin/ecs_run_task
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/ecs_run_task
\ No newline at end of file
diff --git a/bin/events_schedule_create b/bin/events_schedule_create
deleted file mode 120000
index 73b436d..0000000
--- a/bin/events_schedule_create
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/events_schedule_create
\ No newline at end of file
diff --git a/bin/events_schedule_remove b/bin/events_schedule_remove
deleted file mode 120000
index 1f2545a..0000000
--- a/bin/events_schedule_remove
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/events_schedule_remove
\ No newline at end of file
diff --git a/bin/iam_assume_role b/bin/iam_assume_role
deleted file mode 120000
index 14bb150..0000000
--- a/bin/iam_assume_role
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/iam_assume_role
\ No newline at end of file
diff --git a/bin/logs_pull_latest_stream b/bin/logs_pull_latest_stream
deleted file mode 120000
index 46625a3..0000000
--- a/bin/logs_pull_latest_stream
+++ /dev/null
@@ -1 +0,0 @@
-../deploy/fargate/bin/logs_pull_latest_stream
\ No newline at end of file
diff --git a/bin/mkdevenv b/bin/mkdevenv
deleted file mode 100755
index 7c485e4..0000000
--- a/bin/mkdevenv
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-python3 -m venv ./venv
-/bin/bash -c "source ./venv/bin/activate && pip install wheel && deactivate"
-/bin/bash -c "source ./venv/bin/activate && pip install -r requirements.txt && pip install -r requirements_dev.txt && deactivate"
-/bin/bash -c "source ./venv/bin/activate && pip install -r docs/requirements.txt && deactivate"
diff --git a/deploy/fargate b/deploy/fargate
deleted file mode 160000
index 5315917..0000000
--- a/deploy/fargate
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 53159175131060bcdff1ef14ad0b0a1c9ecc0871
diff --git a/docs/01_run_local.md b/docs/01_run_local.md
index 12f5d89..8ace788 100644
--- a/docs/01_run_local.md
+++ b/docs/01_run_local.md
@@ -7,10 +7,10 @@ using project 01_word_count.
 
 Each project directory contains:
 
-```
+```shell
 > ls -l 01_word_count
 ```
-```
+```shell
 
  files
  project.yml
@@ -19,11 +19,11 @@ Each project directory contains:
 
 project.yml looks like:
 
-```
+```shell
 > cat 01_word_count/project.yml
 ```
 
-```
+```shell
 commands:
   - command: cat
     args: "./files/the_great_dictator_speech.txt"
@@ -43,16 +43,16 @@ Here,
 
 The example from 01_word_count runs a command line equivalent of:
 
-```
+```shell
 cat ./files/the_great_dictator_speech.txt | wc -w
 ```
 
 Now let's run. Try entering this command below:
 
-```
+```shell
 > handoff --project 01_word_count --workspace workspace run local
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:35:12,691 - handoff.config - Reading configurations from 01_word_count/project.yml
 INFO - 2020-08-06 03:35:12,693 - handoff.config - Setting environment variables from config.
@@ -69,7 +69,7 @@ INFO - 2020-08-06 03:35:13,130 - handoff.config - Job ended at 2020-08-06 03:35:
 
 If you see the output that looks like:
 
-```
+```shell
 
  INFO - 2020-08-03 04:51:01,971 - handoff.config - Reading configurations from 01_word_count/project.yml
  ...
@@ -81,10 +81,10 @@ If you see the output that looks like:
 Then great! You just ran the first local test. It created a workspace
 directory that looks like:
 
-```
+```shell
 > ls -l workspace
 ```
-```
+```shell
 
  artifacts
  config
@@ -93,11 +93,11 @@ directory that looks like:
 
 And the word count is stored at workspace/artifacts/state. Here is the content:
 
-```
+```shell
 > cat  workspace/artifacts/state
 ```
 
-```
+```shell
 644
 ```
 
@@ -112,7 +112,7 @@ Check out on YouTube: https://www.youtube.com/watch?v=J7GY1Xg6X20
 
 And here is the first few paragraphs of the text:
 
-```
+```shell
 
 I’m sorry, but I don’t want to be an emperor. That’s not my business. I don’t want to rule or conquer anyone. I should like to help everyone - if possible - Jew, Gentile - black man - white. We all want to help one another. Human beings are like that. We want to live by each other’s happiness - not by each other’s misery. We don’t want to hate and despise one another. In this world there is room for everyone. And the good earth is rich and can provide for everyone. The way of life can be free and beautiful, but we have lost the way.
 
@@ -123,11 +123,11 @@ Greed has poisoned men’s souls, has barricaded the world with hate, has goose-
 
 Now to the second example. This time project.yml looks like:
 
-```
+```shell
 > cat 02_collect_stats/project.yml
 ```
 
-```
+```shell
 commands:
   - command: cat
     args: ./files/the_great_dictator_speech.txt
@@ -139,7 +139,7 @@ commands:
 
 ...which is shell equivalent to
 
-```
+```shell
 cat ./files/the_great_dictator_speech.txt | python ./files/stats_collector.py | wc -w
 ```
 
@@ -147,11 +147,11 @@ The script for the second command stats_collector.py can be found in
 02_collect_stats/files directory and it is a Python script that looks like:
 
 
-```
+```shell
 > cat 02_collect_stats/files/stats_collector.py
 ```
 
-```
+```python
 #!/usr/bin/python
 import io, json, logging, sys, os
 
@@ -188,10 +188,10 @@ The raw text is then processed by the third command (wc -w) and it conts the num
 
 Now let's run. Try entering this command below:
 
-```
+```shell
 > handoff --project 02_collect_stats --workspace workspace run local
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:35:13,401 - handoff.config - Reading configurations from 02_collect_stats/project.yml
 INFO - 2020-08-06 03:35:13,402 - handoff.config - Setting environment variables from config.
@@ -209,11 +209,11 @@ INFO - 2020-08-06 03:35:13,881 - handoff.config - Processed in 0:00:00.049824
 Let's check out the contents of the second command:
 
 
-```
+```shell
 > cat workspace/artifacts/collect_stats.json
 ```
 
-```
+```shell
 {"rows_read": 15}
 ```
 
diff --git a/docs/02_exchange_rates.md b/docs/02_exchange_rates.md
index 2c0c03f..214072b 100644
--- a/docs/02_exchange_rates.md
+++ b/docs/02_exchange_rates.md
@@ -10,11 +10,11 @@ in Python vitual environment.
 
 We will use 03_exchange_rates project. project.yml looks like:
 
-```
+```shell
 > cat 03_exchange_rates/project.yml
 ```
 
-```
+```shell
 commands:
   - command: "tap-exchangeratesapi"
     args: "--config config/tap-config.json"
@@ -40,8 +40,9 @@ deploy:
 
 ...which is shell equivalent to
 
+```shell
     tap-exchangeratesapi | python files/stats_collector.py | target-csv
-
+```
 
 
 Before we can run this, we need to install tap-exchangeratesapi and target-csv.
@@ -53,10 +54,10 @@ commands.
 
 To install everything, run this command:
 
-```
+```shell
 > handoff -p 03_exchange_rates -w workspace_03 workspace install
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:35:14,158 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:35:14,240 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -85,10 +86,10 @@ Successfully installed jsonschema-2.6.0 pendulum-1.2.0 python-dateutil-2.8.1 pyt
 
 Now let's run the task. Try entering this command below:
 
-```
+```shell
 > handoff -p 03_exchange_rates -w workspace_03 run local
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:35:29,258 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:35:29,339 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -109,14 +110,14 @@ INFO - 2020-08-06 03:35:33,964 - handoff.config - Processed in 0:00:04.268474
 
 This process should have created a CSV file in artifacts directory:
 
-```
+```shell
 
 exchange_rate-20200806T033530.csv
 ```
 
 ...which looks like:
 
-```
+```shell
 
 CAD,HKD,ISK,PHP,DKK,HUF,CZK,GBP,RON,SEK,IDR,INR,BRL,RUB,HRK,JPY,THB,CHF,EUR,MYR,BGN,TRY,CNY,NOK,NZD,ZAR,USD,MXN,SGD,AUD,ILS,KRW,PLN,date
 0.0127290837,0.0725398406,1.3197211155,0.4630976096,0.0618218792,2.9357569721,0.2215388446,0.007434429,0.0401958831,0.0863047809,135.1005146082,0.7041915671,0.050374336,0.6657569721,0.0625373506,1.0,0.29312749,0.0088188911,0.0083001328,0.0399311089,0.0162333997,0.0642571381,0.0655312085,0.0889467131,0.0142670983,0.158440405,0.0093592297,0.2132744024,0.0130336985,0.0134852258,0.032375498,11.244189907,0.0371372842,2020-07-10T00:00:00Z
diff --git a/docs/03_set_up_aws_account.md b/docs/03_set_up_aws_account.md
index 3aaab92..fb304cf 100644
--- a/docs/03_set_up_aws_account.md
+++ b/docs/03_set_up_aws_account.md
@@ -72,7 +72,7 @@ Enter Access key ID and Secret access key you created in the last step.
 For region, use one of these keys (for USA users, us-east-1 would do):
 
 
-```
+```shell
 
 ap-northeast-1
 ap-northeast-2
diff --git a/docs/04_run_remote_config.md b/docs/04_run_remote_config.md
index d710947..96d7e58 100644
--- a/docs/04_run_remote_config.md
+++ b/docs/04_run_remote_config.md
@@ -7,10 +7,10 @@ store the configurations to the remote data store.
 
 Let's review how 03_exchange_rates project directory is structured:
 
-```
+```shell
 > ls -l 03_exchange_rates
 ```
-```
+```shell
 
  config
  files
@@ -20,10 +20,10 @@ Let's review how 03_exchange_rates project directory is structured:
 
 In the config directory, we have a couple of configuration files:
 
-```
+```shell
 > ls -l 03_exchange_rates/config
 ```
-```
+```shell
 
  tap-config.json
  target-config.json
@@ -32,18 +32,18 @@ In the config directory, we have a couple of configuration files:
 
 ...which look like:
 
-```
+```shell
 > cat 03_exchange_rates/config/tap-config.json
 ```
 
-```
+```shell
 { "base": "JPY", "start_date": "2020-07-10" }
 ```
-```
+```shell
 > cat 03_exchange_rates/config/target-config.json
 ```
 
-```
+```shell
 {
     "delimiter": ",",
     "quotechar": "'",
@@ -60,10 +60,10 @@ key-value storage.
 
 Try running:
 
-```
+```shell
 > handoff -p 03_exchange_rates config push
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:35:35,362 - handoff.config - Compiling config from 03_exchange_rates
 INFO - 2020-08-06 03:35:35,362 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
@@ -79,7 +79,7 @@ INFO - 2020-08-06 03:35:36,184 - handoff.config - See the parameters at https://
 
 Look at the end of the log that says,
 
-```
+```shell
     See the parameters at https://console.aws.amazon.com/systems-manager/parameters/?region=...
 ```
 
@@ -90,10 +90,10 @@ the parameters are uploaded.
 
 We also have some files needed for the task execution:
 
-```
+```shell
 > ls -l 03_exchange_rates/files
 ```
-```
+```shell
 
  stats_collector.py
 ```
@@ -103,10 +103,10 @@ So we will create a cloud data storage (S3 bucket) for that.
 
 Try running:
 
-```
+```shell
 > handoff -p 03_exchange_rates cloud create_bucket
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:35:36,484 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:35:36,567 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -127,7 +127,7 @@ INFO - 2020-08-06 03:35:37,815 - handoff.config - Check the progress at https://
 
 Wait for a minute and check here
 
-```
+```shell
     https://s3.console.aws.amazon.com
 ```
 
@@ -139,10 +139,10 @@ Now it's time to push the files to the bucket.
 
 Try running:
 
-```
+```shell
 > handoff -p 03_exchange_rates files push
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:38:38,087 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:38:38,168 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -163,7 +163,7 @@ INFO - 2020-08-06 03:38:39,279 - handoff.config - See the files at https://s3.co
 
 Look at the end of the log that says,
 
-```
+```shell
     See the files at https://s3.console.aws.amazon.com/s3/...
 ```
 
@@ -174,10 +174,10 @@ directory is uploaded.
 
 Install the workspace:
 
-```
+```shell
 > handoff -p 03_exchange_rates -w workspace workspace install
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:38:39,718 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:38:39,799 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -208,10 +208,10 @@ Now let's run the command by pulling the configurations and files from remote.
 
 Try running:
 
-```
+```shell
 > handoff -p 03_exchange_rates -w workspace run remote_config --push-artifacts
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:38:53,883 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:38:53,963 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -241,8 +241,8 @@ INFO - 2020-08-06 03:38:58,988 - handoff.services.cloud.aws.s3 - Copied s3://xxx
 Notice that we used --push-artifacts option in the last command.
 With this option, we pushed the result to the bucket under
 
-```
-    /last/artifacts
+```shell
+    test-03-exchange-rates/last/artifacts
 
 ```
 
@@ -251,10 +251,10 @@ directory.
 
 Also note that artifacts are automatically archived at each run at
 
-```
+```shell
 
 
-    //runs/
+    test-03-exchange-rates/runs/
 
 
 ```
diff --git a/docs/05_docker.md b/docs/05_docker.md
index 05eee6d..0635f2e 100644
--- a/docs/05_docker.md
+++ b/docs/05_docker.md
@@ -10,10 +10,10 @@ Let's build a Docker image.
 Try running the following command. Enter y when prompted at the beginning.
 The build may take 5~10 minutes.
 
-```
+```shell
 > handoff -p 03_exchange_rates container build
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:38:59,397 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:38:59,478 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -41,10 +41,10 @@ INFO - 2020-08-06 03:41:41,652 - handoff.config - Successfully tagged singer_exc
 
 Now let's run the code in the Docker container.
 
-```
+```shell
 > handoff -p 03_exchange_rates container run
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:41:45,336 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:41:45,509 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -71,7 +71,7 @@ INFO - 2020-08-06 03:41:55,342 - handoff.services.cloud.aws.s3 - Copied s3://xxx
 ```
 
 Confirm the run by checking the logs. Also check the artifacts on S3:
-```
+```shell
 
 
     xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/
@@ -86,10 +86,10 @@ directory.
 Now that we know the Docker container runs fine, let's push it to
 AWS Elastic Container Registry. This may take a few minutes.
 
-```
+```shell
 > handoff -p 03_exchange_rates container push
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:41:57,295 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:41:57,378 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
diff --git a/docs/06_fargate.md b/docs/06_fargate.md
index 79b2b64..9a71252 100644
--- a/docs/06_fargate.md
+++ b/docs/06_fargate.md
@@ -11,10 +11,10 @@ We will first need to create a cluster.
 
 To make this process easy, handoff packed everything up in a command:
 
-```
+```shell
 > handoff -p 03_exchange_rates cloud create_resources
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:44:42,787 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:44:42,867 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -36,7 +36,7 @@ INFO - 2020-08-06 03:44:45,270 - handoff.config - Check the progress at https://
 
 At the end of the log, you should see a line like:
 
-```
+```shell
 
     Check the progress at https://console.aws.amazon.com/cloudformation/home?region=xxxx
 
@@ -45,16 +45,16 @@ At the end of the log, you should see a line like:
 Grab the entire link and open in a browser (you need to login in to AWS) to see
 the progress of the resource creation.
 
-Wait until it says \"CREATE_COMPLETE\"
+Wait until it says "CREATE_COMPLETE"
 
 
 
 Now it's time to deploy the task and here is the command:
 
-```
+```shell
 > handoff -p 03_exchange_rates cloud create_task
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:47:45,531 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:47:45,610 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -77,7 +77,7 @@ INFO - 2020-08-06 03:47:47,363 - handoff.config - Check the progress at https://
 
 Here again, at the end of the log, you should see a line like:
 
-```
+```shell
 
     Check the progress at https://console.aws.amazon.com/cloudformation/home?region=xxxx
 
@@ -92,10 +92,10 @@ Wait until it says "CREATE_COMPLETE"
 Once the task is created, try running on Fargate.
 To do so, run this command:
 
-```
+```shell
 > handoff -p 03_exchange_rates cloud run
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:50:47,634 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:50:47,713 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -117,7 +117,7 @@ INFO - 2020-08-06 03:50:50,202 - handoff.config - Check the task at https://us-e
 
 At the end of the log, you should see a line like:
 
-```
+```shell
 
     Check the task at https://us-east-1.console.aws.amazon.com/ecs/home?region=xxxx
 
diff --git a/docs/07_schedule.md b/docs/07_schedule.md
index 917b989..b0f73e6 100644
--- a/docs/07_schedule.md
+++ b/docs/07_schedule.md
@@ -9,10 +9,10 @@ To schedule a task, use schedule command with target_id and
 We pass those values to handoff with `--data` (`-d` for short) option:
 
 
-```
+```shell
 > handoff -p 03_exchange_rates cloud schedule --data '{"target_id": "1", "cron": "55 03 * * ? *"}'
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:50:50,489 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:50:50,570 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -36,7 +36,7 @@ INFO - 2020-08-06 03:50:52,745 - handoff.config - Check the status at https://co
 
 At the end of the log, you should see a line like:
 
-```
+```shell
 
     Check the progress at Check the status at https://console.aws.amazon.com/ecs/home?region=us-east-1#/clusters/...
 
diff --git a/docs/08_cleanup.md b/docs/08_cleanup.md
index 9fefe07..7551b33 100644
--- a/docs/08_cleanup.md
+++ b/docs/08_cleanup.md
@@ -4,37 +4,28 @@ Let's clean everything up so we won't pay a fraction of penny after forgeting ab
 
 First unschedule the task:
 
-```
+```shell
 > handoff -l warning -p 03_exchange_rates cloud unschedule -d '{"target_id": 1}'
-```
-```
-
 ```
 
 Then delete the task:
 
-```
+```shell
 > handoff -l warning -p 03_exchange_rates cloud delete_task
-```
-```
-
 ```
 
 If there is no other task in the same resource group, we can delete it:
 
-```
+```shell
 > handoff -l warning -p 03_exchange_rates cloud delete_resources
-```
-```
-
 ```
 
 Here is how to delete the configurations from SSM Parameter Store:
 
-```
+```shell
 > handoff -p 03_exchange_rates config delete
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:34:46,049 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:34:46,131 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -47,10 +38,10 @@ INFO - 2020-08-06 03:34:46,496 - botocore.credentials - Found credentials in sha
 
 Here is how to delete the files from S3 bucket:
 
-```
+```shell
 > handoff -p 03_exchange_rates files delete
 ```
-```
+```shell
 
 INFO - 2020-08-06 03:34:47,114 - handoff.config - Reading configurations from 03_exchange_rates/project.yml
 INFO - 2020-08-06 03:34:47,195 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials
@@ -64,10 +55,10 @@ INFO - 2020-08-06 03:34:47,602 - handoff.services.cloud.aws.s3 - GET s3://xxxxxx
 
 If there is no other task in the same resource group, we can delete the bucket, too:
 
-```
+```shell
 > handoff -l warning -p 03_exchange_rates cloud delete_bucket
 ```
-```
+```shell
 
 WARNING - 2020-08-06 03:34:49,044 - handoff.config - This will only delete the CloudFormation stack. The bucket xxxxxxxxxxxx-handoff-test will be retained.
 ```
@@ -75,37 +66,27 @@ WARNING - 2020-08-06 03:34:49,044 - handoff.config - This will only delete the C
 The previous command only deleted the CloudFormation stack, but not the bucket itself.
 Here is how to delete all the files in s3://xxxxxxxxxxxx-handoff-test bucket. This cannot be reversed:
 
-```
+```shell
 > aws s3 rm --recursive s3://xxxxxxxxxxxx-handoff-test/
-```
-```
-
 ```
 
 Here is how to delete s3://xxxxxxxxxxxx-handoff-test bucket. The bucket must be empty. This cannot be reversed:
 
-```
+```shell
 > aws s3 rb s3://xxxxxxxxxxxx-handoff-test
-```
-```
-
 ```
 
 Now delete singer_exchange_rates_to_csv repository from ECR. This cannot be reversed.
 --force option will ignore that we still have images in the repository.
 
-```
+```shell
 > aws ecr delete-repository --repository-name singer_exchange_rates_to_csv --force
 ```
-```
 
-```
-```
-> handoff -l warning -p 03_exchange_rates cloud delete_role
-```
-```
+If you had created a role and want to delete it, do:
 
+```shell
+> handoff -l warning -p 03_exchange_rates cloud delete_role
 ```
 
-That's all.
-
+That's all!
diff --git a/docs/guided_tour.rst b/docs/guided_tour.rst
index 778034e..50fb8e6 100644
--- a/docs/guided_tour.rst
+++ b/docs/guided_tour.rst
@@ -16,10 +16,17 @@ You should be able to finish it in 30 minutes to an hour.
 Below is the web-version of the tutorial.
 
 .. mdinclude:: ./01_run_local.md
+
 .. mdinclude:: ./02_exchange_rates.md
+
 .. mdinclude:: ./03_set_up_aws_account.md
+
 .. mdinclude:: ./04_run_remote_config.md
+
 .. mdinclude:: ./05_docker.md
+
 .. mdinclude:: ./06_fargate.md
+
 .. mdinclude:: ./07_schedule.md
+
 .. mdinclude:: ./08_cleanup.md
diff --git a/handoff/config.py b/handoff/config.py
index 30635d4..3b485da 100644
--- a/handoff/config.py
+++ b/handoff/config.py
@@ -1,4 +1,4 @@
-VERSION = "0.1.2a0"
+VERSION = "0.2.0b0"
 
 import os, re
 from handoff import utils
diff --git a/handoff/services/container/docker/impl.py b/handoff/services/container/docker/impl.py
index a5c08cb..a82377c 100644
--- a/handoff/services/container/docker/impl.py
+++ b/handoff/services/container/docker/impl.py
@@ -67,9 +67,9 @@ def build(project_dir, new_version=None, docker_file=None, nocache=False):
 
     with tempfile.TemporaryDirectory() as build_dir:
         cwd, _ = os.path.split(__file__)
-        pos = cwd.find("handoff")
-        if pos >= 0:
-            handoff_dir = os.path.join(cwd[:pos], "handoff")
+        handoff_dir = os.path.join(cwd, "../../../../../")
+        if os.path.isfile(os.path.join(handoff_dir, "setup.py")):
+            logger.info("Found handoff. Copying to the build directory")
             shutil.copytree(handoff_dir, os.path.join(build_dir, "handoff"))
 
         shutil.copytree(project_dir, os.path.join(build_dir, "project"),
diff --git a/handoff/services/container/docker/script/install_handoff b/handoff/services/container/docker/script/install_handoff
index 3f09980..b27772a 100755
--- a/handoff/services/container/docker/script/install_handoff
+++ b/handoff/services/container/docker/script/install_handoff
@@ -1,7 +1,9 @@
 #!/bin/bash
 
 if [ -e ./handoff ]; then
+    echo Found a local copy of handoff
     pip3 install --no-cache -e ./handoff
 else
+    echo Installing handoff from pypi
     pip3 install handoff
 fi
diff --git a/handoff/test_projects/scripts/aws_get_started/04_run_remote_config b/handoff/test_projects/scripts/aws_get_started/04_run_remote_config
index 60af0fc..5cf7ad0 100755
--- a/handoff/test_projects/scripts/aws_get_started/04_run_remote_config
+++ b/handoff/test_projects/scripts/aws_get_started/04_run_remote_config
@@ -55,7 +55,7 @@ Command "handoff -p 03_exchange_rates config push"
 echo '
 Look at the end of the log that says,
 
-```
+```shell
     See the parameters at https://console.aws.amazon.com/systems-manager/parameters/?region=...
 ```
 
@@ -83,7 +83,7 @@ Command "handoff -p 03_exchange_rates cloud create_bucket"
 echo '
 Wait for a minute and check here
 
-```
+```shell
     https://s3.console.aws.amazon.com
 ```
 
@@ -105,7 +105,7 @@ Command "handoff -p 03_exchange_rates files push"
 echo '
 Look at the end of the log that says,
 
-```
+```shell
     See the files at https://s3.console.aws.amazon.com/s3/...
 ```
 
@@ -132,7 +132,10 @@ echo '
 Notice that we used --push-artifacts option in the last command.
 With this option, we pushed the result to the bucket under
 
-```'
+
+```shell'
+
+TASK=`handoff -p 03_exchange_rates envs get -d '{"key":"HO_TASK"}' -l critical`
 
 echo "    $TASK/last/artifacts"
 
@@ -145,11 +148,11 @@ directory.
 echo '
 Also note that artifacts are automatically archived at each run at
 
-```
+```shell
 '
 
 echo "
-    $BUCKET/$TASK/runs/
+    $TASK/runs/
 "
 
 echo '
diff --git a/handoff/test_projects/scripts/aws_get_started/05_docker b/handoff/test_projects/scripts/aws_get_started/05_docker
index a5607bd..ac8f7e2 100755
--- a/handoff/test_projects/scripts/aws_get_started/05_docker
+++ b/handoff/test_projects/scripts/aws_get_started/05_docker
@@ -52,7 +52,7 @@ TASK=`handoff -p 03_exchange_rates envs get -d '{"key":"HO_TASK"}' -l critical`
 
 echo '
 Confirm the run by checking the logs. Also check the artifacts on S3:
-```
+```shell
 '
 
 echo "
diff --git a/handoff/test_projects/scripts/aws_get_started/06_fargate b/handoff/test_projects/scripts/aws_get_started/06_fargate
index 6f66540..db4ef26 100755
--- a/handoff/test_projects/scripts/aws_get_started/06_fargate
+++ b/handoff/test_projects/scripts/aws_get_started/06_fargate
@@ -32,7 +32,7 @@ echo '
 
 At the end of the log, you should see a line like:
 
-```
+```shell
 
     Check the progress at https://console.aws.amazon.com/cloudformation/home?region=xxxx
 
@@ -41,7 +41,7 @@ At the end of the log, you should see a line like:
 Grab the entire link and open in a browser (you need to login in to AWS) to see
 the progress of the resource creation.
 
-Wait until it says \"CREATE_COMPLETE\"
+Wait until it says "CREATE_COMPLETE"
 '
 
 if [[ $MODE = "auto" ]]; then sleep 3m; fi
@@ -58,7 +58,7 @@ echo '
 
 Here again, at the end of the log, you should see a line like:
 
-```
+```shell
 
     Check the progress at https://console.aws.amazon.com/cloudformation/home?region=xxxx
 
@@ -83,7 +83,7 @@ Command "handoff -p 03_exchange_rates cloud run"
 echo '
 At the end of the log, you should see a line like:
 
-```
+```shell
 
     Check the task at https://us-east-1.console.aws.amazon.com/ecs/home?region=xxxx
 
diff --git a/handoff/test_projects/scripts/aws_get_started/07_schedule b/handoff/test_projects/scripts/aws_get_started/07_schedule
index 1312e0d..3e3ad33 100755
--- a/handoff/test_projects/scripts/aws_get_started/07_schedule
+++ b/handoff/test_projects/scripts/aws_get_started/07_schedule
@@ -34,7 +34,7 @@ echo '
 
 At the end of the log, you should see a line like:
 
-```
+```shell
 
     Check the progress at Check the status at https://console.aws.amazon.com/ecs/home?region=us-east-1#/clusters/...
 
diff --git a/handoff/test_projects/scripts/funcs b/handoff/test_projects/scripts/funcs
index 0129617..f2d0850 100755
--- a/handoff/test_projects/scripts/funcs
+++ b/handoff/test_projects/scripts/funcs
@@ -8,16 +8,17 @@ export YELLOW="\033[33m"
 export BLINK="\033[5m"
 export DIM="\033[2m"
 
+BEGINQUOTE='```shell'
 QUOTE='```'
 PROMPT_STYLE="$GREEN $BLINK"
 PROMPT_CHAR="> "
 
 ListDir() {
 if [[ -z $2 ]]; then STYLE=$DIM; else STYLE=$2; fi
-if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; fi
+if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $BEGINQUOTE; fi
 echo "> ls -l $1"
 if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi
-if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; fi
+if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $BEGINQUOTE; fi
 echo
 for i in `ls $1`; do echo " $i"; done
 if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi
@@ -25,11 +26,11 @@ if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi
 
 Cat() {
 if [[ -z $2 ]]; then STYLE=$DIM; else STYLE=$2; fi
-if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; fi
+if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $BEGINQUOTE; fi
 echo "> cat $1"
 if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi
 echo
-if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; fi
+if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $BEGINQUOTE; fi
 cat $1
 if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi
 }
@@ -37,7 +38,7 @@ if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi
 Run() {
 if [[ -z $2 ]]; then STYLE=$DIM; else STYLE=$2; fi
 if [[ ! -z $3 ]];then h=$3; else if [[ ! -z $HEADTAIL_LIMIT ]]; then h=$HEADTAIL_LIMIT; else h=0; fi; fi
-if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; echo; fi
+if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $BEGINQUOTE; echo; fi
 if [[ $h -lt 1 ]]; then eval "$1"; else eval "$1" | (sed -u ${h}q; t=`tail -n $h`; if [[ `echo "$t" | wc -l` -gt 1 ]]; then echo .; echo .; echo .; echo "$t"; fi); fi
 if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi
 }
@@ -59,16 +60,16 @@ if [[ $REPLY = "quit" ]]; then
   exit 0
 fi
 if [[ $MODE = "auto" || $REPLY = $1 || $REPLY = "lazy" ]]; then
-  echo $QUOTE
+  if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $BEGINQUOTE; fi
   echo "> $1"
-  echo $QUOTE
+  if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi
   Run "$1" "$2" "$3"
 fi
 }
 
 Code() {
 if [[ -z $2 ]]; then STYLE=$DIM; else STYLE=$2; fi
-if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; fi
+if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $BEGINQUOTE; fi
 echo -e $1
 if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi
 }
diff --git a/setup.py b/setup.py
index 6445268..8f53402 100755
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 from setuptools import setup
 
-VERSION="0.1.2-alpha"
+VERSION="0.2.0b0"
 
 with open("README.md", "r") as fh:
     long_description = fh.read()
@@ -16,7 +16,7 @@
     author="Daigo Tanaka, Anelen Co., LLC",
     url="http://dev.handoff.cloud",
     classifiers=[
-        "Development Status :: 3 - Alpha",
+        "Development Status :: 4 - Beta",
         "License :: OSI Approved :: Apache Software License",
 
         "Operating System :: MacOS :: MacOS X",
@@ -27,15 +27,15 @@
         "Programming Language :: Python :: 3.8",
     ],
     install_requires=[
-        "attr==0.3.1",
-        "boto3==1.10.0",
-        "botocore==1.13.0",
-        "docker==4.2.2",
+        "attr>=0.3.1",
+        "boto3>=1.10.0",
+        "botocore>=1.13.0",
+        "docker>=4.0.0",
         "lxml>=4.2.0",
-        "python-dateutil==2.8.0",
-        "pyyaml==5.3.1",
+        "python-dateutil>=2.8.0",
+        "pyyaml>=5.1",
         "requests>=2.19.0",
-        "s3transfer==0.2.1",
+        "s3transfer>=0.2.0",
     ],
     entry_points="""
     [console_scripts]