diff --git a/docs/01_run_local.md b/docs/01_run_local.md new file mode 100644 index 0000000..12f5d89 --- /dev/null +++ b/docs/01_run_local.md @@ -0,0 +1,224 @@ +## Running a task locally + +In this section, we will learn how to run a task with handoff locally, +using project 01_word_count. + + + +Each project directory contains: + +``` +> ls -l 01_word_count +``` +``` + + files + project.yml +``` + + +project.yml looks like: + +``` +> cat 01_word_count/project.yml +``` + +``` +commands: + - command: cat + args: "./files/the_great_dictator_speech.txt" + - command: wc + args: "-w" +envs: + - key: TITLE + value: "The Great Dictator" +``` + + +Here, + +- `commands` lists the commands and arguments. +- `envs` lists the environment varaibles. + + +The example from 01_word_count runs a command line equivalent of: + +``` +cat ./files/the_great_dictator_speech.txt | wc -w +``` + +Now let's run. Try entering this command below: + +``` +> handoff --project 01_word_count --workspace workspace run local +``` +``` + +INFO - 2020-08-06 03:35:12,691 - handoff.config - Reading configurations from 01_word_count/project.yml +INFO - 2020-08-06 03:35:12,693 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:35:12,771 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:35:13,056 - handoff.config - You have the access to AWS resources. +WARNING - 2020-08-06 03:35:13,123 - handoff.config - Environment variable HO_BUCKET is not set. Remote file read/write will fail. +INFO - 2020-08-06 03:35:13,123 - handoff.config - Writing configuration files in the workspace configuration directory workspace/config +INFO - 2020-08-06 03:35:13,123 - handoff.config - Copying files from the local project directory 01_word_count +INFO - 2020-08-06 03:35:13,124 - handoff.config - Running run local in workspace directory +INFO - 2020-08-06 03:35:13,124 - handoff.config - Job started at 2020-08-06 03:35:13.124542 +INFO - 2020-08-06 03:35:13,130 - handoff.config - Job ended at 2020-08-06 03:35:13.130391 +``` + + +If you see the output that looks like: + +``` + + INFO - 2020-08-03 04:51:01,971 - handoff.config - Reading configurations from 01_word_count/project.yml + ... + INFO - 2020-08-03 04:51:02,690 - handoff.config - Processed in 0:00:00.005756 + +``` + + +Then great! You just ran the first local test. It created a workspace +directory that looks like: + +``` +> ls -l workspace +``` +``` + + artifacts + config + files +``` + +And the word count is stored at workspace/artifacts/state. Here is the content: + +``` +> cat workspace/artifacts/state +``` + +``` +644 +``` + + +By the way, the example text is from the awesome speech by Charlie Chaplin's +in the movie the Great Dictator. + +Here is a link to the famous speech scene. +Check out on YouTube: https://www.youtube.com/watch?v=J7GY1Xg6X20 + + + +And here is the first few paragraphs of the text: + +``` + +I’m sorry, but I don’t want to be an emperor. That’s not my business. I don’t want to rule or conquer anyone. I should like to help everyone - if possible - Jew, Gentile - black man - white. We all want to help one another. Human beings are like that. We want to live by each other’s happiness - not by each other’s misery. We don’t want to hate and despise one another. In this world there is room for everyone. And the good earth is rich and can provide for everyone. The way of life can be free and beautiful, but we have lost the way. + +Greed has poisoned men’s souls, has barricaded the world with hate, has goose-stepped us into misery and bloodshed. We have developed speed, but we have shut ourselves in. Machinery that gives abundance has left us in want. Our knowledge has made us cynical. Our cleverness, hard and unkind. We think too much and feel too little. More than machinery we need humanity. More than cleverness we need kindness and gentleness. Without these qualities, life will be violent and all will be lost…. + +``` + + +Now to the second example. This time project.yml looks like: + +``` +> cat 02_collect_stats/project.yml +``` + +``` +commands: + - command: cat + args: ./files/the_great_dictator_speech.txt + - command: python files/stats_collector.py + - command: wc + args: -w +``` + + +...which is shell equivalent to + +``` +cat ./files/the_great_dictator_speech.txt | python ./files/stats_collector.py | wc -w +``` + +The script for the second command stats_collector.py can be found in +02_collect_stats/files directory and it is a Python script that looks like: + + +``` +> cat 02_collect_stats/files/stats_collector.py +``` + +``` +#!/usr/bin/python +import io, json, logging, sys, os + +LOGGER = logging.getLogger() + +def collect_stats(outfile): + """ + Read from stdin and count the lines. Output to a file after done. + """ + lines = io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8") + output = {"rows_read": 0} + for line in lines: + try: + o = json.loads(line) + print(json.dumps(o)) + if o["type"].lower() == "record": + output["rows_read"] += 1 + except json.decoder.JSONDecodeError: + print(line) + output["rows_read"] += 1 + with open(outfile, "w") as f: + json.dump(output, f) + f.write("\n") + + +if __name__ == "__main__": + collect_stats("artifacts/collect_stats.json") +``` + +The script reads from stdin and counts the lines while passing the raw input to stdout. +The raw text is then processed by the third command (wc -w) and it conts the number of words. + + + +Now let's run. Try entering this command below: + +``` +> handoff --project 02_collect_stats --workspace workspace run local +``` +``` + +INFO - 2020-08-06 03:35:13,401 - handoff.config - Reading configurations from 02_collect_stats/project.yml +INFO - 2020-08-06 03:35:13,402 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:35:13,481 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:35:13,765 - handoff.config - You have the access to AWS resources. +WARNING - 2020-08-06 03:35:13,830 - handoff.config - Environment variable HO_BUCKET is not set. Remote file read/write will fail. +INFO - 2020-08-06 03:35:13,830 - handoff.config - Writing configuration files in the workspace configuration directory workspace/config +INFO - 2020-08-06 03:35:13,830 - handoff.config - Copying files from the local project directory 02_collect_stats +INFO - 2020-08-06 03:35:13,831 - handoff.config - Running run local in workspace directory +INFO - 2020-08-06 03:35:13,831 - handoff.config - Job started at 2020-08-06 03:35:13.831683 +INFO - 2020-08-06 03:35:13,881 - handoff.config - Job ended at 2020-08-06 03:35:13.881507 +INFO - 2020-08-06 03:35:13,881 - handoff.config - Processed in 0:00:00.049824 +``` + +Let's check out the contents of the second command: + + +``` +> cat workspace/artifacts/collect_stats.json +``` + +``` +{"rows_read": 15} +``` + + +In the next section, we will try pullin the currency exchange rate data. +You will also learn how to create Python virtual enviroments for each command +and pip-install commands. + diff --git a/docs/02_exchange_rates.md b/docs/02_exchange_rates.md new file mode 100644 index 0000000..2c0c03f --- /dev/null +++ b/docs/02_exchange_rates.md @@ -0,0 +1,130 @@ +## Virtual environment and install + +In this section, we will retrieve currency exchange rates and write out to CSV +file. + +We will install singer.io (https://singer.io), a data collection framework, +in Python vitual environment. + + + +We will use 03_exchange_rates project. project.yml looks like: + +``` +> cat 03_exchange_rates/project.yml +``` + +``` +commands: + - command: "tap-exchangeratesapi" + args: "--config config/tap-config.json" + venv: "proc_01" + installs: + - "pip install tap-exchangeratesapi" + - command: "python files/stats_collector.py" + venv: "proc_01" + - command: "target-csv" + args: "--config config/target-config.json" + venv: "proc_02" + installs: + - "pip install target-csv" +deploy: + provider: "aws" + platform: "fargate" + envs: + resource_group: "handoff-test" + docker_image: "singer_exchange_rates_to_csv" + task: "test-03-exchange-rates" +``` + + +...which is shell equivalent to + + tap-exchangeratesapi | python files/stats_collector.py | target-csv + + + +Before we can run this, we need to install tap-exchangeratesapi and target-csv. +The instructions for the install are listed in install section of project.yml. + +Notice `venv` entries for each command. handoff can create Python virtual +enviroment for each command to avoid conflicting dependencies among the +commands. + +To install everything, run this command: + +``` +> handoff -p 03_exchange_rates -w workspace_03 workspace install +``` +``` + +INFO - 2020-08-06 03:35:14,158 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:35:14,240 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:35:14,524 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:35:14,524 - handoff.config - Platform: aws +INFO - 2020-08-06 03:35:19,456 - handoff.config - Running /bin/bash -c "source proc_01/bin/activate && pip install wheel && pip install tap-exchangeratesapi" +Requirement already satisfied: wheel in ./proc_01/lib/python3.6/site-packages (0.34.2) +Processing /home/ubuntu/.cache/pip/wheels/1f/73/f9/xxxxxxxx0dba8423841c1404f319bb/tap_exchangeratesapi-0.1.1-cp36-none-any.whl +Processing /home/ubuntu/.cache/pip/wheels/6e/07/1b/xxxxxxxx6d9ce55c05f67a69127e25/singer_python-5.3.3-cp36-none-any.whl +Processing /home/ubuntu/.cache/pip/wheels/fc/d8/34/xxxxxxxx027b62dfcf922fdf8e396d/backoff-1.3.2-cp36-none-any.whl +Collecting requests==2.21.0 +. +. +. +Collecting python-dateutil + Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB) +Collecting pytzdata + Using cached pytzdata-2020.1-py2.py3-none-any.whl (489 kB) +Collecting pytz + Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB) +Collecting six>=1.5 + Using cached six-1.15.0-py2.py3-none-any.whl (10 kB) +Installing collected packages: jsonschema, simplejson, pytz, tzlocal, six, python-dateutil, pytzdata, pendulum, singer-python, target-csv +Successfully installed jsonschema-2.6.0 pendulum-1.2.0 python-dateutil-2.8.1 pytz-2020.1 pytzdata-2020.1 simplejson-3.11.1 singer-python-2.1.4 six-1.15.0 target-csv-0.3.0 tzlocal-2.1 +``` + +Now let's run the task. Try entering this command below: + +``` +> handoff -p 03_exchange_rates -w workspace_03 run local +``` +``` + +INFO - 2020-08-06 03:35:29,258 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:35:29,339 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:35:29,626 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:35:29,626 - handoff.config - Platform: aws +INFO - 2020-08-06 03:35:29,626 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:35:29,693 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:35:29,693 - handoff.config - Writing configuration files in the workspace configuration directory workspace_03/config +INFO - 2020-08-06 03:35:29,694 - handoff.config - Copying files from the local project directory 03_exchange_rates +INFO - 2020-08-06 03:35:29,695 - handoff.config - Running run local in workspace_03 directory +INFO - 2020-08-06 03:35:29,695 - handoff.config - Job started at 2020-08-06 03:35:29.695732 +. +. +. +INFO - 2020-08-06 03:35:33,964 - handoff.config - Job ended at 2020-08-06 03:35:33.964206 +INFO - 2020-08-06 03:35:33,964 - handoff.config - Processed in 0:00:04.268474 +``` + +This process should have created a CSV file in artifacts directory: + +``` + +exchange_rate-20200806T033530.csv +``` + +...which looks like: + +``` + +CAD,HKD,ISK,PHP,DKK,HUF,CZK,GBP,RON,SEK,IDR,INR,BRL,RUB,HRK,JPY,THB,CHF,EUR,MYR,BGN,TRY,CNY,NOK,NZD,ZAR,USD,MXN,SGD,AUD,ILS,KRW,PLN,date +0.0127290837,0.0725398406,1.3197211155,0.4630976096,0.0618218792,2.9357569721,0.2215388446,0.007434429,0.0401958831,0.0863047809,135.1005146082,0.7041915671,0.050374336,0.6657569721,0.0625373506,1.0,0.29312749,0.0088188911,0.0083001328,0.0399311089,0.0162333997,0.0642571381,0.0655312085,0.0889467131,0.0142670983,0.158440405,0.0093592297,0.2132744024,0.0130336985,0.0134852258,0.032375498,11.244189907,0.0371372842,2020-07-10T00:00:00Z +0.0126573311,0.072330313,1.313014827,0.4612685338,0.061324547,2.9145799012,0.2195057661,0.007408402,0.0399036244,0.085529654,134.613509061,0.7019439868,0.049830313,0.6601894563,0.0620593081,1.0,0.2929324547,0.0088014827,0.0082372323,0.0397907743,0.0161103789,0.0641054366,0.0653286656,0.0878500824,0.0141894563,0.1562817133,0.0093319605,0.209931631,0.0129678748,0.013383855,0.0321466227,11.2139209226,0.0368682043,2020-07-13T00:00:00Z +``` + + +Now that we know how to run locally, we will gradually thinking about how to deploy this in the cloud *severlessly*. +We will learn how to save and fetch the configurations to the remote storage. +Before doing that, we will cover how to set up AWS account and profile in the next section. + diff --git a/docs/03_set_up_aws_account.md b/docs/03_set_up_aws_account.md new file mode 100644 index 0000000..3aaab92 --- /dev/null +++ b/docs/03_set_up_aws_account.md @@ -0,0 +1,102 @@ +## Setting up an AWS account and profile + +handoff helps you to deploy the task in the cloud computing services. +Our goal in this tutorial is to deploy the currency exchange +rates task to [AWS Fargate](https://aws.amazon.com/fargate/). + +Before we can deploy, we need to set up the AWS account. + + + +**Do I need a credit card to use AWS? How much does it cost?** + +Yes, you need a credit card. But it won't cost a cup of coffee for this tutorial. +(Probably not even a dollar.) +Some of the AWS services we use comes with a Free Tier and you won't be +charged for the eligible usage. + +Learn more about [Free Tier](https://aws.amazon.com/free). + + + +Here is the complete list of the services we use: + +- Simple Storage Service, free with Free Tier usage: + https://aws.amazon.com/s3/pricing/#AWS_Free_Tier +- Fargate, almost free: + Price calculator http://fargate-pricing-calculator.site.s3-website-us-east-1.amazonaws.com/ + (We use 0.25vCPU with 0.5GB RAM for less than 5 minutes.) +- Systems Management Parameter Store, free (Standard tier): + Pricing: https://aws.amazon.com/systems-manager/pricing/#Parameter_Store +- Elastic Container Registry, free for 500MB per month + Pricing: https://aws.amazon.com/ecr/pricing/ + + + +AWS sign up is easy. Just go to: + + https://aws.amazon.com/free + +and complete the sign up process. + +The rest of the tutorial assumes that you have an active AWS account. + + + +If you haven't, install AWS CLI (command line interface): + + https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html + + + +Also generate access key ID and secret access key by following: + + https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html#xxxxxxxxds + +Take note of the values that looks like: + +- Access key ID: AKIAIOSFODNN7EXAMPLE +- Secret access key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +The only time that you can view or download the secret access key is when you +create the keys. You cannot recover them later. So keep a local copy. + + + +Last step before going back to handoff. From the console, run: + + aws configure --profile default + +Enter Access key ID and Secret access key you created in the last step. + +For region, use one of these keys (for USA users, us-east-1 would do): + + +``` + +ap-northeast-1 +ap-northeast-2 +ap-south-1 +ap-southeast-1 +ap-southeast-2 +ca-central-1 +eu-central-1 +eu-north-1 +eu-west-1 +eu-west-2 +eu-west-3 +sa-east-1 +us-east-1 +us-east-2 +us-west-1 +us-west-2 +``` + +Learn more about AWS profile +[here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) + + + +Now that we have set up the AWS account, let's store the configurations +in the cloud in the next section. + diff --git a/docs/04_run_remote_config.md b/docs/04_run_remote_config.md new file mode 100644 index 0000000..d710947 --- /dev/null +++ b/docs/04_run_remote_config.md @@ -0,0 +1,267 @@ +## Running task locally with remotely stored configurations + +We will continue using 03_exchange_rates example. But this time, we will +store the configurations to the remote data store. + + + +Let's review how 03_exchange_rates project directory is structured: + +``` +> ls -l 03_exchange_rates +``` +``` + + config + files + project.yml +``` + + +In the config directory, we have a couple of configuration files: + +``` +> ls -l 03_exchange_rates/config +``` +``` + + tap-config.json + target-config.json +``` + + +...which look like: + +``` +> cat 03_exchange_rates/config/tap-config.json +``` + +``` +{ "base": "JPY", "start_date": "2020-07-10" } +``` +``` +> cat 03_exchange_rates/config/target-config.json +``` + +``` +{ + "delimiter": ",", + "quotechar": "'", + "destination_path": "artifacts" +} +``` + +Often times, such configuration files contain sensitive information. +So we push the configrations to a secure key store such as +AWS Systems Manager (SSM) Parameter Store (In SecuresString format) +handoff wraps this proces by a single command. handoff packs all the JSON +format files under the configuration directory and send it to the remote +key-value storage. + +Try running: + +``` +> handoff -p 03_exchange_rates config push +``` +``` + +INFO - 2020-08-06 03:35:35,362 - handoff.config - Compiling config from 03_exchange_rates +INFO - 2020-08-06 03:35:35,362 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:35:35,443 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:35:35,732 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:35:35,732 - handoff.config - Platform: aws +INFO - 2020-08-06 03:35:35,732 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:35:35,799 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:35:35,799 - handoff.config - Putting the config to AWS SSM Parameter Store with Standard tier +INFO - 2020-08-06 03:35:35,814 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:35:36,184 - handoff.config - See the parameters at https://console.aws.amazon.com/systems-manager/parameters/?region=us-east-1&tab=Table#list_parameter_filters=Name:Contains:xxxxxxxxe-rates-config +``` + +Look at the end of the log that says, + +``` + See the parameters at https://console.aws.amazon.com/systems-manager/parameters/?region=... +``` + +Grab the link and open in the browswer (AWS login required) to confirm that +the parameters are uploaded. + + + +We also have some files needed for the task execution: + +``` +> ls -l 03_exchange_rates/files +``` +``` + + stats_collector.py +``` + +We need to store this somewhere accessible. +So we will create a cloud data storage (S3 bucket) for that. + +Try running: + +``` +> handoff -p 03_exchange_rates cloud create_bucket +``` +``` + +INFO - 2020-08-06 03:35:36,484 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:35:36,567 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:35:36,854 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:35:36,854 - handoff.config - Platform: aws +INFO - 2020-08-06 03:35:36,855 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:35:36,921 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:35:37,361 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:35:37,364 - handoff.config - Platform: aws +INFO - 2020-08-06 03:35:37,365 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:35:37,381 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +. +. +. +INFO - 2020-08-06 03:35:37,815 - handoff.config - {'StackId': 'arn:aws:cloudformation:us-east-1:xxxxxxxxxxxx:stack/handoff-test-bucket/xxxxxxxx861a6983', 'ResponseMetadata': {'RequestId': 'xxxxxxxx36ff08b9', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': 'xxxxxxxx36ff08b9', 'content-type': 'text/xml', 'content-length': '389', 'date': 'Thu, 06 Aug 2020 03:35:37 GMT'}, 'RetryAttempts': 0}} +INFO - 2020-08-06 03:35:37,815 - handoff.config - Check the progress at https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/stackinfo?viewNested=true&hideStacks=false&stackId=arn:aws:cloudformation:us-east-1:xxxxxxxxxxxx:stack/handoff-test-bucket/xxxxxxxx861a6983 +``` + +Wait for a minute and check here + +``` + https://s3.console.aws.amazon.com +``` + +to make sure the bucket is created before you proceed. + + + +Now it's time to push the files to the bucket. + +Try running: + +``` +> handoff -p 03_exchange_rates files push +``` +``` + +INFO - 2020-08-06 03:38:38,087 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:38:38,168 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:38:38,456 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:38:38,456 - handoff.config - Platform: aws +INFO - 2020-08-06 03:38:38,456 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:38:38,523 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:38:38,538 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:38:38,620 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:38:38,914 - handoff.services.cloud.aws.s3 - Uploading 03_exchange_rates/files to bucket xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:38:38,915 - handoff.services.cloud.aws.s3 - Files to be uploaded: ['03_exchange_rates/files/stats_collector.py'] +. +. +. +INFO - 2020-08-06 03:38:38,915 - handoff.services.cloud.aws.s3 - Uploading 03_exchange_rates/files/stats_collector.py to Amazon S3 bucket xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/files/stats_collector.py +INFO - 2020-08-06 03:38:39,279 - handoff.config - See the files at https://s3.console.aws.amazon.com/s3/buckets/xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/files/ +``` + +Look at the end of the log that says, + +``` + See the files at https://s3.console.aws.amazon.com/s3/... +``` + +Grab the link and open in the browswer (AWS login required) to see the files +directory is uploaded. + + + +Install the workspace: + +``` +> handoff -p 03_exchange_rates -w workspace workspace install +``` +``` + +INFO - 2020-08-06 03:38:39,718 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:38:39,799 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:38:40,082 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:38:40,082 - handoff.config - Platform: aws +INFO - 2020-08-06 03:38:44,491 - handoff.config - Running /bin/bash -c "source proc_01/bin/activate && pip install wheel && pip install tap-exchangeratesapi" +Requirement already satisfied: wheel in ./proc_01/lib/python3.6/site-packages (0.34.2) +Processing /home/ubuntu/.cache/pip/wheels/1f/73/f9/xxxxxxxx0dba8423841c1404f319bb/tap_exchangeratesapi-0.1.1-cp36-none-any.whl +Collecting requests==2.21.0 + Using cached requests-2.21.0-py2.py3-none-any.whl (57 kB) +Processing /home/ubuntu/.cache/pip/wheels/fc/d8/34/xxxxxxxx027b62dfcf922fdf8e396d/backoff-1.3.2-cp36-none-any.whl +. +. +. +Collecting tzlocal + Using cached tzlocal-2.1-py2.py3-none-any.whl (16 kB) +Collecting python-dateutil + Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB) +Collecting pytz + Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB) +Collecting six>=1.5 + Using cached six-1.15.0-py2.py3-none-any.whl (10 kB) +Installing collected packages: pytzdata, pytz, tzlocal, six, python-dateutil, pendulum, simplejson, singer-python, jsonschema, target-csv +Successfully installed jsonschema-2.6.0 pendulum-1.2.0 python-dateutil-2.8.1 pytz-2020.1 pytzdata-2020.1 simplejson-3.11.1 singer-python-2.1.4 six-1.15.0 target-csv-0.3.0 tzlocal-2.1 +``` + +Now let's run the command by pulling the configurations and files from remote. + +Try running: + +``` +> handoff -p 03_exchange_rates -w workspace run remote_config --push-artifacts +``` +``` + +INFO - 2020-08-06 03:38:53,883 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:38:53,963 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:38:54,248 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:38:54,248 - handoff.config - Platform: aws +INFO - 2020-08-06 03:38:54,248 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:38:54,315 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:38:54,315 - handoff.config - Writing configuration files in the workspace configuration directory workspace/config +INFO - 2020-08-06 03:38:54,315 - handoff.config - Reading configurations from remote parameter store. +INFO - 2020-08-06 03:38:54,316 - handoff.config - Reading precompiled config from remote. +INFO - 2020-08-06 03:38:54,330 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +. +. +. +INFO - 2020-08-06 03:38:57,479 - handoff.services.cloud.aws.s3 - Files to be uploaded: ['workspace/artifacts/state', 'workspace/artifacts/exchange_rate-20200806T033855.csv', 'workspace/artifacts/collect_stats.json'] +INFO - 2020-08-06 03:38:57,479 - handoff.services.cloud.aws.s3 - Uploading workspace/artifacts/state to Amazon S3 bucket xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/state +INFO - 2020-08-06 03:38:57,631 - handoff.services.cloud.aws.s3 - Uploading workspace/artifacts/exchange_rate-20200806T033855.csv to Amazon S3 bucket xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/exchange_rate-20200806T033855.csv +INFO - 2020-08-06 03:38:57,991 - handoff.services.cloud.aws.s3 - Uploading workspace/artifacts/collect_stats.json to Amazon S3 bucket xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/collect_stats.json +INFO - 2020-08-06 03:38:58,139 - handoff.config - See the files at https://s3.console.aws.amazon.com/s3/buckets/xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/ +INFO - 2020-08-06 03:38:58,139 - handoff.services.cloud.aws.s3 - Copying recursively from s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/* to s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/runs/2020-08-06T03:38:58.139874/* +INFO - 2020-08-06 03:38:58,445 - handoff.services.cloud.aws.s3 - Copied s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/collect_stats.json to s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/runs/2020-08-06T03:38:58.139874/artifacts/collect_stats.json +INFO - 2020-08-06 03:38:58,644 - handoff.services.cloud.aws.s3 - Copied s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/exchange_rate-20200806T033855.csv to s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/runs/2020-08-06T03:38:58.139874/artifacts/exchange_rate-20200806T033855.csv +INFO - 2020-08-06 03:38:58,822 - handoff.services.cloud.aws.s3 - Copied s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/state to s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/runs/2020-08-06T03:38:58.139874/artifacts/state +INFO - 2020-08-06 03:38:58,988 - handoff.services.cloud.aws.s3 - Copied s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/files/stats_collector.py to s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/runs/2020-08-06T03:38:58.139874/files/stats_collector.py +``` + +Notice that we used --push-artifacts option in the last command. +With this option, we pushed the result to the bucket under + +``` + /last/artifacts + +``` + +directory. + + +Also note that artifacts are automatically archived at each run at + +``` + + + //runs/ + + +``` + +directory. + + + +Next step is to prepare a Docker image and test running it locally. + diff --git a/docs/05_docker.md b/docs/05_docker.md new file mode 100644 index 0000000..05eee6d --- /dev/null +++ b/docs/05_docker.md @@ -0,0 +1,127 @@ +## Building, running, and pushing a Docker image + +We will continue using 03_exchange_rates example. +Instead of running the task on the host machine, let's run on Docker. + + + +Let's build a Docker image. + +Try running the following command. Enter y when prompted at the beginning. +The build may take 5~10 minutes. + +``` +> handoff -p 03_exchange_rates container build +``` +``` + +INFO - 2020-08-06 03:38:59,397 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:38:59,478 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:38:59,766 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:38:59,766 - handoff.config - Platform: aws +INFO - 2020-08-06 03:38:59,766 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:38:59,833 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:38:59,913 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:38:59,913 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:38:59,916 - handoff.config - Platform: aws +INFO - 2020-08-06 03:38:59,916 - handoff.config - Setting environment variables from config. +. +. +. + +INFO - 2020-08-06 03:41:40,008 - handoff.config - Step 27/27 : CMD handoff ${COMMAND:-run remote_config} -w workspace -a -d ${DATA:-{}} -a +INFO - 2020-08-06 03:41:40,044 - handoff.config - ---> Running in 21a548c43c0b + +INFO - 2020-08-06 03:41:40,148 - handoff.config - ---> 1bbadd87b10d + +INFO - 2020-08-06 03:41:41,621 - handoff.config - Successfully built 1bbadd87b10d + +INFO - 2020-08-06 03:41:41,652 - handoff.config - Successfully tagged singer_exchange_rates_to_csv:0.5 +``` + +Now let's run the code in the Docker container. + +``` +> handoff -p 03_exchange_rates container run +``` +``` + +INFO - 2020-08-06 03:41:45,336 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:41:45,509 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:41:45,964 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:41:45,965 - handoff.config - Platform: aws +INFO - 2020-08-06 03:41:45,965 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:41:46,029 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:41:46,108 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:41:46,108 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:41:46,111 - handoff.config - Platform: aws +INFO - 2020-08-06 03:41:46,112 - handoff.config - Setting environment variables from config. +. +. +. +INFO - 2020-08-06 03:41:54,603 - handoff.services.cloud.aws.s3 - Copied s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/collect_stats.json to s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/runs/2020-08-06T03:41:54.279771/artifacts/collect_stats.json + +INFO - 2020-08-06 03:41:54,797 - handoff.services.cloud.aws.s3 - Copied s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/exchange_rate-20200806T033855.csv to s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/runs/2020-08-06T03:41:54.279771/artifacts/exchange_rate-20200806T033855.csv + +INFO - 2020-08-06 03:41:55,002 - handoff.services.cloud.aws.s3 - Copied s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/exchange_rate-20200806T034150.csv to s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/runs/2020-08-06T03:41:54.279771/artifacts/exchange_rate-20200806T034150.csv + +INFO - 2020-08-06 03:41:55,172 - handoff.services.cloud.aws.s3 - Copied s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/state to s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/runs/2020-08-06T03:41:54.279771/artifacts/state + +INFO - 2020-08-06 03:41:55,342 - handoff.services.cloud.aws.s3 - Copied s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/files/stats_collector.py to s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/runs/2020-08-06T03:41:54.279771/files/stats_collector.py +``` + +Confirm the run by checking the logs. Also check the artifacts on S3: +``` + + + xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/artifacts/ + + +``` + +directory. + + + +Now that we know the Docker container runs fine, let's push it to +AWS Elastic Container Registry. This may take a few minutes. + +``` +> handoff -p 03_exchange_rates container push +``` +``` + +INFO - 2020-08-06 03:41:57,295 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:41:57,378 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:41:57,659 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:41:57,659 - handoff.config - Platform: aws +INFO - 2020-08-06 03:41:57,660 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:41:57,725 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:41:57,804 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:41:57,804 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:41:57,807 - handoff.config - Platform: aws +INFO - 2020-08-06 03:41:57,808 - handoff.config - Setting environment variables from config. +. +. +. +INFO - 2020-08-06 03:44:22,758 - handoff.config - id: 6734fdd29c24 [================================================> ] 526.1MB/540.8MB +INFO - 2020-08-06 03:44:23,478 - handoff.config - id: c836a53f9c0b [=============================================> ] 401MB/439.5MB +INFO - 2020-08-06 03:44:24,493 - handoff.config - id: 44aecb8afa22 [=============================================> ] 400.4MB/439.5MB +INFO - 2020-08-06 03:44:29,789 - handoff.config - id: c836a53f9c0b [================================================> ] 426.4MB/439.5MB +INFO - 2020-08-06 03:44:29,849 - handoff.config - id: 6734fdd29c24 status: Pushed +INFO - 2020-08-06 03:44:30,083 - handoff.config - id: 44aecb8afa22 [================================================> ] 426.9MB/439.5MB +INFO - 2020-08-06 03:44:35,306 - handoff.config - id: c836a53f9c0b [==================================================>] 453.1MB +INFO - 2020-08-06 03:44:35,744 - handoff.config - id: 44aecb8afa22 [==================================================>] 453.7MB +INFO - 2020-08-06 03:44:37,333 - handoff.config - id: c836a53f9c0b status: Pushed +INFO - 2020-08-06 03:44:39,716 - handoff.config - id: 44aecb8afa22 status: Pushed +``` + +Confirm that the Docker image has been uploaded to: + +https://console.aws.amazon.com/ecr/repositories?region=us-east-1 + + + +Now that the Docker image is prepared, we will finally deploy the task on +AWS Fargate in the next tutorial. + diff --git a/docs/06_fargate.md b/docs/06_fargate.md new file mode 100644 index 0000000..79b2b64 --- /dev/null +++ b/docs/06_fargate.md @@ -0,0 +1,166 @@ +## Deploying on AWS Fargate + +We will finally deploy the task to AWS Fargate so we can automate the +recurring task. + + +Fargate is part of AWS Elastic Container Service (ECS). +The Fargate task run on the ECS cluster. +We will first need to create a cluster. +(You will only be charged for the usage.) + +To make this process easy, handoff packed everything up in a command: + +``` +> handoff -p 03_exchange_rates cloud create_resources +``` +``` + +INFO - 2020-08-06 03:44:42,787 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:44:42,867 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:44:43,152 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:44:43,152 - handoff.config - Platform: aws +INFO - 2020-08-06 03:44:43,152 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:44:43,218 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:44:44,678 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:44:44,686 - handoff.config - Platform: aws +INFO - 2020-08-06 03:44:44,686 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:44:44,709 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +. +. +. +INFO - 2020-08-06 03:44:45,269 - handoff.config - {'StackId': 'arn:aws:cloudformation:us-east-1:xxxxxxxxxxxx:stack/handoff-test-resources/xxxxxxxxdd50faa5', 'ResponseMetadata': {'RequestId': 'xxxxxxxx20b2ef81', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': 'xxxxxxxx20b2ef81', 'content-type': 'text/xml', 'content-length': '392', 'date': 'Thu, 06 Aug 2020 03:44:45 GMT'}, 'RetryAttempts': 0}} +INFO - 2020-08-06 03:44:45,270 - handoff.config - Check the progress at https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/stackinfo?viewNested=true&hideStacks=false&stackId=arn:aws:cloudformation:us-east-1:xxxxxxxxxxxx:stack/handoff-test-resources/xxxxxxxxdd50faa5 +``` + + +At the end of the log, you should see a line like: + +``` + + Check the progress at https://console.aws.amazon.com/cloudformation/home?region=xxxx + +``` + +Grab the entire link and open in a browser (you need to login in to AWS) to see +the progress of the resource creation. + +Wait until it says \"CREATE_COMPLETE\" + + + +Now it's time to deploy the task and here is the command: + +``` +> handoff -p 03_exchange_rates cloud create_task +``` +``` + +INFO - 2020-08-06 03:47:45,531 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:47:45,610 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:47:45,897 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:47:45,897 - handoff.config - Platform: aws +INFO - 2020-08-06 03:47:45,898 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:47:45,964 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:47:46,436 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:47:46,439 - handoff.config - Platform: aws +INFO - 2020-08-06 03:47:46,439 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:47:46,521 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +. +. +. +INFO - 2020-08-06 03:47:46,847 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:47:47,363 - handoff.config - {'StackId': 'arn:aws:cloudformation:us-east-1:xxxxxxxxxxxx:stack/test-03-exchange-rates/xxxxxxxxa78ee1f9', 'ResponseMetadata': {'RequestId': 'xxxxxxxxdb623334', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': 'xxxxxxxxdb623334', 'content-type': 'text/xml', 'content-length': '392', 'date': 'Thu, 06 Aug 2020 03:47:46 GMT'}, 'RetryAttempts': 0}} +INFO - 2020-08-06 03:47:47,363 - handoff.config - Check the progress at https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/stackinfo?viewNested=true&hideStacks=false&stackId=arn:aws:cloudformation:us-east-1:xxxxxxxxxxxx:stack/test-03-exchange-rates/xxxxxxxxa78ee1f9 +``` + + +Here again, at the end of the log, you should see a line like: + +``` + + Check the progress at https://console.aws.amazon.com/cloudformation/home?region=xxxx + +``` + +Grab the entire link and open in a browser. + +Wait until it says "CREATE_COMPLETE" + + + +Once the task is created, try running on Fargate. +To do so, run this command: + +``` +> handoff -p 03_exchange_rates cloud run +``` +``` + +INFO - 2020-08-06 03:50:47,634 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:50:47,713 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:50:48,005 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:50:48,005 - handoff.config - Platform: aws +INFO - 2020-08-06 03:50:48,006 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:50:48,073 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:50:48,546 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:50:48,550 - handoff.config - Platform: aws +INFO - 2020-08-06 03:50:48,550 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:50:48,565 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +. +. +. +INFO - 2020-08-06 03:50:48,602 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:50:50,202 - handoff.config - {'tasks': [{'taskArn': 'arn:aws:ecs:us-east-1:xxxxxxxxxxxx:task/xxxxxxxx2f7337f3', 'clusterArn': 'arn:aws:ecs:us-east-1:xxxxxxxxxxxx:cluster/xxxxxxxxe-rates', 'taskDefinitionArn': 'arn:aws:ecs:us-east-1:xxxxxxxxxxxx:task-definition/xxxxxxxxe-rates:11', 'overrides': {'containerOverrides': [{'name': 'singer_exchange_rates_to_csv', 'environment': [{'name': 'TASK_TRIGGERED_AT', 'value': '2020-08-06T03:50:48.550977'}]}], 'inferenceAcceleratorOverrides': []}, 'lastStatus': 'PROVISIONING', 'desiredStatus': 'RUNNING', 'cpu': '256', 'memory': '512', 'containers': [{'containerArn': 'arn:aws:ecs:us-east-1:xxxxxxxxxxxx:container/xxxxxxxxd07ec661', 'taskArn': 'arn:aws:ecs:us-east-1:xxxxxxxxxxxx:task/xxxxxxxx2f7337f3', 'name': 'singer_exchange_rates_to_csv', 'image': 'xxxxxxxxxxxx.dkr.ecr.us-east-1.amazonaws.com/singer_exchange_rates_to_csv:0.5', 'lastStatus': 'PENDING', 'networkInterfaces': [], 'cpu': '256', 'memory': '512'}], 'version': 1, 'createdAt': datetime.datetime(2020, 8, 6, 3, 50, 50, 108000, tzinfo=tzlocal()), 'group': 'family:xxxxxxxxe-rates', 'launchType': 'FARGATE', 'platformVersion': '1.3.0', 'attachments': [{'id': 'xxxxxxxx028447f7', 'type': 'ElasticNetworkInterface', 'status': 'PRECREATED', 'details': [{'name': 'subnetId', 'value': 'subnet-0c65a427feccde6be'}]}], 'tags': []}], 'failures': [], 'ResponseMetadata': {'RequestId': 'xxxxxxxx2456c334', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': 'xxxxxxxx2456c334', 'content-type': 'application/x-amz-json-1.1', 'content-length': '1370', 'date': 'Thu, 06 Aug 2020 03:50:49 GMT'}, 'RetryAttempts': 0}} +INFO - 2020-08-06 03:50:50,202 - handoff.config - Check the task at https://us-east-1.console.aws.amazon.com/ecs/home?region=us-east-1#/clusters/xxxxxxxxe-rates/tasks/xxxxxxxx2f7337f3 +``` + +At the end of the log, you should see a line like: + +``` + + Check the task at https://us-east-1.console.aws.amazon.com/ecs/home?region=xxxx + +``` + +Grab the entire link and open in a browser. + +At the top you see: + Task : 5a232313-e390.... + +In Details tab, + +Cluster xxxxxxxxe-rates +Launch type FARGATE +... +Last status PROVISIONING +Desired status RUNNING + + + +At the bottom of the page, you see: + +| Name | Container Runtime ID | Status | Image | ..... | +| :---------------- | :------------------- | :----------- | :---- | :---- | +| singer_excha... | 1371.... | PROVISIONING | xxxxx | ..... | + +Expand the section by clicking on a dark side-way trible ">" by the Task name. + +The status should change in this order: + +1. PROVIONING +2. RUNNING +3. STOPPED + +Once the status becomes RUNNING or STOPPED, you should be able to see +the execution log by clicking "View logs in CloudWatch" link at the bottom +of the section. + +The log should be similar to the output from the local execution. + + + +We confirmed that the task run in the same way as local execution. +Now let's go to the next section to learn how to schedule the task +so it runs periodically. + diff --git a/docs/07_schedule.md b/docs/07_schedule.md new file mode 100644 index 0000000..917b989 --- /dev/null +++ b/docs/07_schedule.md @@ -0,0 +1,54 @@ +## Scheduling a task on Fargate + +In this section, we will learn how to schedule a task so it runs automatically. + + +To schedule a task, use schedule command with target_id and +[CRON format]://en.wikipedia.org/wiki/Cron). + +We pass those values to handoff with `--data` (`-d` for short) option: + + +``` +> handoff -p 03_exchange_rates cloud schedule --data '{"target_id": "1", "cron": "55 03 * * ? *"}' +``` +``` + +INFO - 2020-08-06 03:50:50,489 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:50:50,570 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:50:50,855 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:50:50,856 - handoff.config - Platform: aws +INFO - 2020-08-06 03:50:50,856 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:50:50,922 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:50:51,364 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:50:51,368 - handoff.config - Platform: aws +INFO - 2020-08-06 03:50:51,373 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:50:51,396 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +. +. +. +INFO - 2020-08-06 03:50:51,760 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:50:52,131 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:50:52,745 - handoff.config - {'FailedEntryCount': 0, 'FailedEntries': [], 'ResponseMetadata': {'RequestId': 'xxxxxxxxef778e23', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': 'xxxxxxxxef778e23', 'content-type': 'application/x-amz-json-1.1', 'content-length': '41', 'date': 'Thu, 06 Aug 2020 03:50:52 GMT'}, 'RetryAttempts': 0}} +INFO - 2020-08-06 03:50:52,745 - handoff.config - Check the status at https://console.aws.amazon.com/ecs/home?region=us-east-1#/clusters/xxxxxxxxe-rates/scheduledTasks +``` + + +At the end of the log, you should see a line like: + +``` + + Check the progress at Check the status at https://console.aws.amazon.com/ecs/home?region=us-east-1#/clusters/... + +``` + + +Grab the entire link and open in a browser (you need to login in to AWS) to see +it's scheduled. + + + +We confirmed that the task run in the same way as local execution. +Now let's go to the next module to learn how to schedule the task +so it runs periodically. + diff --git a/docs/08_cleanup.md b/docs/08_cleanup.md new file mode 100644 index 0000000..9fefe07 --- /dev/null +++ b/docs/08_cleanup.md @@ -0,0 +1,111 @@ +## Cleaning up + +Let's clean everything up so we won't pay a fraction of penny after forgeting about this exercise. + +First unschedule the task: + +``` +> handoff -l warning -p 03_exchange_rates cloud unschedule -d '{"target_id": 1}' +``` +``` + +``` + +Then delete the task: + +``` +> handoff -l warning -p 03_exchange_rates cloud delete_task +``` +``` + +``` + +If there is no other task in the same resource group, we can delete it: + +``` +> handoff -l warning -p 03_exchange_rates cloud delete_resources +``` +``` + +``` + +Here is how to delete the configurations from SSM Parameter Store: + +``` +> handoff -p 03_exchange_rates config delete +``` +``` + +INFO - 2020-08-06 03:34:46,049 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:34:46,131 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:34:46,415 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:34:46,415 - handoff.config - Platform: aws +INFO - 2020-08-06 03:34:46,416 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:34:46,481 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:34:46,496 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +``` + +Here is how to delete the files from S3 bucket: + +``` +> handoff -p 03_exchange_rates files delete +``` +``` + +INFO - 2020-08-06 03:34:47,114 - handoff.config - Reading configurations from 03_exchange_rates/project.yml +INFO - 2020-08-06 03:34:47,195 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:34:47,479 - handoff.config - You have the access to AWS resources. +INFO - 2020-08-06 03:34:47,480 - handoff.config - Platform: aws +INFO - 2020-08-06 03:34:47,480 - handoff.config - Setting environment variables from config. +INFO - 2020-08-06 03:34:47,545 - handoff.config - Environment variable HO_BUCKET was set autoamtically as xxxxxxxxxxxx-handoff-test +INFO - 2020-08-06 03:34:47,560 - botocore.credentials - Found credentials in shared credentials file: ~/.aws/credentials +INFO - 2020-08-06 03:34:47,602 - handoff.services.cloud.aws.s3 - GET s3://xxxxxxxxxxxx-handoff-test/test-03-exchange-rates/last/files +``` + +If there is no other task in the same resource group, we can delete the bucket, too: + +``` +> handoff -l warning -p 03_exchange_rates cloud delete_bucket +``` +``` + +WARNING - 2020-08-06 03:34:49,044 - handoff.config - This will only delete the CloudFormation stack. The bucket xxxxxxxxxxxx-handoff-test will be retained. +``` + +The previous command only deleted the CloudFormation stack, but not the bucket itself. +Here is how to delete all the files in s3://xxxxxxxxxxxx-handoff-test bucket. This cannot be reversed: + +``` +> aws s3 rm --recursive s3://xxxxxxxxxxxx-handoff-test/ +``` +``` + +``` + +Here is how to delete s3://xxxxxxxxxxxx-handoff-test bucket. The bucket must be empty. This cannot be reversed: + +``` +> aws s3 rb s3://xxxxxxxxxxxx-handoff-test +``` +``` + +``` + +Now delete singer_exchange_rates_to_csv repository from ECR. This cannot be reversed. +--force option will ignore that we still have images in the repository. + +``` +> aws ecr delete-repository --repository-name singer_exchange_rates_to_csv --force +``` +``` + +``` +``` +> handoff -l warning -p 03_exchange_rates cloud delete_role +``` +``` + +``` + +That's all. + diff --git a/docs/docker.rst b/docs/docker.rst deleted file mode 100644 index 17f08eb..0000000 --- a/docs/docker.rst +++ /dev/null @@ -1,43 +0,0 @@ -Building and running Docker -=========================== - -Notice: The planned release 0.2.0 will replace this with a handoff subcommand -(`handoff docker_build` and etc). - -Dockerfile will try to install the project from `./project` directory. So copy -what you want to build from handoff repository root: - -.. code-block:: shell - - cp -r ./test_projects/03_exchange_rates ./project - -Then use fgops commands to build and test-running the image. - -.. mdinclude:: ../deploy/fargate/docker.md - -Run behavior -~~~~~~~~~~~~ - -By default, Dockerfile is configured to execute -`handoff run -p ./project -w ./workspace` assuming that the remote -configurations are set. - -Or you can specify the function to run together with the data via additional -environment variables in : - -.. code-block:: shell - - COMMAND=show_commands - DATA={"start_at":"1990-01-01T00:00:00","end_at":"2030-01-01T00:00:00"} - -...that would be picked up by Docker just as - -.. code-block:: shell - - CMD handoff ${COMMAND:-run} -w workspace -d ${DATA:-{}} -a - -See Dockerfile_ for details. - -.. _Dockerfile: https://github.com/anelendata/handoff/blob/v0.1.2-alpha/Dockerfile - -Next: :doc:`fargate` diff --git a/docs/envvar_outputs.md b/docs/envvar_outputs.md deleted file mode 100644 index 9e8620e..0000000 --- a/docs/envvar_outputs.md +++ /dev/null @@ -1,62 +0,0 @@ -# Environment varaible - -If you want to pass environment variables to the subprocesses, you can do so -by defining them in `project.yml`. Here is a singer.io example to fetch -the exchange rates then write out to BigQuery. Access to BigQuery requires -a credential file, and the environment variable `GOOGLE_APPLICATION_CREDENTIALS` -must indicate the path to the file: - -``` -commands: - - command: "tap-exchangeratesapi" - args: "--config config/tap-config.json" - venv: "proc_01" - installs: - - "pip install tap-exchangeratesapi" - - command: "python files/stats_collector.py" - venv: "proc_01" - - command: "target-bigquery" - args: "--config config/target-config.json" - venv: "proc_02" - installs: - - "pip install target-bigquery" -envs: - - key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "config/google_client_secret.json" -``` - -The above example should be accompanied by the following files stored in -`/config` directory: - -- tap-config.json -- target-config.json -- google_client_secret.json - -# Outputs - -## State file - -For each run, `/artifacts/state` is generated. This is a copy of -stderr output from the last command in the pipeline: -``` -{"start_date": "2020-07-12"} -``` - -A state file is a convenient way of passing information from one run to another -especially in serverless environment such as AWS Fargate. More about this later. - -## Environment files - -The current exchange rate example in [the last section](./venv_config) -also creates a file `/config/tap-config.json`: -``` -workspace -└── config - └── tap-config.json -``` - -This is extracted from the project config then written out during the run. - -The files can be fetched from a remote folder. This will be explained later. - -Next: [fgops & IAM role creation](role) diff --git a/docs/essential_commands.rst b/docs/essential_commands.rst new file mode 100644 index 0000000..5748d6e --- /dev/null +++ b/docs/essential_commands.rst @@ -0,0 +1,62 @@ +.. _essential_commands: + +Essential Commands at a Glance +============================== + +Use this page as a cheat-sheet after finishing the guided-tour (tutorial). + +Next: :doc:`guided_tour` + +Here is the essential commands in order of the workflow from the local testing +to Fargate deployment: + +.. code-block:: shell + + handoff -p -w workspace install + handoff -p -w run local + handoff -p config push + handoff -p cloud create_bucket + handoff -p files push + handoff -p -w run remote_config + handoff -p container build + handoff -p container run + handoff -p container push + handoff -p cloud create_resources + handoff -p cloud create_task + handoff -p cloud schedule + +Here are the commands to take down: + +.. code-block:: shell + + handoff -p cloud unschedule + handoff -p cloud delete_task + handoff -p cloud delete_resources + handoff -p files delete + handoff -p config delete + handoff -p cloud delete_bucket + +And here are the commands to remove the additional resources: + +.. code-block:: shell + + aws s3 rm --recursive s3://-// + aws s3 rb s3://// + aws ecr delete-repository --repository-name + +Here are the commands to create and delete a role (e.g. AWS Role): + +.. code-block:: shell + + handoff -p cloud create_role + handoff -p cloud delete_role + +handoff shows help document at the root level or subcommand-level: + +.. code-block:: shell + + handoff --help + handoff --help # Admin or Run command such as 'run local' + handoff cloud [] --help + handoff container [] --help + handoff [] --help diff --git a/docs/fargate.rst b/docs/fargate.rst deleted file mode 100644 index 8ba5c38..0000000 --- a/docs/fargate.rst +++ /dev/null @@ -1,8 +0,0 @@ -Fargate deployment -================== - -Now that the Docker image was built and pushed to the remote repository, -it's time to deploy the task on Fargate. Follow these commands under -:code:`./deploy/fargate`. - -.. mdinclude:: ../deploy/fargate/deploy.md diff --git a/docs/guided_tour.rst b/docs/guided_tour.rst new file mode 100644 index 0000000..778034e --- /dev/null +++ b/docs/guided_tour.rst @@ -0,0 +1,25 @@ +.. _guided_tour: + +A Guided Tour +============= + +This page is the web version of the interactive commandline tutorial. + +The best way of getting started with handoff is to go through interactively. +You should be able to finish it in 30 minutes to an hour. + +.. code-block:: shell + + handoff quick_start make + ./projects/start + +Below is the web-version of the tutorial. + +.. mdinclude:: ./01_run_local.md +.. mdinclude:: ./02_exchange_rates.md +.. mdinclude:: ./03_set_up_aws_account.md +.. mdinclude:: ./04_run_remote_config.md +.. mdinclude:: ./05_docker.md +.. mdinclude:: ./06_fargate.md +.. mdinclude:: ./07_schedule.md +.. mdinclude:: ./08_cleanup.md diff --git a/docs/index.rst b/docs/index.rst index ec61aae..37de9ad 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,12 +15,8 @@ Contents :maxdepth: 2 quick_example - venv_config - envvar_outputs - role - remote_config - docker - fargate + essential_commands + guided_tour faq contributing code_of_conduct diff --git a/docs/quick_example.md b/docs/quick_example.md index 677c7be..13be3f2 100644 --- a/docs/quick_example.md +++ b/docs/quick_example.md @@ -22,42 +22,34 @@ source venv/bin/activate pip install handoff ``` -The rest of the instruction uses the test project definitions in the repository. -Clone [repo](https://github.com/anelendata/handoff) & initialize the submodules: -``` -git clone https://github.com/anelendata/handoff.git -git submodule init -git submodule update -``` +## A Super Quick Example + +Run this command to create a few example project under the current directory: -Note: Instead of installing from Python Package Index with `pip` command, you can -also install handoff from the repository. This is good for the people who -wants to try the unreleased version or wants to improve the project: ``` -python -m venv ./venv -source venv/bin/activate -python setup.py install +handoff quick_start make ``` -## A Super Quick Example +You will find `project.yml` file in `projects/01_word_count` directory. + +The project file defines the commands to be executed as a pipeline: -There is a `project.yml` file in `test_projects/01_word_count` directory at -the root of the repository ([here](https://github.com/anelendata/handoff/blob/master/test_projects/01_word_count/project.yml)). -The project file defines the commands to be executed as a pipeline, for example: ``` +> cat projects/project.yml + commands: - command: cat - args: "../../README.md" + args: "./files/the_great_dictator_speech.txt" - command: wc - args: "-l" + args: "-w" ``` This project file defines a shell-script equivalent of -`cat ../../README.md | wc -l`. +`cat ./files/the_great_dictator_speech.txt | wc -w`. Try runing: ``` -handoff run_local -p test_projects/01_word_count -w test_workspaces/01_word_count +handoff -p projects/01_word_count -w workspace run local ``` You get console outputs like this: @@ -68,14 +60,27 @@ INFO - 2020-07-16 22:04:34,010 - handoff: Job ended at 2020-07-16 22:04:34.01002 INFO - 2020-07-16 22:04:34,010 - handoff: Processed in 0:00:00.005556 ``` -It creates `test_workspaces/01_word_count/artifacts/state` whose content: +It will create `workspace/artifacts/state` whose content looks like: ``` -42 +644 ``` ...which is the equivalent of running: ``` -cat ./README.md | wc -l +cat ./files/the_great_dictator_speech.txt | wc -w ``` -from the repository's root directory. +It counted the word in the file. + +## A Guided Tour + +The project files you just created with `handoff quick_start make` has an +interactive command-line tutorial. Each section is very short (5~10 minutes +to complete.) To start the interactive tutorial, enter: + +``` +cd projects +./start +``` + +Otherwise, you can browse the same content from the next page: -Next: [Try fetching currency exchange rates](venv_config) +Next: [Essential Commands at a Glance](./essential_commands) diff --git a/docs/remote_config.md b/docs/remote_config.md deleted file mode 100644 index e8f55a2..0000000 --- a/docs/remote_config.md +++ /dev/null @@ -1,111 +0,0 @@ -# Remote configurations and files - -We continue to use -[the currency exchange rate example](venv_config) from the previous -sections, so if you have not done so, please do it before trying the following -example. - -We also assume the following environment variables are exported by assuming -the IAM role from [the previous section](role). - -## Pushing config to SSM Parameter Store - -First push the project configurations at AWS SSM Parameter Store: - -``` -handoff config push -p test_projects/03_exchange_rates -``` - -You can check the currently stored values by print command: - -``` -handoff config print -p test_projects/03_exchange_rates -``` - -It should return a string like this: -``` -{"commands": [{"command": "tap-exchangeratesapi", "args": "--config config/tap-config.json", "venv": "proc_01", "installs": ["pip install tap-exchangeratesapi"]}, {"command": "files/stats_collector.py", "venv": "proc_01"}, {"command": "target-csv", "venv": "proc_02", "installs": ["pip install target-csv"]}], "files": [{"name": "tap-config.json", "value": "{ \"base\": \"JPY\", \"start_date\": \"2020-07-09\" }\n"}]} -``` - -The string is "compiled" from profile.yml and the JSON configuration files under -`/config` directory. - -Note: The maximum size of a value in Parameter Store is 4KB with Standard -tier. You can bump it up to 8KB with Advanced tier with `--allow-advanced-tier` option: - -``` -handoff config print -p test_projects/03_exchange_rates --allow-advanced-tier -``` - -## Pushing files to S3 - -In the project directory may contain `files` subdirectory and store the -files needed at run-time. The files should be first pushed to the remote -store (AWS S3) by running: - -``` -handoff files push -p test_projects/03_exchange_rates -``` - -You see handoff fetching the files into `workspace/files` by runnig: -``` -handoff files get -p test_projects/03_exchange_rates -``` - -You do not need to run the above command explicitly as handoff automatically -downlods them into the workspace as described in the next section. - -## Run - -Let's clean up the local workspace before running with the remotely stored -configs: -``` -rm -fr test_workspaces/03_exchange_rates/* -``` - -First we need to create the virtual environments and install the commands: -``` -handoff workspace install -w test_workspaces/03_exchange_rates -``` - -Then run: -``` -handoff run -w test_workspaces/03_exchange_rates -a -``` - -Notice that we dropped `-p` option in the last two commands. The project -configurations are fetched from remote this time. - -After the run, you find the output from the exchange rate tap example from -the previous example. - -In the above run command, we added `-a` option. This is short for `--push-artifacts`. -With this option, the files under `/artifacts` will be push to -the remote storage after the run. You can download the artifacts from the -remote storage by runing: -``` -handoff artifacts get -w -``` - -You can also push the contents of `/artifacts` manually: -``` -handoff artifacts push -w -``` - -In the remote storage, the copy of artifacts from the last run is found at -`//last`. The historical files are archived in -`//runs`. - -### Other useful commands - -`default` is a function defined in `impl.py`. Any function defined in `impl.py` can be invoked - -in the same manner. - -``` -source ./venv/root/bin/activate && python code show_commands && deactivate -``` - -This shows the commands in the pipeline. - -Next: [Building and running Docker image](docker) diff --git a/docs/role.rst b/docs/role.rst deleted file mode 100644 index d6ed70f..0000000 --- a/docs/role.rst +++ /dev/null @@ -1,56 +0,0 @@ -Remote configurations -===================== - -Why use cloud storages? -~~~~~~~~~~~~~~~~~~~~~~~ - -handoff not only lets you run the task locally with `hanoff run_local` command, -but also let you deploy the task in cloud platform (currently supports AWS -Fargate). In production, you do not want to store sensitive information in -Docker images. You may also want to avoid building new image just because -the password or other configurations slightly changes. - -So, we store sensitive information in AWS SSM Parameter Store with SecureString -format. And also store other data files that may change frequently to S3. In -the future, we want to support non-AWS storages and deployment options. - -We can first store the configurations and files in cloud, and still test -the task locally with the remotely stored configs and files. Then we can -deploy the task to Fargate. - -But first, we need to set up the AWS permissions (Policy and Role). This -section explain how best this is achieved. - -fgops ------ - -Remote configurations and deployment are supported by -`fgops `_. -fgops can be found at -`./deploy/fargate `_ -in handoff repository. fgops commands are also symlinked from the `bin` directory from handoff -repository's root. - -Note: The future plan is to implement a `handoff` subcommand to build Docker -image, run Docker locally, and deloy the task to Fargate. - -Creating IAM Role -================= - -handoff separates the code (Docker image) and configurations. -We use -`AWS Systems Manager Parameter Store `_ -to store the parameter file derived from `project.yml` and other files -under `config` is stoed as a `SecureString`. -Less-sensitive files, prepared in `/files` are copied to AWS S3. - -To start accessing the remote resources, we first need to set up the AWS -account. - -Change directory to :code:`./deploy/fargate` to follow these instructions. - -.. mdinclude:: ../deploy/fargate/configuration.md - -.. mdinclude:: ../deploy/fargate/role.md - -Next: :doc:`remote_config` diff --git a/docs/venv_config.md b/docs/venv_config.md deleted file mode 100644 index 0292804..0000000 --- a/docs/venv_config.md +++ /dev/null @@ -1,164 +0,0 @@ -# Virtual envs and configs - -## Running a Python Program in a Virtual Environment - -You can specify a virtual environments in case the command is a Python program. -In `project.yml`, you can also define virtual environment for Python program. -Here is the content of -[test_projects/02_collect_stats/project.yml](https://github.com/anelendata/handoff/blob/master/test_projects/02_collect_stats/project.yml): - -``` -commands: - - command: cat - args: "../../README.md" - - command: "python files/stats_collector.py" - venv: "proc_01" - - command: "wc" -``` - -[test_projects/02_collect_stats/files/stats_collector.py](https://github.com/anelendata/handoff/blob/master/test_projects/02_collect_stats/files/stats_collector.py) -is a simple Python program that pass on from stdin to stdout while counting the -number of rows. - -The file looks like this: -``` -#!/usr/bin/python -import io, json, logging, sys, os - -LOGGER = logging.getLogger() - -def collect_stats(outfile): - """ - Read from stdin and count the lines. Output to a file after done. - """ - lines = io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8") - output = {"rows_read": 0} - for line in lines: - try: - o = json.loads(line) - print(json.dumps(o)) - if o["type"].lower() == "record": - output["rows_read"] += 1 - except json.decoder.JSONDecodeError: - print(line) - output["rows_read"] += 1 - with open(outfile, "w") as f: - json.dump(output, f) - f.write("\n") - - -if __name__ == "__main__": - collect_stats("artifacts/collect_stats.json") -``` - -To run this project, make the virtual environment first: -``` -handoff workspace install -p test_projects/02_collect_stats -w test_workspaces/02_collect_stats -``` - -Try runing: -``` -handoff run_local -p test_projects/02_collect_stats -w test_workspaces/02_collect_stats -``` - -This time, you will get `test_workspaces/02_collect_stats/artifacts/collect_stats.json ` -that looks like: -``` -{"rows_read": 42} -``` - -Also notice that we chained 3 commands instead 2. This time, we got -`test_workspaces/02_collect_stats/artifacts/state` that contains the output of `wc` command: -``` - 84 213 1715 -``` - -Note: The example above is useful for the singer.io users who wants to insert -a transform process between tap and target. - -## How to use config files: Fetching exchange rates with singer.io - -The configuration files information go to `/config` directory. - -Such configuration files include singer.io tap and target's JSON config files, -and Google Cloud Platform key JSON file. - -Note: Currently, only JSON files are supported for remote config store. - -Example: - -The project file -[test_projects/03_exchange_rates/project.yml](https://github.com/anelendata/handoff/blob/master/test_projects/03_exchange_rates/project.yml) looks like this: -``` -commands: - - command: "tap-exchangeratesapi" - args: "--config config/tap-config.json" - venv: "proc_01" - installs: - - "pip install tap-exchangeratesapi" - - command: "python files/stats_collector.py" - venv: "proc_01" - - command: "target-csv" - args: "--config config/target-config.json" - venv: "proc_02" - installs: - - "pip install target-csv" -``` - -1. Make environments defined in the project and install commands -``` -handoff workspace install -p test_projects/03_exchange_rates -w test_workspaces/03_exchange_rates -``` - -2. Create a copy of config file for tap-exchangeratesapi: - -Let's limit the days of the data collection to the last 7 days. So use these -shell scripts to generate `tap-config.json`. - -In Debian/Ubuntu: -``` -echo '{ "base": "JPY", "start_date": "'`date -d "$date -7 days" +'%Y-%m-%d'`'" }' \ - > test_projects/03_exchange_rates/config/tap-config.json -``` - -In MacOS: -``` -echo '{ "base": "JPY", "start_date": "'`date -v -7d +'%Y-%m-%d'`'" }' \ - > test_projects/03_exchange_rates/config/tap-config.json -``` - -`tap-config.json` should looks something like: -``` -{ "base": "JPY", "start_date": "2020-07-06" } -``` - -3. Run -``` -handoff run_local -p test_projects/03_exchange_rates -w test_workspaces/03_exchange_rates -``` - -This should output something like: -``` -INFO - 2020-07-12 08:52:13,240 - impl.runner: Running run data:{} -INFO - 2020-07-12 08:52:13,240 - impl.runner: Running run -INFO - 2020-07-12 08:52:13,241 - impl.runner: Reading parameters from file: .env/params.json -INFO - 2020-07-12 08:52:13,241 - impl.runner: Job started at 2020-07-12 15:52:13.241577 -INFO Replicating exchange rate data from 2020-07-10 using base JPY -INFO Sending version information to singer.io. To disable sending anonymous usage data, set the config parameter "disable_collection" to true -INFO Replicating exchange rate data from 2020-07-11 using base JPY -INFO Replicating exchange rate data from 2020-07-12 using base JPY -INFO Tap exiting normally -INFO - 2020-07-12 08:52:13,950 - impl.runner: Job ended at 2020-07-12 15:52:13.950879 -INFO - 2020-07-12 08:52:13,951 - impl.runner: Processed in 0:00:00.709302 -``` - -and produce a file `exchange_rate-{timestamp}.csv` in `test_workspaces/03_exchange_rates/artifacts` that looks like: -``` -CAD,HKD,ISK,PHP,DKK,HUF,CZK,GBP,RON,SEK,IDR,INR,BRL,RUB,HRK,JPY,THB,CHF,EUR,MYR,BGN,TRY,CNY,NOK,NZD,ZAR,USD,MXN,SGD,AUD,ILS,KRW,PLN,date -0.0127290837,0.0725398406,1.3197211155,0.4630976096,0.0618218792,2.9357569721,0.2215388446,0.007434429,0.0401958831,0.0863047809,135.1005146082,0.7041915671,0.050374336,0.6657569721,0.0625373506,1.0,0.29312749,0.0088188911,0.0083001328,0.0399311089,0.0162333997,0.0642571381,0.0655312085,0.0889467131,0.0142670983,0.158440405,0.0093592297,0.2132744024,0.0130336985,0.0134852258,0.032375498,11.244189907,0.0371372842,2020-07-10T00:00:00Z -``` - -It also leaves `state` and `collector_stats.json` files under -`test_workspaces/03_exchange_rates/artifacts` directory. - -Next: [Environment variables and outputs](envvar_outputs) diff --git a/handoff/core/admin.py b/handoff/core/admin.py index de1c751..7eaad27 100644 --- a/handoff/core/admin.py +++ b/handoff/core/admin.py @@ -214,11 +214,10 @@ def files_push(project_dir, workspace_dir, data, **kwargs): def files_delete(project_dir, workspace_dir, data, **kwargs): - state = get_state() + _ = config_get_local(project_dir, workspace_dir, data) platform = cloud._get_platform() - state.validate_env() - LOGGER.info("Deleting files from the remote storage " + - state.get(BUCKET)) + files_dir = os.path.join(project_dir, FILES_DIR) + prefix = os.path.join(BUCKET_CURRENT_PREFIX, FILES_DIR) dir_name = os.path.join(BUCKET_CURRENT_PREFIX, FILES_DIR) platform.delete_dir(dir_name) @@ -308,9 +307,8 @@ def config_push(project_dir, workspace_dir, data, **kwargs): def config_delete(project_dir, workspace_dir, data, **kwargs): - state = get_state() + _ = config_get_local(project_dir, workspace_dir, data) platform = cloud._get_platform() - state.validate_env() platform.delete_parameter("config") diff --git a/handoff/plugins/quick_start/__init__.py b/handoff/plugins/quick_start/__init__.py index 1959d26..4bb4bc2 100644 --- a/handoff/plugins/quick_start/__init__.py +++ b/handoff/plugins/quick_start/__init__.py @@ -4,7 +4,7 @@ LOGGER = _get_logger(__name__) -def start(project_dir, workspace_dir, data, **kwargs): +def make(project_dir, workspace_dir, data, **kwargs): """Copy the test projects to the test_projects under the current directory """ d, f = os.path.split(__file__) @@ -15,6 +15,6 @@ def start(project_dir, workspace_dir, data, **kwargs): print("It looks like you already copied the test projects to ./projects") else: print("Copied the test projects to ./projects") - print("Now just enter:") - print(" ./projects/begin") + print("Now just do:") + print(" ./projects/start") print("to start the even-monkeys-can-follow tutorial.") diff --git a/handoff/services/cloud/aws/__init__.py b/handoff/services/cloud/aws/__init__.py index a9e5184..77d3fc5 100644 --- a/handoff/services/cloud/aws/__init__.py +++ b/handoff/services/cloud/aws/__init__.py @@ -130,17 +130,31 @@ def assume_role(role_arn=None, target_account_id=None, external_id=None): def get_parameter(key): - state = get_state() - return ssm.get_parameter(state.get(RESOURCE_GROUP) + "-" + - state.get(TASK), key) + try: + state = get_state() + prefix = state.get(RESOURCE_GROUP) + "-" + state.get(TASK) + prefix_key = prefix + "-" + key + value = ssm.get_parameter(prefix_key) + except Exception as e: + LOGGER.error("Cannot get %s - %s" % (prefix_key, str(e))) + LOGGER.error("See the parameters at https://console.aws.amazon.com/" + + "systems-manager/parameters/?region=" + + state.get("AWS_REGION") + + "&tab=Table#list_parameter_filters=Name:Contains:" + + prefix_key) + return value def push_parameter(key, value, allow_advanced_tier=False, **kwargs): state = get_state() state.validate_env([RESOURCE_GROUP, TASK, "AWS_REGION"]) + prefix = state.get(RESOURCE_GROUP) + "-" + state.get(TASK) + prefix_key = prefix + "-" + key + if allow_advanced_tier: - LOGGER.info("Allowing AWS SSM Parameter Store to store with Advanced tier (max 8KB)") + LOGGER.info("Allowing AWS SSM Parameter Store to store with " + + "Advanced tier (max 8KB)") tier = "Standard" if len(value) > 8192: raise Exception("Parameter string must be less than 8192kb!") @@ -148,21 +162,23 @@ def push_parameter(key, value, allow_advanced_tier=False, **kwargs): if allow_advanced_tier: tier = "Advanced" else: - raise Exception("Parameter string is %s > 4096 byte and allow_advanced_tier=False" % len(value)) - LOGGER.info("Putting the config to AWS SSM Parameter Store with %s tier" % tier) - ssm.put_parameter(state.get(RESOURCE_GROUP) + "-" + - state.get(TASK), - key, value, tier=tier) + raise Exception(("Parameter string is %s > 4096 byte and " + + "allow_advanced_tier=False") % len(value)) + LOGGER.info("Putting the config to AWS SSM Parameter Store with %s tier" % + tier) + ssm.put_parameter(prefix_key, value, tier=tier) LOGGER.info("See the parameters at https://console.aws.amazon.com/" + "systems-manager/parameters/?region=" + state.get("AWS_REGION") + - "&tab=Table") + "&tab=Table#list_parameter_filters=Name:Contains:" + + prefix_key) + def delete_parameter(key): state = get_state() - ssm.delete_parameter(state.get(RESOURCE_GROUP) + "-" + - state.get(TASK), - key) + prefix = state.get(RESOURCE_GROUP) + "-" + state.get(TASK) + prefix_key = prefix + "-" + key + ssm.delete_parameter(prefix_key) def set_env_var_from_ssm(project, name): @@ -184,7 +200,10 @@ def upload_dir(src_dir_name, dest_prefix): state = get_state() state.validate_env([BUCKET, "AWS_REGION"]) dest_prefix = os.path.join(state.get(TASK), dest_prefix) - s3.upload_dir(src_dir_name, dest_prefix, state.get(BUCKET)) + bucket = state.get(BUCKET) + s3.upload_dir(src_dir_name, dest_prefix, bucket) + LOGGER.info(("See the files at https://s3.console.aws.amazon.com/s3/" + + "buckets/%s/%s/") % (bucket, dest_prefix)) def delete_dir(remote_dir): @@ -440,15 +459,21 @@ def schedule_task(target_id, cronexp, role_arn=None): if not role_arn: roles = iam.list_roles() for r in roles: - print(r["RoleName"]) if r["RoleName"] == role_name: role_arn = r["Arn"] break if not role_arn: raise Exception("Role %s not found" % role_name) - response = events.schedule_task(task_stack, resource_group_stack, region, - target_id, cronexp, role_arn) + try: + response = events.schedule_task(task_stack, resource_group_stack, region, + target_id, cronexp, role_arn) + except Exception as e: + LOGGER.error("Scheduling task failed for %s target_id: %s cron: %s" % + (task_stack, target_id, cronexp)) + LOGGER.critical(str(e)) + return + LOGGER.info(response) params = { "region": state.get("AWS_REGION"), diff --git a/handoff/services/cloud/aws/ssm.py b/handoff/services/cloud/aws/ssm.py index 7b63d74..bd41610 100644 --- a/handoff/services/cloud/aws/ssm.py +++ b/handoff/services/cloud/aws/ssm.py @@ -12,14 +12,14 @@ def get_client(): return cred.get_client("ssm") -def get_parameter(project, name): +def get_parameter(key): client = get_client() - param = client.get_parameter(Name=project + "_" + name, WithDecryption=True) + param = client.get_parameter(Name=key, WithDecryption=True) value = param["Parameter"]["Value"] return value -def put_parameter(project, key, value, +def put_parameter(key, value, description="", type_="SecureString", key_id=None, @@ -35,7 +35,7 @@ def put_parameter(project, key, value, """ client = get_client() - kwargs = {"Name": project + "_" + key, + kwargs = {"Name": key, "Value": value, "Type": type_, "Overwrite": overwrite, @@ -55,8 +55,8 @@ def put_parameter(project, key, value, return response -def delete_parameter(project, key): +def delete_parameter(key): client = get_client() - kwargs = {"Name": project + "_" + key} + kwargs = {"Name": key} response = client.delete_parameter(**kwargs) return response diff --git a/handoff/services/container/docker/impl.py b/handoff/services/container/docker/impl.py index 1997e9d..a5c08cb 100644 --- a/handoff/services/container/docker/impl.py +++ b/handoff/services/container/docker/impl.py @@ -66,7 +66,7 @@ def build(project_dir, new_version=None, docker_file=None, nocache=False): logger.info("Building %s:%s" % (image_name, new_version)) with tempfile.TemporaryDirectory() as build_dir: - cwd = os.getcwd() + cwd, _ = os.path.split(__file__) pos = cwd.find("handoff") if pos >= 0: handoff_dir = os.path.join(cwd[:pos], "handoff") diff --git a/handoff/test_projects/01_word_count/project.yml b/handoff/test_projects/01_word_count/project.yml index 0d84fbe..4c29d10 100644 --- a/handoff/test_projects/01_word_count/project.yml +++ b/handoff/test_projects/01_word_count/project.yml @@ -4,5 +4,5 @@ commands: - command: wc args: "-w" envs: - - key: title - value: The Great Dictator + - key: TITLE + value: "The Great Dictator" diff --git a/handoff/test_projects/auto_run b/handoff/test_projects/auto_run new file mode 100755 index 0000000..a8eb4fe --- /dev/null +++ b/handoff/test_projects/auto_run @@ -0,0 +1,17 @@ +#!/bin/bash +if [[ -z $2 ]];then HEADTAIL_LIMIT=0; else HEADTAIL_LIMIT=$2;fi +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source scripts/funcs + +echo "Automatically run and generate docs" + +if [[ -z "$SCRIPT" ]]; then +CMD="ls scripts/aws_get_started | grep -v funcs | grep -v role" +Choose "$CMD" +SCRIPT=${OPTIONS[$CHOICE]} +fi + +rm -fr ./workspace* +yes | ./scripts/aws_get_started/$SCRIPT auto 10 | sed 's/[0-9]\{12\}/xxxxxxxxxxxx/g' | sed 's/[a-z0-9-]\{28\}/xxxxxxxx/g' > ../../docs/$SCRIPT.md + +echo See the doc at ../../docs/$SCRIPT.md diff --git a/handoff/test_projects/auto_run_all b/handoff/test_projects/auto_run_all new file mode 100755 index 0000000..67c5160 --- /dev/null +++ b/handoff/test_projects/auto_run_all @@ -0,0 +1,10 @@ +#!/bin/bash +if [[ -z $2 ]];then HEADTAIL_LIMIT=0; else HEADTAIL_LIMIT=$2;fi +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source scripts/funcs + +for i in `ls scripts/aws_get_started | grep -v funcs | grep -v cleanup`; +do +export SCRIPT=$i +./auto_run auto 10 +done diff --git a/handoff/test_projects/begin b/handoff/test_projects/begin deleted file mode 100755 index e14b326..0000000 --- a/handoff/test_projects/begin +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -echo $PROJECT_DIR -source $PROJECT_DIR/scripts/funcs - -if [ ! -d $PROJECT_DIR/03_exchange_rates ]; then - echo The project does not exist - exit 1 -fi -echo "Welcome to handoff tutorial -Each module is very short (5 ~ 10 minutes to complete) -" - -CMD="ls $PROJECT_DIR/scripts | grep -v funcs && printf \0" - -Choose "$CMD" - -$PROJECT_DIR/scripts/${OPTIONS[$CHOICE]} diff --git a/handoff/test_projects/scripts/01_run_local b/handoff/test_projects/scripts/01_run_local deleted file mode 100755 index 49a670c..0000000 --- a/handoff/test_projects/scripts/01_run_local +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash -PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." -source $PROJECT_DIR/scripts/funcs - -if [ ! -d $PROJECT_DIR/03_exchange_rates ]; then -echo " -The project does not exist...try running - handoff quick_start start -in a new directory. -" - exit 1 -fi - -cd $PROJECT_DIR - -echo " -Welcome to module 01_run_local -We will learn how to run handoff local test using a project 01_word_count -" - -echo "Ready? (Hit q at prompt to quit anytime. Any other key to proceed.)" -Prompt - -echo Each project directory contains: -echo -for i in `ls 01_word_count`; do echo - $i; done - -Prompt - -echo project.yml looks like: -Cat "01_word_count/project.yml" $DIM - -Prompt - -echo commands lists the commands and arguments. env lists the environment varaibles. - -echo " -This example from 01_word_count runs a command line equivalent of: - - cat ./files/the_great_dictator_speech.txt | wc -w -" - -echo Now let\'s run. Try entering this command below: -Command "handoff --project 01_word_count --workspace workspace run local" - -echo If you see the output that looks like: - -echo -e $DIM -echo "INFO - 2020-08-03 04:51:01,971 - handoff.config - Reading configurations from 01_word_count/project.yml" -echo "..." -echo "INFO - 2020-08-03 04:51:02,690 - handoff.config - Processed in 0:00:00.005756" -echo -e $RESET - -echo Then great! You just ran the first local test. It created a workspace directory that looks like: -echo -for i in `ls workspace`; do echo - $i; done -echo And the word count is stored at workspace/artifacts/state. Here is the content: -echo -cat workspace/artifacts/state -echo - -Prompt - -echo " -By the way, the example text is from the awesome speech by Charlie Chaplin\'s in the movie the Great Dictator. -Here is a link to the famous speech scene. Check out on YouTube: https://www.youtube.com/watch?v=J7GY1Xg6X20 -" - -Prompt - -echo And here is the first few paragraphs of the text: -echo -e $BLUE -head -n 4 01_word_count/files/the_great_dictator_speech.txt -echo -e $RESET - -Prompt - -echo Now to the second example. This time project.yml looks like: -Cat 02_collect_stats/project.yml $DIM - -Prompt - -echo " -...which is shell equivalent to - - cat ./files/the_great_dictator_speech.txt | python ./files/stats_collector.py | wc -w -" - -echo " -The script for the second command stats_collector.py can be found in 02_collect_stats/files directory -and it is a Python script that looks like: -" - -Prompt - -Cat "02_collect_stats/files/stats_collector.py" $DIM - -echo " -The script reads from stdin and counts the lines while passing the raw input to stdout. -The raw text is then processed by the third command (wc -w) and it conts the number of words. -" - -Prompt - -echo Now let\'s run. Try entering this command below: -Command "handoff --project 02_collect_stats --workspace workspace run local" - -echo Let\'s check out the contents of the second command: - -Prompt - -Cat "workspace/artifacts/collect_stats.json" $DIM - -Prompt - -echo Great. This concludes this section of the tutorial. - -Prompt - -Thanks - -echo " -In the next section, we will try pullin the currency exchange rate data. -You will also learn how to create Python virtual enviroments for each command and pip-install commands. -" - -Continue - -$PROJECT_DIR/scripts/02_exchange_rates diff --git a/handoff/test_projects/scripts/02_exchange_rates b/handoff/test_projects/scripts/02_exchange_rates deleted file mode 100755 index 3180eea..0000000 --- a/handoff/test_projects/scripts/02_exchange_rates +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." -source $PROJECT_DIR/scripts/funcs - -if [ ! -d $PROJECT_DIR/03_exchange_rates ]; then -echo " -The project does not exist...try running - handoff quick_start start -in a new directory. -" - exit 1 -fi - -cd $PROJECT_DIR - -echo " -Welcome to module 02_exchange_rates! -We will retrieve currency exchange rates and write out to CSV file. -We will install singer.io (https://singer.io), a data collection framework, -in Python vitual environment. -" - -echo "Ready? (Hit q at prompt to quit anytime. Any other key to proceed.)" - -Prompt - -echo " -We use 03_exchange_rates project. project.yml looks like: -" -Cat "03_exchange_rates/project.yml" $DIM - -Prompt - -echo " -...which is shell equivalent to - - tap-exchangeratesapi | python files/stats_collector.py | target-csv -" - -Prompt - -rm -fr workspace_03 - -echo " -Before we can run this, we need to install tap-exchangeratesapi and target-csv. -The installation instructions are listed in install section of project.yml. -Notice venv entries for each command. handoff can create Python virtual enviroment for each command to avoid conflicting dependencies between the commands. - -To install everything, run this command: -" -Command "handoff -p 03_exchange_rates -w workspace_03 workspace install" - - -echo " -Now let's run. Try entering this command below: -" -Command "handoff -p 03_exchange_rates -w workspace_03 run local" - -echo This process should have created a CSV file in artifacts directory: - -FILES=`ls workspace_03/artifacts | grep csv` -echo $FILES - -echo - -echo ...which looks like: - -for f in $FILES; do head -n 3 workspace_03/artifacts/$f; done -echo ... -for f in $FILES; do tail -n 3 workspace_03/artifacts/$f; done - -Prompt - -echo Great. This concludes this section of the tutorial. - -Prompt - -Thanks - -echo " -Now that we know how to run locally, we will gradually thinking about how to deploy this in the cloud *severlessly*. -We will learn how to save and fetch the configurations to the remote storage. -Before doing that, we will cover how to set up AWS Role with handoff in the next section. -" - -Continue - -$PROJECT_DIR/scripts/03_set_up_aws_account diff --git a/handoff/test_projects/scripts/04_run_remote_config b/handoff/test_projects/scripts/04_run_remote_config deleted file mode 100755 index 507b195..0000000 --- a/handoff/test_projects/scripts/04_run_remote_config +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash -PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." -source $PROJECT_DIR/scripts/funcs - -if [ ! -d $PROJECT_DIR/03_exchange_rates ]; then - echo The project does not exist...try running - echo " handoff quick_start start" - echo in a new directory. - exit 1 -fi - -cd $PROJECT_DIR - -echo " -Welcome to module 04_run_remote_config! - -We will continue using 03_exchange_rates example. But this time, we will -store the configurations to the remote data store. - -Ready? (ctl-c to quit anytime) -" - -Prompt - -CheckAws - -ChooseAwsProfile - -echo " -Review how 03_exchange_rates project directory is structured: -" - -echo 03_exchange_rates: -ListDir "03_exchange_rates" $DIM - -Prompt - -echo " -In the config directory, we have a couple of configuration files: -" -echo 03_exchange_rates/config: -ListDir "03_exchange_rates/config" $DIM - -Prompt - -echo " -...which look like: - -tap-config.json: -" -Cat "03_exchange_rates/config/tap-config.json" $DIM -echo " - -target-config.json: -" -Cat "03_exchange_rates/config/target-config.json" $DIM - -echo " -Often times, such configuration files contain sensitive information. -So we push the configrations to a secure key store such as -AWS Systems Manager (SSM) Parameter Store (In SecuresString format) -handoff wraps this proces by a command so you can push everything inside -JSON files under config directory. - -Try running: -" -Command "handoff -p 03_exchange_rates config push" - -echo " -We also have some files needed for the task execution: -" -echo 03_exchange_rates/files: -ListDir "03_exchange_rates/files" $DIM - -echo " -We need to store this somewhere accessible. -So we will create a cloud data storage (S3 bucket) for that. - -Try running: -" -Command "handoff -p 03_exchange_rates cloud create_bucket" - -echo " -Wait for a minute and check here - -https://s3.console.aws.amazon.com/s3 - -to make sure the bucket is created before you can proceed. -" - -Prompt - -echo " -Now it's time to push the files to the bucket. - -Try running: -" -Command "handoff -p 03_exchange_rates files push" - -BUCKET_URL=https://s3.console.aws.amazon.com/s3/buckets -BUCKET=`handoff -p 03_exchange_rates envs get -d '{"key":"HO_BUCKET"}' -l critical` -TASK=`handoff -p 03_exchange_rates envs get -d '{"key":"HO_TASK"}' -l critical` - -echo " -Check - - $BUCKET_URL/$BUCKET/$TASK/last/ - -to see the files directory is uploaded. -" - -Prompt -rm -fr workspace -echo " -Install the workspace: -" -Command "handoff -p 03_exchange_rates -w workspace workspace install" - -Prompt - -echo " -Now let's run the command by pulling the configurations and files from remote. - -Try running: -" -Command "handoff -p 03_exchange_rates -w workspace run remote_config --push-artifacts" - -Prompt - -echo " -Notice that we used --push-artifacts option in - - $CMD - -We pushed the result to the bucket: - - $BUCKET_URL/$BUCKET/$TASK/last/artifacts/ - -Also note that artifacts are automatically archived at each run at - - $BUCKET_URL/$BUCKET/$TASK/runs/ -" - -Prompt - -echo Great. This concludes this section of the tutorial. - -Prompt - -Thanks - -echo " -Next step is to prepare a Docker image and test running it locally. -" - -Continue - -$PROJECT_DIR/scripts/05_docker diff --git a/handoff/test_projects/scripts/aws_get_started/01_run_local b/handoff/test_projects/scripts/aws_get_started/01_run_local new file mode 100755 index 0000000..b9b65fa --- /dev/null +++ b/handoff/test_projects/scripts/aws_get_started/01_run_local @@ -0,0 +1,151 @@ +#!/bin/bash +MODE=$1 +if [[ -z $2 ]];then HEADTAIL_LIMIT=0; else HEADTAIL_LIMIT=$2;fi +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../.." +source scripts/funcs + +echo "## Running a task locally + +In this section, we will learn how to run a task with handoff locally, +using project 01_word_count. +" + +Prompt "Ready? (Hit q at prompt to quit anytime. Any other key to proceed.)" + +echo " +Each project directory contains: +" + +ListDir "01_word_count" + +Prompt + +echo " +project.yml looks like: +" + +Cat "01_word_count/project.yml" + +Prompt + +echo ' +Here, + +- `commands` lists the commands and arguments. +- `envs` lists the environment varaibles. +' + +echo " +The example from 01_word_count runs a command line equivalent of: +" + +Code "cat ./files/the_great_dictator_speech.txt | wc -w" + +echo " +Now let's run. Try entering this command below: +" + +Command "handoff --project 01_word_count --workspace workspace run local" + +Prompt + +echo " +If you see the output that looks like: +" + +Code "\n +INFO - 2020-08-03 04:51:01,971 - handoff.config - Reading configurations from 01_word_count/project.yml\n +...\n +INFO - 2020-08-03 04:51:02,690 - handoff.config - Processed in 0:00:00.005756\n +" + +Prompt + +echo " +Then great! You just ran the first local test. It created a workspace +directory that looks like: +" + +ListDir workspace + +echo " +And the word count is stored at workspace/artifacts/state. Here is the content: +" + +Cat " workspace/artifacts/state" + +Prompt + +echo " +By the way, the example text is from the awesome speech by Charlie Chaplin's +in the movie the Great Dictator. + +Here is a link to the famous speech scene. +Check out on YouTube: https://www.youtube.com/watch?v=J7GY1Xg6X20 +" + +Prompt + +echo " +And here is the first few paragraphs of the text: +" + +Run "head -n 4 01_word_count/files/the_great_dictator_speech.txt" $BLUE 0 + +Prompt + +echo " +Now to the second example. This time project.yml looks like: +" + +Cat 02_collect_stats/project.yml + +Prompt + +echo " +...which is shell equivalent to +" + +Code "cat ./files/the_great_dictator_speech.txt | python ./files/stats_collector.py | wc -w" + +echo " +The script for the second command stats_collector.py can be found in +02_collect_stats/files directory and it is a Python script that looks like: +" + +Prompt + +Cat "02_collect_stats/files/stats_collector.py" + +echo " +The script reads from stdin and counts the lines while passing the raw input to stdout. +The raw text is then processed by the third command (wc -w) and it conts the number of words. +" + +Prompt + +echo " +Now let's run. Try entering this command below: +" + +Command "handoff --project 02_collect_stats --workspace workspace run local" $DIM 0 + +echo " +Let's check out the contents of the second command: +" + +Prompt + +Cat "workspace/artifacts/collect_stats.json" $DIM 0 + +Prompt + +Thanks + +echo " +In the next section, we will try pullin the currency exchange rate data. +You will also learn how to create Python virtual enviroments for each command +and pip-install commands. +" + +Continue scripts/aws_get_started/02_exchange_rates diff --git a/handoff/test_projects/scripts/aws_get_started/02_exchange_rates b/handoff/test_projects/scripts/aws_get_started/02_exchange_rates new file mode 100755 index 0000000..6f10275 --- /dev/null +++ b/handoff/test_projects/scripts/aws_get_started/02_exchange_rates @@ -0,0 +1,75 @@ +#!/bin/bash +MODE=$1 +if [[ -z $2 ]];then HEADTAIL_LIMIT=0; else HEADTAIL_LIMIT=$2;fi +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../.." +source scripts/funcs + +echo "## Virtual environment and install + +In this section, we will retrieve currency exchange rates and write out to CSV +file. + +We will install singer.io (https://singer.io), a data collection framework, +in Python vitual environment. +" + +Prompt "Ready? (Hit q at prompt to quit anytime. Any other key to proceed.)" + +echo " +We will use 03_exchange_rates project. project.yml looks like: +" + +Cat "03_exchange_rates/project.yml" $DIM 0 + +Prompt + +echo " +...which is shell equivalent to + + tap-exchangeratesapi | python files/stats_collector.py | target-csv +" + +Prompt + +echo ' +Before we can run this, we need to install tap-exchangeratesapi and target-csv. +The instructions for the install are listed in install section of project.yml. + +Notice `venv` entries for each command. handoff can create Python virtual +enviroment for each command to avoid conflicting dependencies among the +commands. + +To install everything, run this command: +' + +Command "handoff -p 03_exchange_rates -w workspace_03 workspace install" + +echo " +Now let's run the task. Try entering this command below: +" + +Command "handoff -p 03_exchange_rates -w workspace_03 run local" + +echo " +This process should have created a CSV file in artifacts directory: +" + +Run "ls workspace_03/artifacts | grep csv" $DIM 0 + +echo " +...which looks like: +" + +Run 'for f in `ls workspace_03/artifacts | grep csv`; do head -n 3 workspace_03/artifacts/$f; done' $DIM 3 + +Prompt + +Thanks + +echo " +Now that we know how to run locally, we will gradually thinking about how to deploy this in the cloud *severlessly*. +We will learn how to save and fetch the configurations to the remote storage. +Before doing that, we will cover how to set up AWS account and profile in the next section. +" + +Continue scripts/aws_get_started/03_set_up_aws_account diff --git a/handoff/test_projects/scripts/03_set_up_aws_account b/handoff/test_projects/scripts/aws_get_started/03_set_up_aws_account similarity index 62% rename from handoff/test_projects/scripts/03_set_up_aws_account rename to handoff/test_projects/scripts/aws_get_started/03_set_up_aws_account index 1d61af5..cc4990b 100755 --- a/handoff/test_projects/scripts/03_set_up_aws_account +++ b/handoff/test_projects/scripts/aws_get_started/03_set_up_aws_account @@ -1,35 +1,29 @@ #!/bin/bash -PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." -source $PROJECT_DIR/scripts/funcs +MODE=$1 +if [[ -z $2 ]];then HEADTAIL_LIMIT=0; else HEADTAIL_LIMIT=$2;fi +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../.." +source scripts/funcs -if [ ! -d $PROJECT_DIR/03_exchange_rates ]; then -echo " -The project does not exist...try running - handoff quick_start start -in a new directory. -" # echo - exit 1 -fi - -cd $PROJECT_DIR +echo "## Setting up an AWS account and profile -echo " -Welcome to module 03_set_up_aws_account! handoff helps you to deploy the task in the cloud computing services. -In this tutorial series, our goal is to deploy the currency exchange -rates task to AWS Fargate. https://aws.amazon.com/fargate/ -Before we can deploy, we need to set up the account and roles. +Our goal in this tutorial is to deploy the currency exchange +rates task to [AWS Fargate](https://aws.amazon.com/fargate/). + +Before we can deploy, we need to set up the AWS account. " Prompt echo " -Do I need a credit card to use AWS? How much does it cost? -Yes, you need a credit card. But it won\'t cost a cup of coffee for this tutorial. -Some of the AWS services we use comes with a Free Tier and you won\'t be +**Do I need a credit card to use AWS? How much does it cost?** + +Yes, you need a credit card. But it won't cost a cup of coffee for this tutorial. +(Probably not even a dollar.) +Some of the AWS services we use comes with a Free Tier and you won't be charged for the eligible usage. -Learn more about Free Tier: https://aws.amazon.com/free +Learn more about [Free Tier](https://aws.amazon.com/free). " Prompt @@ -50,8 +44,7 @@ Here is the complete list of the services we use: Prompt -a=`cat ~/.aws/credentials` -if [[ -f ~/.aws/credentials && ! -z "$a" ]] +if [[ ! $MODE="auto" && -f ~/.aws/credentials ]] then echo " It looks like you already have AWS CLI set up. You may just scan through @@ -105,30 +98,14 @@ Enter Access key ID and Secret access key you created in the last step. For region, use one of these keys (for USA users, us-east-1 would do): -- ap-northeast-1 -- ap-northeast-2 -- ap-south-1 -- ap-southeast-1 -- ap-southeast-2 -- ca-central-1 -- eu-central-1 -- eu-north-1 -- eu-west-1 -- eu-west-2 -- eu-west-3 -- sa-east-1 -- us-east-1 -- us-east-2 -- us-west-1 -- us-west-2 - -Learn more about AWS profile here: - https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html " -Prompt +Run 'for i in `aws ec2 describe-regions --output text | cut -f 4 | sort`; do echo "$i"; done' $RESET 0 -echo Great. This concludes this section of the tutorial. +echo " +Learn more about AWS profile +[here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) +" Prompt @@ -136,9 +113,7 @@ Thanks echo " Now that we have set up the AWS account, let's store the configurations -in the cloud. +in the cloud in the next section. " -Continue - -$PROJECT_DIR/scripts/04_run_remote_config +Continue scripts/aws_get_started/04_run_remote_config diff --git a/handoff/test_projects/scripts/aws_get_started/04_run_remote_config b/handoff/test_projects/scripts/aws_get_started/04_run_remote_config new file mode 100755 index 0000000..60af0fc --- /dev/null +++ b/handoff/test_projects/scripts/aws_get_started/04_run_remote_config @@ -0,0 +1,169 @@ +#!/bin/bash +MODE=$1 +if [[ -z $2 ]];then HEADTAIL_LIMIT=0; else HEADTAIL_LIMIT=$2;fi +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../.." +source scripts/funcs + +echo "## Running task locally with remotely stored configurations + +We will continue using 03_exchange_rates example. But this time, we will +store the configurations to the remote data store. +" + +Prompt + +CheckAws + +ChooseAwsProfile + +echo " +Let's review how 03_exchange_rates project directory is structured: +" + +ListDir "03_exchange_rates" + +Prompt + +echo " +In the config directory, we have a couple of configuration files: +" +ListDir "03_exchange_rates/config" + +Prompt + +echo " +...which look like: +" + +Cat "03_exchange_rates/config/tap-config.json" $DIM 0 + +Cat "03_exchange_rates/config/target-config.json" $DIM 0 + +echo " +Often times, such configuration files contain sensitive information. +So we push the configrations to a secure key store such as +AWS Systems Manager (SSM) Parameter Store (In SecuresString format) +handoff wraps this proces by a single command. handoff packs all the JSON +format files under the configuration directory and send it to the remote +key-value storage. + +Try running: +" + +Command "handoff -p 03_exchange_rates config push" + +echo ' +Look at the end of the log that says, + +``` + See the parameters at https://console.aws.amazon.com/systems-manager/parameters/?region=... +``` + +Grab the link and open in the browswer (AWS login required) to confirm that +the parameters are uploaded. +' + +Prompt + +echo " +We also have some files needed for the task execution: +" + +ListDir "03_exchange_rates/files" + +echo " +We need to store this somewhere accessible. +So we will create a cloud data storage (S3 bucket) for that. + +Try running: +" + +Command "handoff -p 03_exchange_rates cloud create_bucket" + +echo ' +Wait for a minute and check here + +``` + https://s3.console.aws.amazon.com +``` + +to make sure the bucket is created before you proceed. +' + +if [[ $MODE = "auto" ]]; then sleep 3m; fi + +Prompt + +echo " +Now it's time to push the files to the bucket. + +Try running: +" + +Command "handoff -p 03_exchange_rates files push" + +echo ' +Look at the end of the log that says, + +``` + See the files at https://s3.console.aws.amazon.com/s3/... +``` + +Grab the link and open in the browswer (AWS login required) to see the files +directory is uploaded. +' + +Prompt + +echo " +Install the workspace: +" + +Command "handoff -p 03_exchange_rates -w workspace workspace install" + +echo " +Now let's run the command by pulling the configurations and files from remote. + +Try running: +" +Command "handoff -p 03_exchange_rates -w workspace run remote_config --push-artifacts" + +echo ' +Notice that we used --push-artifacts option in the last command. +With this option, we pushed the result to the bucket under + +```' + +echo " $TASK/last/artifacts" + +echo ' +``` + +directory. +' + +echo ' +Also note that artifacts are automatically archived at each run at + +``` +' + +echo " + $BUCKET/$TASK/runs/ +" + +echo ' +``` + +directory. +' + +Prompt + +Thanks + +echo " +Next step is to prepare a Docker image and test running it locally. +" + +Continue scripts/aws_get_started/05_docker diff --git a/handoff/test_projects/scripts/05_docker b/handoff/test_projects/scripts/aws_get_started/05_docker similarity index 75% rename from handoff/test_projects/scripts/05_docker rename to handoff/test_projects/scripts/aws_get_started/05_docker index 3ae7fcc..a5607bd 100755 --- a/handoff/test_projects/scripts/05_docker +++ b/handoff/test_projects/scripts/aws_get_started/05_docker @@ -1,18 +1,10 @@ #!/bin/bash -PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." -source $PROJECT_DIR/scripts/funcs - -if [ ! -d $PROJECT_DIR/03_exchange_rates ]; then - echo The project does not exist...try running - echo " handoff quick_start start" - echo in a new directory. - exit 1 -fi - -cd $PROJECT_DIR +MODE=$1 +if [[ -z $2 ]];then HEADTAIL_LIMIT=0; else HEADTAIL_LIMIT=$2;fi +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../.." +source scripts/funcs -echo " -Welcome to module 05_docker! +echo "## Building, running, and pushing a Docker image We will continue using 03_exchange_rates example. Instead of running the task on the host machine, let's run on Docker. @@ -45,29 +37,41 @@ Let's build a Docker image. Try running the following command. Enter y when prompted at the beginning. The build may take 5~10 minutes. " + Command "handoff -p 03_exchange_rates container build" echo " Now let's run the code in the Docker container. " + Command "handoff -p 03_exchange_rates container run" BUCKET_URL=https://s3.console.aws.amazon.com/s3/buckets BUCKET=`handoff -p 03_exchange_rates envs get -d '{"key":"HO_BUCKET"}' -l critical` TASK=`handoff -p 03_exchange_rates envs get -d '{"key":"HO_TASK"}' -l critical` -echo " +echo ' Confirm the run by checking the logs. Also check the artifacts on S3: +``` +' - $BUCKET_URL/$BUCKET/$TASK/last/artifacts/ +echo " + $BUCKET/$TASK/last/artifacts/ " +echo ' +``` + +directory. +' + Prompt echo " Now that we know the Docker container runs fine, let's push it to AWS Elastic Container Registry. This may take a few minutes. " + Command "handoff -p 03_exchange_rates container push" REGION=`handoff -p 03_exchange_rates envs get -d '{"key":"AWS_REGION"}' -l critical` @@ -79,10 +83,6 @@ https://console.aws.amazon.com/ecr/repositories?region=$REGION Prompt -echo Great. This concludes this section of the tutorial. - -Prompt - Thanks echo " @@ -90,6 +90,4 @@ Now that the Docker image is prepared, we will finally deploy the task on AWS Fargate in the next tutorial. " -Continue - -$PROJECT_DIR/scripts/06_fargate +Continue scripts/aws_get_started/06_fargate diff --git a/handoff/test_projects/scripts/aws_get_started/06_fargate b/handoff/test_projects/scripts/aws_get_started/06_fargate new file mode 100755 index 0000000..6f66540 --- /dev/null +++ b/handoff/test_projects/scripts/aws_get_started/06_fargate @@ -0,0 +1,140 @@ +#!/bin/bash +MODE=$1 +if [[ -z $2 ]];then HEADTAIL_LIMIT=0; else HEADTAIL_LIMIT=$2;fi +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../.." +source scripts/funcs + +echo "## Deploying on AWS Fargate + +We will finally deploy the task to AWS Fargate so we can automate the +recurring task. +" + +CheckAws + +if [[ -z "$AWS_PROFILE" ]] +then +ChooseAwsProfile +fi + +echo " +Fargate is part of AWS Elastic Container Service (ECS). +The Fargate task run on the ECS cluster. +We will first need to create a cluster. +(You will only be charged for the usage.) + +To make this process easy, handoff packed everything up in a command: +" + +Command "handoff -p 03_exchange_rates cloud create_resources" + +echo ' + +At the end of the log, you should see a line like: + +``` + + Check the progress at https://console.aws.amazon.com/cloudformation/home?region=xxxx + +``` + +Grab the entire link and open in a browser (you need to login in to AWS) to see +the progress of the resource creation. + +Wait until it says \"CREATE_COMPLETE\" +' + +if [[ $MODE = "auto" ]]; then sleep 3m; fi + +Prompt + +echo " +Now it's time to deploy the task and here is the command: +" + +Command "handoff -p 03_exchange_rates cloud create_task" + +echo ' + +Here again, at the end of the log, you should see a line like: + +``` + + Check the progress at https://console.aws.amazon.com/cloudformation/home?region=xxxx + +``` + +Grab the entire link and open in a browser. + +Wait until it says "CREATE_COMPLETE" +' + +if [[ $MODE = "auto" ]]; then sleep 3m; fi + +Prompt + +echo " +Once the task is created, try running on Fargate. +To do so, run this command: +" + +Command "handoff -p 03_exchange_rates cloud run" + +echo ' +At the end of the log, you should see a line like: + +``` + + Check the task at https://us-east-1.console.aws.amazon.com/ecs/home?region=xxxx + +``` + +Grab the entire link and open in a browser. + +At the top you see: + Task : 5a232313-e390.... + +In Details tab, + +Cluster handoff-test-test-03-exchange-rates +Launch type FARGATE +... +Last status PROVISIONING +Desired status RUNNING +' + +Prompt + +echo ' +At the bottom of the page, you see: + +| Name | Container Runtime ID | Status | Image | ..... | +| :---------------- | :------------------- | :----------- | :---- | :---- | +| singer_excha... | 1371.... | PROVISIONING | xxxxx | ..... | + +Expand the section by clicking on a dark side-way trible ">" by the Task name. + +The status should change in this order: + +1. PROVIONING +2. RUNNING +3. STOPPED + +Once the status becomes RUNNING or STOPPED, you should be able to see +the execution log by clicking "View logs in CloudWatch" link at the bottom +of the section. + +The log should be similar to the output from the local execution. +' + +Prompt + +Thanks + +echo " +We confirmed that the task run in the same way as local execution. +Now let's go to the next section to learn how to schedule the task +so it runs periodically. +" + +Continue scripts/aws_get_started/07_schedule diff --git a/handoff/test_projects/scripts/aws_get_started/07_schedule b/handoff/test_projects/scripts/aws_get_started/07_schedule new file mode 100755 index 0000000..1312e0d --- /dev/null +++ b/handoff/test_projects/scripts/aws_get_started/07_schedule @@ -0,0 +1,60 @@ +#!/bin/bash +MODE=$1 +if [[ -z $2 ]];then HEADTAIL_LIMIT=0; else HEADTAIL_LIMIT=$2;fi +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../.." +source scripts/funcs + +echo "## Scheduling a task on Fargate + +In this section, we will learn how to schedule a task so it runs automatically. +" + +CheckAws + +if [[ -z "$AWS_PROFILE" ]] +then +ChooseAwsProfile +fi + +echo ' +To schedule a task, use schedule command with target_id and +[CRON format]://en.wikipedia.org/wiki/Cron). + +We pass those values to handoff with `--data` (`-d` for short) option: +' + +Prompt "The following command is set to trigger the task 5 minutes later from now:" + +hour=`date -u +"%H"` +min=`date -u +"%M"` +min=$(($min + 5)) +Command "handoff -p 03_exchange_rates cloud schedule --data '{\"target_id\": \"1\", \"cron\": \"$min $hour * * ? *\"}'" + +echo ' + +At the end of the log, you should see a line like: + +``` + + Check the progress at Check the status at https://console.aws.amazon.com/ecs/home?region=us-east-1#/clusters/... + +``` +' + +echo " +Grab the entire link and open in a browser (you need to login in to AWS) to see +it's scheduled. +" + + +Prompt + +Thanks + +echo " +We confirmed that the task run in the same way as local execution. +Now let's go to the next module to learn how to schedule the task +so it runs periodically. +" + +Continue scripts/aws_get_started/08_cleanup diff --git a/handoff/test_projects/scripts/aws_get_started/08_cleanup b/handoff/test_projects/scripts/aws_get_started/08_cleanup new file mode 100755 index 0000000..5f8210a --- /dev/null +++ b/handoff/test_projects/scripts/aws_get_started/08_cleanup @@ -0,0 +1,74 @@ +#!/bin/bash +MODE=$1 +if [[ -z $2 ]];then HEADTAIL_LIMIT=0; else HEADTAIL_LIMIT=$2;fi +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../.." +source scripts/funcs + +echo "## Cleaning up + +Let's clean everything up so we won't pay a fraction of penny after forgeting about this exercise. + +First unschedule the task: +" + +Command "handoff -l warning -p 03_exchange_rates cloud unschedule -d '{\"target_id\": 1}'" + +echo " +Then delete the task: +" + +Command "handoff -l warning -p 03_exchange_rates cloud delete_task" + +echo " +If there is no other task in the same resource group, we can delete it: +" + +Command "handoff -l warning -p 03_exchange_rates cloud delete_resources" + +echo " +Here is how to delete the configurations from SSM Parameter Store: +" + +Command "handoff -p 03_exchange_rates config delete" + + +echo " +Here is how to delete the files from S3 bucket: +" + +Command "handoff -p 03_exchange_rates files delete" + +echo " +If there is no other task in the same resource group, we can delete the bucket, too: +" + +Command "handoff -l warning -p 03_exchange_rates cloud delete_bucket" + +BUCKET=s3://`handoff -p 03_exchange_rates envs get -d '{"key":"HO_BUCKET"}' -l critical` + +echo " +The previous command only deleted the CloudFormation stack, but not the bucket itself. +Here is how to delete all the files in $BUCKET bucket. This cannot be reversed: +" + +Command "aws s3 rm --recursive $BUCKET/" + +echo " +Here is how to delete $BUCKET bucket. The bucket must be empty. This cannot be reversed: +" + +Command "aws s3 rb $BUCKET" + +REPO=`handoff -p 03_exchange_rates envs get -d '{"key":"HO_DOCKER_IMAGE"}' -l critical` +echo " +Now delete $REPO repository from ECR. This cannot be reversed. +--force option will ignore that we still have images in the repository. +" + +Command "aws ecr delete-repository --repository-name $REPO --force" + +Command "handoff -l warning -p 03_exchange_rates cloud delete_role" + +echo " +That's all. +" diff --git a/handoff/test_projects/scripts/funcs b/handoff/test_projects/scripts/funcs index fe859bc..0129617 100755 --- a/handoff/test_projects/scripts/funcs +++ b/handoff/test_projects/scripts/funcs @@ -8,23 +8,82 @@ export YELLOW="\033[33m" export BLINK="\033[5m" export DIM="\033[2m" +QUOTE='```' +PROMPT_STYLE="$GREEN $BLINK" +PROMPT_CHAR="> " + +ListDir() { +if [[ -z $2 ]]; then STYLE=$DIM; else STYLE=$2; fi +if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; fi +echo "> ls -l $1" +if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi +if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; fi +echo +for i in `ls $1`; do echo " $i"; done +if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi +} + +Cat() { +if [[ -z $2 ]]; then STYLE=$DIM; else STYLE=$2; fi +if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; fi +echo "> cat $1" +if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi +echo +if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; fi +cat $1 +if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi +} + +Run() { +if [[ -z $2 ]]; then STYLE=$DIM; else STYLE=$2; fi +if [[ ! -z $3 ]];then h=$3; else if [[ ! -z $HEADTAIL_LIMIT ]]; then h=$HEADTAIL_LIMIT; else h=0; fi; fi +if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; echo; fi +if [[ $h -lt 1 ]]; then eval "$1"; else eval "$1" | (sed -u ${h}q; t=`tail -n $h`; if [[ `echo "$t" | wc -l` -gt 1 ]]; then echo .; echo .; echo .; echo "$t"; fi); fi +if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi +} + +Command() { +if [[ ! $MODE = "auto" ]]; then + echo -e $GREEN $1$RESET + echo ' Hint: type "skip" to skip. "lazy" to run it without typing the command. type "quit" exit this tutorial.' + echo + read -p "> " + while [[ ! $REPLY = $1 && ! $REPLY = "skip" && ! $REPLY = "lazy" && ! $REPLY = "quit" ]] + do + echo Hmm...you did not enter correctly, try again. + read -p "> " + done +fi +if [[ $REPLY = "quit" ]]; then + echo "bye!" + exit 0 +fi +if [[ $MODE = "auto" || $REPLY = $1 || $REPLY = "lazy" ]]; then + echo $QUOTE + echo "> $1" + echo $QUOTE + Run "$1" "$2" "$3" +fi +} + +Code() { +if [[ -z $2 ]]; then STYLE=$DIM; else STYLE=$2; fi +if [[ ! $MODE = "auto" ]]; then echo -e $STYLE$QUOTE; else echo $QUOTE; fi +echo -e $1 +if [[ ! $MODE = "auto" ]]; then echo -e $QUOTE$RESET; else echo $QUOTE; fi +} + Prompt() { -echo -e $GREEN $BLINK; read -p "> " -n 1 -r; echo -e $RESET # Prompt +if [[ ! $MODE = "auto" ]]; then +echo -e $1$PROMPT_STYLE; read -p $PROMPT_CHAR -n 1 -r; echo -e $RESET if [[ $REPLY =~ ^[Qq]$ ]] then echo "bye!" exit 0 fi -} - -ListDir() { -echo -e $2 -for i in `ls $1`; do echo - $i; done -echo -e $RESET -} - -Cat() { -echo -e $2; cat $1; echo -e $RESET +else +echo +fi } Choose() { @@ -44,18 +103,6 @@ echo CHOICE=$(($REPLY-1)) } -Command() { -echo -e $GREEN $1$RESET -echo -read -p "> " -while [[ ! $REPLY = $1 ]] -do - echo Hmm...you did not enter correctly, try again. - read -p "> " -done -echo -e $DIM; $1; echo -e $RESET -} - CheckAws() { a=`cat ~/.aws/credentials` if [[ ! -f ~/.aws/credentials ]] @@ -91,7 +138,10 @@ fi } Thanks() { +if [[ ! $MODE = "auto" ]]; then echo " +Great! This concludes this section of the tutorial. + Thanks for completing this section of tutorial. Oh, before you go, do you use Twitter? Can you do me a favor and let your fellow engineers know about handoff? " @@ -107,9 +157,11 @@ else echo "OK! In case you change your mind, here is the link to auto-draft your tweets (you can edit before you tweet)" echo -e $GREEN$URL$RESET fi +fi } Continue () { +if [[ ! $MODE = "auto" ]]; then echo -e $GREEN; read -p "Continue? (Y/n) " -n 1 -r; echo -e $RESET # Prompt echo # (optional) move to a new line if [[ $REPLY =~ ^[Nn]$ ]] @@ -120,5 +172,8 @@ then echo echo "bye!" exit 0 +else +$1 +fi fi } diff --git a/handoff/test_projects/start b/handoff/test_projects/start new file mode 100755 index 0000000..c8da432 --- /dev/null +++ b/handoff/test_projects/start @@ -0,0 +1,15 @@ +#!/bin/bash +MODE=$1 +if [[ -z $2 ]];then HEADTAIL_LIMIT=0; else HEADTAIL_LIMIT=$2;fi +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source scripts/funcs + +echo "Welcome to handoff tutorial +Each module is very short (5 ~ 10 minutes to complete) +" + +CMD="ls scripts/aws_get_started | grep -v funcs | grep -v role" + +Choose "$CMD" + +scripts/aws_get_started/${OPTIONS[$CHOICE]} diff --git a/tests/remote_config/test_03_exchange_rates.py b/tests/remote_config/test_03_exchange_rates.py index f5bf102..582d694 100644 --- a/tests/remote_config/test_03_exchange_rates.py +++ b/tests/remote_config/test_03_exchange_rates.py @@ -32,9 +32,12 @@ def test_03_exchange_rates(): with tempfile.TemporaryDirectory() as root_dir: project_dir = os.path.join(root_dir, "project") shutil.copytree(orig_project_dir, project_dir) - with open(os.path.join(project_dir, CONFIG_DIR, "tap-config.json"), "w") as f: + with open(os.path.join(project_dir, CONFIG_DIR, "tap-config.json"), + "w") as f: config = {"base": "JPY", - "start_date": (datetime.datetime.now() - datetime.timedelta(days=7)).isoformat()[:10]} + "start_date": (datetime.datetime.now() - + datetime.timedelta(days=7) + ).isoformat()[:10]} json.dump(config, f) workspace_dir = os.path.join(root_dir, "workspace") @@ -51,11 +54,11 @@ def test_03_exchange_rates(): handoff.do("run", "remote_config", None, workspace_dir, data, push_artifacts=True) - handoff.do("files", "delete", None, workspace_dir, data, + handoff.do("files", "delete", project_dir, None, data, push_artifacts=False) - handoff.do("artifacts", "delete", None, workspace_dir, data, + handoff.do("artifacts", "delete", project_dir, None, data, push_artifacts=False) - handoff.do("config", "delete", None, workspace_dir, data, + handoff.do("config", "delete", project_dir, None, data, push_artifacts=False) files = os.listdir(os.path.join(workspace_dir, ARTIFACTS_DIR)) @@ -64,12 +67,14 @@ def test_03_exchange_rates(): if fn[:len("exchange_rate-")] == "exchange_rate-": rate_file = fn break - with open(os.path.join(workspace_dir, ARTIFACTS_DIR, rate_file), "r") as f: + with open(os.path.join(workspace_dir, ARTIFACTS_DIR, rate_file), + "r") as f: rows = csv.DictReader(f) for row in rows: - jpy = row["JPY"] - assert(float(jpy)==1.0) + jpy = row["JPY"] + assert(float(jpy) == 1.0) - with open(os.path.join(workspace_dir, ARTIFACTS_DIR, "collect_stats.json"), "r") as f: + with open(os.path.join(workspace_dir, ARTIFACTS_DIR, + "collect_stats.json"), "r") as f: stats = json.load(f) assert(stats.get("rows_read") is not None)