diff --git a/dcicutils/_version.py b/dcicutils/_version.py index 871f68450..ec563719b 100644 --- a/dcicutils/_version.py +++ b/dcicutils/_version.py @@ -1,4 +1,4 @@ """Version information.""" # The following line *must* be the last in the module, exactly as formatted: -__version__ = "0.6.4" +__version__ = "0.7.0" diff --git a/dcicutils/beanstalk_utils.py b/dcicutils/beanstalk_utils.py index ad39800f2..b0a67baad 100755 --- a/dcicutils/beanstalk_utils.py +++ b/dcicutils/beanstalk_utils.py @@ -1,83 +1,66 @@ -''' -given and env in beanstalk do the follow -2. backup database -1. clone the existing environment to new beanstalk - eb clone -3. set env variables on new beanstalk to point to database backup -4. for each s3bucket in existing environment: - a. create new bucket with proper naming - b. move files from existing bucket to new bucket -5. new ES instance? (probably not covered by this script yet) +'''Utilities related to ElasticBeanstalk deployment and management. +This includes, but is not limited to: ES, s3, RDS, Auth0, and Foursight. ''' from __future__ import print_function import subprocess import logging -import argparse import boto3 import os import json import requests import time +from datetime import datetime from dcicutils import ff_utils from botocore.exceptions import ClientError +logging.basicConfig() logger = logging.getLogger('logger') logger.setLevel(logging.INFO) +# input vs. raw_input for python 2/3 +try: + use_input = raw_input +except NameError: + use_input = input + FOURSIGHT_URL = 'https://foursight.4dnucleome.org/api/' +# magic CNAME corresponds to data.4dnucleome +MAGIC_CNAME = 'fourfront-webprod.9wzadzju3p.us-east-1.elasticbeanstalk.com' GOLDEN_DB = "fourfront-webprod.co3gwj7b7tpq.us-east-1.rds.amazonaws.com" REGION = 'us-east-1' -# TODO: Maybe -''' -class EnvConfigData(OrderedDictionary): - - def to_aws_bs_options() - - def get_val_for_env() - - def is_data() - - def is_staging() - - def url() - - def bucket() - - def buckets() - - def part(self, componenet_name): - self.get(componenet_name) - - def db(self): - return self.part('db') - - def es(self): - return self.part('es') - - def foursight(self): - return self.part('foursight') - - def higlass(self): - return self.part('higlass') -''' - class WaitingForBoto3(Exception): pass -def delete_db(db_identifier, take_snapshot=True): - client = boto3.client('rds', region_name=REGION) +def delete_db(db_identifier, take_snapshot=True, allow_delete_prod=False): + """ + Given db_identifier, delete an RDS instance. If take_snapshot is true, + will create a final snapshot named "-final-". + + Args: + db_identifier (str): name of RDS instance + take_snapshot (bool): If True, take a final snapshot before deleting + allow_delete_prod (bool): Must be True to allow deletion of 'webprod' DB + + Returns: + dict: boto3 response from delete_db_instance + """ + # safety. Do not allow accidental programmatic deletion of webprod DB + if 'webprod' in db_identifier and not allow_delete_prod: + raise Exception('Must set allow_delete_prod to True to delete RDS instance' % db_identifier) + client = boto3.client('rds') + timestamp = datetime.strftime(datetime.utcnow(), "%Y-%m-%d") if take_snapshot: try: resp = client.delete_db_instance( DBInstanceIdentifier=db_identifier, SkipFinalSnapshot=False, - FinalDBSnapshotIdentifier=db_identifier + "-final" + FinalDBSnapshotIdentifier=db_identifier + "-final-" + timestamp ) except: # noqa: E722 - # try without the snapshot + # Snapshot cannot be made. Likely a date conflict resp = client.delete_db_instance( DBInstanceIdentifier=db_identifier, SkipFinalSnapshot=True, @@ -88,107 +71,160 @@ def delete_db(db_identifier, take_snapshot=True): SkipFinalSnapshot=True, ) print(resp) + return resp -def get_health_page_info(bs_url): - """ - Different use cases than ff_utils.get_health_page (that one is oriented - towards external API usage and this one is more internal) +def get_es_from_bs_config(env): """ - if not bs_url.endswith('/'): - bs_url += "/" - if not bs_url.startswith('http'): - bs_url = 'http://' + bs_url - - health_res = requests.get(bs_url + 'health?format=json') - return health_res.json() - + Given an ElasticBeanstalk environment name, get the corresponding + Elasticsearch url from the EB configurationock -# TODO: think about health page query parameter to get direct from config -def get_es_from_health_page(bs_url): - health = get_health_page_info(bs_url) - es = health['elasticsearch'].strip(':80') - return es + Args: + env (str): ElasticBeanstalk environment name - -def get_es_from_bs_config(env): + Returns: + str: Elasticsearch url without port info + """ bs_env = get_bs_env(env) for item in bs_env: if item.startswith('ES_URL'): return item.split('=')[1].strip(':80') -def is_indexing_finished(bs, version=None): - is_beanstalk_ready(bs) - bs_url = get_beanstalk_real_url(bs) +def is_indexing_finished(env, prev_version=None, travis_build_id=None): + """ + Checker function used with torb waitfor lambda; output must be standarized. + Check to see if indexing of a Fourfront environment corresponding to given + ElasticBeanstalk environment is finished by looking at the /counts page. + + Args: + env (str): ElasticBeanstalk environment name + prev_version (str): optional EB version of the previous configuration + travis_build_id (int): optional ID for a Travis build + + Returns: + bool, list: True if done, results from /counts page + + Raises: + Exception: if Travis done and bad EB environment VersionLabel + WaitingForBoto3: on a retryable waitfor condition + """ + # is_beanstalk_ready will raise WaitingForBoto3 if not ready + is_beanstalk_ready(env) + bs_url = get_beanstalk_real_url(env) if not bs_url.endswith('/'): bs_url += "/" - # server not up yet - try: - # check to see if our version is updated - if version: - info = beanstalk_info(bs) - if version == info.get('VersionLabel'): - raise Exception("Beanstalk version has not updated from %s" % version) - health_res = ff_utils.authorized_request(bs_url + 'counts?format=json', ff_env=bs) - totals = health_res.json().get('db_es_total').split() + # retry if the beanstalk version has not updated from previous version, + # unless the travis build has failed (in which case it will never update). + # If failed, let is_indexing continue as usual so the deployment will + # complete, despite being on the wrong EB application version + if prev_version and beanstalk_info(env).get('VersionLabel') == prev_version: + if travis_build_id: + try: + trav_done, trav_details = is_travis_finished(travis_build_id) + except Exception as exc: + # if the build failed, let the indexing check continue + if 'Build Failed' in str(exc): + logger.info("EB version has not updated from %s." + "Associated travis build %s has failed." + % (prev_version, travis_build_id)) + else: + raise WaitingForBoto3("EB version has not updated from %s. " + "Encountered error when getting build" + " %s from Travis. Error: %s" + % (prev_version, travis_build_id, exc)) + else: + # Travis build is running/has passed + if trav_done is True: + logger.info("EB version has not updated from %s." + "Associated travis build %s has finished." + % (prev_version, travis_build_id)) + else: + raise WaitingForBoto3("EB version has not updated from %s." + "Associated travis build is %s and " + "has not yet finished. Details: %s" + % (prev_version, travis_build_id, trav_details)) + else: + # no build ID provided; must retry on not updated version + raise WaitingForBoto3("EB version has not updated from %s" + % prev_version) + + # check counts from the portal to determine indexing state + try: + counts_res = ff_utils.authorized_request(bs_url + 'counts?format=json', + ff_env=env) + totals = counts_res.json().get('db_es_total').split() - # DB: 74048 ES: 74048 parse totals + # example value of split totals: ["DB:", "74048", "ES:", "74048"] db_total = totals[1] es_total = totals[3] if int(db_total) > int(es_total): - status = False + is_ready = False else: - status = True - except Exception as e: - print(e) - status = False - totals = 0 - - return status, totals + is_ready = True + except Exception as exc: + logger.info('Error on is_indexing_finished: %s' % exc) + is_ready = False + totals = [] + return is_ready, totals def swap_cname(src, dest): - # TODO clients should be global functions - client = boto3.client('elasticbeanstalk', region_name=REGION) + """ + Swap the CNAMEs of two ElasticBeanstalk (EB) environments, given by + src and dest. Will restart the app servers after swapping. + Should be used for swapping production/staging environments. + Args: + src (str): EB environment name of production + dest (str): EB environment name of staging + + Returns: + None + """ + client = boto3.client('elasticbeanstalk', region_name=REGION) + print("Swapping CNAMEs %s and %s..." % (src, dest)) client.swap_environment_cnames(SourceEnvironmentName=src, DestinationEnvironmentName=dest) - import time - print("waiting for swap environment cnames") + print("Giving CNAMEs 10 seconds to update...") time.sleep(10) + print("Restarting app servers for %s and %s..." % (src, dest)) client.restart_app_server(EnvironmentName=src) client.restart_app_server(EnvironmentName=dest) def whodaman(): ''' - determines which evironment is currently hosting data.4dnucleome.org - ''' - magic_cname = 'fourfront-webprod.9wzadzju3p.us-east-1.elasticbeanstalk.com' + Determines which ElasticBeanstalk environment is currently hosting + data.4dnucleome.org. Requires IAM permissions for EB! + Returns: + str: EB environment name hosting data.4dnucleome + ''' client = boto3.client('elasticbeanstalk', region_name=REGION) res = describe_beanstalk_environments(client, ApplicationName="4dn-web") logger.info(res) for env in res['Environments']: logger.info(env) - if env.get('CNAME') == magic_cname: + if env.get('CNAME') == MAGIC_CNAME: # we found data return env.get('EnvironmentName') -def beanstalk_config(env, appname='4dn-web'): - client = boto3.client('elasticbeanstalk', region_name=REGION) - return client.describe_configuration_settings(EnvironmentName=env, - ApplicationName=appname) +def beanstalk_info(env): + """ + Describe a ElasticBeanstalk environment given an environment name + Args: + env (str): ElasticBeanstalk environment name -def beanstalk_info(env): + Returns: + dict: Environments result from describe_beanstalk_environments + """ client = boto3.client('elasticbeanstalk', region_name=REGION) res = describe_beanstalk_environments(client, EnvironmentNames=[env]) - return res['Environments'][0] @@ -196,6 +232,12 @@ def get_beanstalk_real_url(env): """ Return the real url for the elasticbeanstalk with given environment name. Name can be 'data', 'staging', or an actual environment. + + Args: + env (str): ElasticBeanstalk environment name + + Returns: + str: url of the ElasticBeanstalk environment """ url = '' urls = {'staging': 'http://staging.4dnucleome.org', @@ -219,21 +261,43 @@ def get_beanstalk_real_url(env): def is_beanstalk_ready(env): + """ + Checker function used with torb waitfor lambda; output must be standarized. + Check to see if a ElasticBeanstalk environment status is "Ready" + + Args: + env (str): ElasticBeanstalk environment name + + Returns: + bool, str: True if done, ElasticBeanstalk url + + Raises: + WaitingForBoto3: if EB environment status != "Ready" + """ client = boto3.client('elasticbeanstalk', region_name=REGION) res = describe_beanstalk_environments(client, EnvironmentNames=[env]) status = res['Environments'][0]['Status'] if status != 'Ready': - raise WaitingForBoto3("Beanstalk enviornment status is %s" % status) + raise WaitingForBoto3("Beanstalk environment status is %s" % status) - return status, 'http://' + res['Environments'][0].get('CNAME') + return True, 'http://' + res['Environments'][0].get('CNAME') def describe_beanstalk_environments(client, **kwargs): """ Generic function for retrying client.describe_environments to avoid - AWS throttling errors - Passes all given kwargs to client.describe_environments + AWS throttling errors. Passes all given kwargs to describe_environments + + Args: + client (botocore.client.ElasticBeanstalk): boto3 client + + Returns: + dict: response from client.describe_environments + + Raises: + Exception: if a non-ClientError exception is encountered during + describe_environments or cannot complete within retry framework """ env_info = kwargs.get('EnvironmentNames', kwargs.get('ApplicationName', 'Unknown environment')) for retry in [1, 1, 1, 1, 2, 2, 2, 4, 4, 6, 8, 10, 12, 14, 16, 18, 20]: @@ -251,39 +315,81 @@ def describe_beanstalk_environments(client, **kwargs): def is_snapshot_ready(snapshot_name): + """ + Checker function used with torb waitfor lambda; output must be standarized. + Check to see if an RDS snapshot with given name is available + + Args: + snapshot_name (str): RDS snapshot name + + Returns: + bool, str: True if done, identifier of snapshot + """ client = boto3.client('rds', region_name=REGION) resp = client.describe_db_snapshots(DBSnapshotIdentifier=snapshot_name) - status = resp['DBSnapshots'][0]['Status'] - return status.lower() == 'available', resp['DBSnapshots'][0]['DBSnapshotIdentifier'] + db_status = resp['DBSnapshots'][0]['Status'] + is_ready = db_status.lower() == 'available' + return is_ready, resp['DBSnapshots'][0]['DBSnapshotIdentifier'] def is_es_ready(es_name): + """ + Checker function used with torb waitfor lambda; output must be standarized. + Check to see if an ES instance is ready and has an endpoint + + Args: + es_name (str): ES instance name + + Returns: + bool, str: True if done, ES url + """ es = boto3.client('es', region_name=REGION) describe_resp = es.describe_elasticsearch_domain(DomainName=es_name) endpoint = describe_resp['DomainStatus'].get('Endpoint', None) - status = True + is_ready = True if endpoint is None: - status = False + is_ready = False else: endpoint = endpoint + ":80" - return status, endpoint + return is_ready, endpoint + +def is_db_ready(db_identifier): + """ + Checker function used with torb waitfor lambda; output must be standarized. + Check to see if an RDS instance with given name is ready + + Args: + db_identifier (str): RDS instance identifier -def is_db_ready(snapshot_name): + Returns: + bool, str: True if done, RDS address + """ client = boto3.client('rds', region_name=REGION) - status = False - resp = client.describe_db_instances(DBInstanceIdentifier=snapshot_name) + is_ready = False + resp = client.describe_db_instances(DBInstanceIdentifier=db_identifier) details = resp endpoint = resp['DBInstances'][0].get('Endpoint') if endpoint and endpoint.get('Address'): - print("we got an endpoint:", endpoint['Address']) details = endpoint['Address'] - status = True + is_ready = True - return status, details + return is_ready, details def create_db_snapshot(db_identifier, snapshot_name): + """ + Given an RDS instance indentifier, create a snapshot using the given name. + If a snapshot with given name already exists, attempt to delete and return + "Deleting". Otherwise, return snapshot ARN. + + Args: + db_identifier (str): RDS instance identifier + snapshot_name (str): identifier/ARN of RDS snapshot to create + + Returns: + str: resource ARN if successful, otherwise "Deleting" + """ client = boto3.client('rds', region_name=REGION) try: response = client.create_db_snapshot( @@ -300,92 +406,112 @@ def create_db_snapshot(db_identifier, snapshot_name): return response -def create_db_from_snapshot(db_name, snapshot_name=None): - if not snapshot_name: - snapshot_name = db_name +def create_db_from_snapshot(db_identifier, snapshot_name, delete_db=True): + """ + Given an RDS instance indentifier and a snapshot ARN/name, create an RDS + instance from the snapshot. If an instance already exists with the given + identifier and delete_db is True, attempt to delete and return "Deleting". + Otherwise, return instance ARN. + + Args: + db_identifier (str): RDS instance identifier + snapshot_name (str): identifier/ARN of RDS snapshot to restore from + + Returns: + str: resource ARN if successful, otherwise "Deleting" + """ client = boto3.client('rds', region_name=REGION) try: response = client.restore_db_instance_from_db_snapshot( - DBInstanceIdentifier=db_name, - DBSnapshotIdentifier=snapshot_name, - DBInstanceClass='db.t2.medium') - except ClientError: - # drop target database no backup - try: - delete_db(db_name, True) - except ClientError: - pass - return "Deleting" + DBInstanceIdentifier=db_identifier, + DBSnapshotIdentifier=snapshot_name, + DBInstanceClass='db.t2.medium', + ) + except ClientError: + # Something went wrong + # Even if delete_db, never allow deletion of a db with 'webprod' in it + if delete_db: + # Drop target database with final snapshot + try: + delete_db(db_identifier, True) + except ClientError: + pass + return "Deleting" + else: + return "Error" return response['DBInstance']['DBInstanceArn'] +def is_travis_started(request_url): + """ + Checker function used with torb waitfor lambda; output must be standarized. + Check the requests url to see if a given build has stared and been issued + a build id, which can in turn be used for is_travis_finished + + Args: + request_url (str): Travis request url + + Returns: + bool, dict: True if started, Travis response JSON + + Raises: + Exception: if Travis key not in environ + """ + if 'travis_key' not in os.environ: + raise Exception('Must have travis_key environment variable defined') + is_ready = False + details = 'requested build has not started' + headers = {'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Travis-API-Version': '3', + 'User-Agent': 'tibanna/0.1.0', + 'Authorization': 'token %s' % os.environ['travis_key']} + resp = requests.get(request_url, headers=headers) + if resp.ok: + logger.info("Travis request response (okay): %s" % resp.json()) + details = resp.json() + if len(resp.json().get('builds', [])) == 1: + is_ready = True + return is_ready, details + + def is_travis_finished(build_id): - travis_key = os.environ.get('travis_key') - status = False + """ + Checker function used with torb waitfor lambda; output must be standarized. + Check to see if a given travis build has passed + + Args: + build_id (str): Travis build identifier + + Returns: + bool, dict: True if done, Travis response JSON + + Raises: + Exception: if the Travis build failed or Travis key not in environ + """ + if 'travis_key' not in os.environ: + raise Exception('Must have travis_key environment variable defined') + is_ready = False details = 'build not done or not found' headers = {'Content-Type': 'application/json', 'Accept': 'application/json', 'Travis-API-Version': '3', 'User-Agent': 'tibanna/0.1.0', - 'Authorization': 'token %s' % travis_key - } + 'Authorization': 'token %s' % os.environ['travis_key']} url = 'https://api.travis-ci.org/build/%s' % build_id - logger.info("url: %s" % url) + logger.info("Travis build url: %s" % url) resp = requests.get(url, headers=headers) - logger.info(resp.text) + logger.info("Travis build response: %s" % resp.text) state = resp.json()['state'] if resp.ok and state == 'failed': raise Exception('Build Failed') elif resp.ok and state == 'passed': - status = True + is_ready = True details = resp.json() - - return status, details - - -def snapshot_db(db_identifier, snapshot_name): - client = boto3.client('rds', region_name=REGION) - try: - response = client.create_db_snapshot( - DBSnapshotIdentifier=snapshot_name, - DBInstanceIdentifier=db_identifier) - except ClientError: - # probably the guy already exists - client.delete_db_snapshot(DBSnapshotIdentifier=snapshot_name) - response = client.create_db_snapshot( - DBSnapshotIdentifier=snapshot_name, - DBInstanceIdentifier=db_identifier) - print("Response from create db snapshot", response) - print("waiting for snapshot to create") - waiter = client.get_waiter('db_snapshot_completed') - waiter.wait(DBSnapshotIdentifier=snapshot_name) - print("done waiting, let's create a new database") - try: - response = client.restore_db_instance_from_db_snapshot( - DBInstanceIdentifier=snapshot_name, - DBSnapshotIdentifier=snapshot_name, - DBInstanceClass='db.t2.medium') - except ClientError: - # drop target database - delete_db(snapshot_name) - - waiter = client.get_waiter('db_instance_available') - print("waiting for db to be restore... this might take some time") - # waiter.wait(DBInstanceIdentifier=snapshot_name) - # This doesn't mean the database is done creating, but - # we now have enough information to continue to the next step - endpoint = '' - while not endpoint: - resp = client.describe_db_instances(DBInstanceIdentifier=snapshot_name) - endpoint = resp['DBInstances'][0].get('Endpoint') - if endpoint and endpoint.get('Address'): - print("we got an endpoint:", endpoint['Address']) - return endpoint['Address'] - print(".") - time.sleep(10) + return is_ready, details def make_envvar_option(name, value): @@ -395,37 +521,17 @@ def make_envvar_option(name, value): } -def set_bs_env(envname, var, template=None): - client = boto3.client('elasticbeanstalk', region_name=REGION) - options = [] - - try: - # add default environment from existing env - # allowing them to be overwritten by var - env_vars = get_bs_env(envname) - for evar in env_vars: - k, v = evar.split('=') - if var.get(k, None) is None: - var[k] = v - except: # noqa: E722 - pass - - for key, val in var.iteritems(): - options.append(make_envvar_option(key, val)) - - logging.info("About to update beanstalk with options as %s" % str(options)) - - if template: - return client.update_environment(EnvironmentName=envname, - OptionSettings=options, - TemplateName=template - ) - else: - return client.update_environment(EnvironmentName=envname, - OptionSettings=options) +def get_bs_env(envname): + """ + Given an ElasticBeanstalk environment name, get the env variables from that + environment and return them. Returned variables are in form: = + Args: + envname (str): name of ElasticBeanstalk environment -def get_bs_env(envname): + Returns: + list: of environment variables in = form + """ client = boto3.client('elasticbeanstalk', region_name=REGION) data = client.describe_configuration_settings(EnvironmentName=envname, @@ -436,84 +542,181 @@ def get_bs_env(envname): return env_vars.split(',') -def update_bs_config(envname, template, keep_env_vars=False): +def update_bs_config(envname, template=None, keep_env_vars=False, + env_override=None): + """ + Update the configuration for an existing ElasticBeanstalk environment. + Requires the environment name. Can optionally specify a configuration + template, as well as keep all environment variables from the existing + environment with optional variable overrides. + + Args: + envname (str): name of the EB environment + template (str): configuration template to use. Default None + keep_env_vars (bool): if True, keep existing env vars. Default False + env_override (dict): if provided, overwrite existing env vars using the + given key/values. Must use keep_env_vars to work. Default None + + Returns: + dict: update_environment response + """ + if template is None and not keep_env_vars: + # nothing to update + logger.info("update_bs_config: nothing to update for env %s!" % envname) + return None client = boto3.client('elasticbeanstalk', region_name=REGION) - - # get important env variables + options = [] # used to hold env vars if keep_env_vars: - options = [] env_vars = get_bs_env(envname) for var in env_vars: key, value = var.split('=') - options.append(make_envvar_option(key, value)) + if env_override and env_override.get(key): + options.append(make_envvar_option(key, env_override[key])) + else: + options.append(make_envvar_option(key, value)) + # update template and/or env var options + if options and template: return client.update_environment(EnvironmentName=envname, TemplateName=template, OptionSettings=options) - - return client.update_environment(EnvironmentName=envname, - TemplateName=template) + elif template: + return client.update_environment(EnvironmentName=envname, + TemplateName=template) + else: + return client.update_environment(EnvironmentName=envname, + OptionSettings=options) def create_bs(envname, load_prod, db_endpoint, es_url, for_indexing=False): + """ + Create a beanstalk environment given an envname. Use customized options, + configuration template, and environment variables. If adding new env vars, + make sure to overwrite them here. + If the environment already exists, will update it instead + + Args: + envname (str): ElasticBeanstalk (EB) enviroment name + load_prod (bool): sets the LOAD_FUNCTION EB env var + db_endpoint (str): sets the RDS_HOSTNAME EB env var + es_url (str): sets the ES_URL EB env var + for_indexing (bool): If True, use 'fourfront-indexing' config template + + Returns: + dict: boto3 res from create_environment/update_environment + """ client = boto3.client('elasticbeanstalk', region_name=REGION) + # deterimine the configuration template for Elasticbeanstal template = 'fourfront-base' if for_indexing: template = 'fourfront-indexing' load_value = 'load_test_data' if load_prod: load_value = 'load_prod_data' - options = [make_envvar_option('RDS_HOSTNAME', db_endpoint), - make_envvar_option('ENV_NAME', envname), - make_envvar_option('ES_URL', es_url), - make_envvar_option('LOAD_FUNCTION', load_value) - ] + + options = [ + make_envvar_option('RDS_HOSTNAME', db_endpoint), + make_envvar_option('ENV_NAME', envname), + make_envvar_option('ES_URL', es_url), + make_envvar_option('LOAD_FUNCTION', load_value) + ] + + # logic for mirrorEsEnv, which is used to coordinate elasticsearch + # changes between fourfront data and staging + if 'fourfront-webprod' in envname: + other_env = 'fourfront-webprod2' if envname == 'fourfront-webprod' else 'fourfront-webprod' + mirror_es = get_es_build_status(other_env, max_tries=3) + if mirror_es: + options.append(make_envvar_option('mirrorEnvEs', mirror_es)) + try: - res = client.create_environment(ApplicationName='4dn-web', - EnvironmentName=envname, - TemplateName=template, - OptionSettings=options, - ) + res = client.create_environment( + ApplicationName='4dn-web', + EnvironmentName=envname, + TemplateName=template, + OptionSettings=options, + ) except ClientError: - # already exists update it - res = client.update_environment(EnvironmentName=envname, - TemplateName=template, - OptionSettings=options) + # environment already exists update it + res = client.update_environment( + EnvironmentName=envname, + TemplateName=template, + OptionSettings=options + ) return res -def clone_bs_env(old, new, load_prod, db_endpoint, es_url): - env = 'RDS_HOSTNAME=%s,ENV_NAME=%s,ES_URL=%s' % (db_endpoint, new, es_url) - if load_prod is True: - env += ",LOAD_FUNCTION=load_prod_data" - subprocess.check_call(['./eb', 'clone', old, '-n', new, - '--envvars', env, - '--exact', '--nohang']) - - -def log_to_foursight(event, lambda_name, status='WARN', full_output=None): +def log_to_foursight(event, lambda_name='', overrides=None): + """ + Use Foursight as a logging tool within in a lambda function by doing a PUT + to /api/checks. Requires that the event has "_foursight" key, which is a + subobject with the following: + fields: + "check": required, in form "/" + "log_desc": will set "summary" and "description" if those are missing + "full_output": optional. If not provided, use to provide info on lambda + "brief_output": optional + "summary": optional. If not provided, use "log_desc" value + "description": optional. If not provided, use "log_desc" value + "status": optional. If not provided, use "WARN" + Can also optionally provide an dictionary to overrides param, which will + update the event["_foursight"] + + Args: + event (dict): Event input, most likely from a lambda with a workflow + lambda_name (str): Name of the lambda that is calling this + overrides (dict): Optionally override event['_foursight'] with this + + Returns: + Response object from foursight + + Raises: + Exception: if cannot get body from Foursight response + """ fs = event.get('_foursight') - if not full_output: - full_output = '%s started to run' % lambda_name - if fs: - data = {'status': status, - 'description': fs.get('log_desc'), - 'full_output': full_output - } - ff_auth = os.environ.get('FS_AUTH') - headers = {'content-type': "application/json", - 'Authorization': ff_auth} - url = FOURSIGHT_URL + 'checks/' + fs.get('check') + if fs and fs.get('check'): + if overrides is not None and isinstance(overrides, dict): + fs.update(overrides) + # handles these fields. set full_output as a special case + full_output = fs.get('full_output', '%s started to run' % lambda_name) + brief_output = fs.get('brief_output') + summary = fs.get('summary', fs.get('log_desc')) + description = fs.get('description', fs.get('log_desc')) + status = fs.get('status', 'WARN') + + data = {'status': status, 'summary': summary, 'description': description, + 'full_output': full_output, 'brief_output': brief_output} + fs_auth = os.environ.get('FS_AUTH') + headers = {'content-type': "application/json", 'Authorization': fs_auth} + # fs['check'] should be in form: "/" + url = FOURSIGHT_URL + 'checks/' + fs['check'] res = requests.put(url, data=json.dumps(data), headers=headers) - print(res.text) - return res + print('Foursight response from %s: %s' % (url, res.text)) + try: + return res.json() + except Exception as exc: + raise Exception('Error putting FS check to %s with body %s. ' + 'Exception: %s. Response text: %s' + % (url, data, exc, res.text)) def create_foursight_auto(dest_env): + """ + Call `create_foursight` to create a Foursight environment based off a + just a ElasticBeanstalk environment name. Determines a number of fields + needed for the environment creation automatically. Also causes initial + checks to be run on the new FS environment. + + Args: + dest_env (str): ElasticBeanstalk environment name + + Returns: + dict: response from Foursight PUT /api/environments + """ fs = {'dest_env': dest_env} - # whats our url + # automatically determine info for FS environ creation fs['bs_url'] = get_beanstalk_real_url(dest_env) fs['fs_url'] = get_foursight_env(dest_env, fs['bs_url']) fs['es_url'] = get_es_from_bs_config(dest_env) @@ -526,24 +729,48 @@ def create_foursight_auto(dest_env): def get_foursight_env(dest_env, bs_url=None): + """ + Get a Foursight environment name corresponding the given ElasticBeanstalk + environment name, with optionally providing the EB url for must robustness + + Args: + dest_env (str): ElasticBeanstalk environment name + bs_url (str): optional url for the ElasticBeanstalk instance + Returns: + str: Foursight environment name + """ if not bs_url: bs_url = get_beanstalk_real_url(dest_env) - env = dest_env if 'data.4dnucleome.org' in bs_url: env = 'data' elif 'staging.4dnucleome.org' in bs_url: env = 'staging' - return env def create_foursight(dest_env, bs_url, es_url, fs_url=None): - ''' - creates a new environment on foursight to be used for monitoring - ''' + """ + Creates a new Foursight environment based off of dest_env. Since Foursight + environments don't include "fourfront-" by convention, remove this if part + of the env. Take some other options for settings on the env + + Note: this will cause all checks in all schedules to be run, to initialize + environment. + + Args: + dest_env (str): ElasticBeanstalk environment name + bs_url (str): url of the ElasticBeanstalk for FS env + es_url (str): url of the ElasticSearch for FS env + fs_url (str): If provided, use to override dest-env based FS url + + Returns: + dict: response from Foursight PUT to /api/environments + Raises: + Exception: if cannot get body from Foursight response + """ # we want some url like thing if not bs_url.startswith('http'): bs_url = 'http://' + bs_url @@ -559,30 +786,213 @@ def create_foursight(dest_env, bs_url, es_url, fs_url=None): # environments on foursight don't include fourfront if not fs_url: fs_url = dest_env - if "-" in fs_url: - fs_url = fs_url.split("-")[1] + if fs_url.startswith('fourfront-'): + fs_url = fs_url[len('fourfront-'):] foursight_url = FOURSIGHT_URL + 'environments/' + fs_url - payload = {"fourfront": bs_url, - "es": es_url, - "ff_env": dest_env, - } - logger.info("Hitting up Foursight url %s with payload %s" % - (foursight_url, json.dumps(payload))) + payload = {"fourfront": bs_url, "es": es_url, "ff_env": dest_env} ff_auth = os.environ.get('FS_AUTH') - headers = {'content-type': "application/json", - 'Authorization': ff_auth} - res = requests.put(foursight_url, - data=json.dumps(payload), - headers=headers) + headers = {'content-type': "application/json", 'Authorization': ff_auth} + res = requests.put(foursight_url, data=json.dumps(payload), headers=headers) try: return res.json() + except Exception as exc: + raise Exception('Error creating FS environ to %s with body %s. ' + 'Exception: %s. Response text: %s' + % (foursight_url, payload, exc, res.text)) + + +def create_new_es(new): + """ + Create a new Elasticsearch domain with given name. See the + args below for the settings used. + + TODO: do we want to add cognito and access policy setup here? + I think not, since that's a lot of info to put out there... + + Args: + new (str): Elasticsearch instance name + + Returns: + dict: response from boto3 client + """ + es = boto3.client('es', region_name=REGION) + resp = es.create_elasticsearch_domain( + DomainName=new, + ElasticsearchVersion='5.3', + ElasticsearchClusterConfig={ + 'InstanceType': 'm4.xlarge.elasticsearch', + 'InstanceCount': 1, + 'DedicatedMasterEnabled': True, + 'DedicatedMasterType': 't2.small.elasticsearch', + 'DedicatedMasterCount': 3 + }, + EBSOptions={ + "EBSEnabled": True, + "VolumeType": "standard", + "VolumeSize": 100 + } + ) + print('=== CREATED NEW ES INSTANCE %s ===' % new) + print('MAKE SURE TO UPDATE COGNITO AND ACCESS POLICY USING THE GUI!') + print(resp) + + return resp + + +def get_es_build_status(new, max_tries=None): + """ + Check the build status of an Elasticsearch instance with given name. + If max_tries is provided, only allow that many iterative checks to ES. + Returns the ES endpoint plus port (80) + + Args: + new (str): ES instance name + max_tries (int): max number of times to check. Default None (no limit) + + Returns: + str: ES endpoint plus port, or None if not found in max_tries + """ + es = boto3.client('es', region_name=REGION) + endpoint = None + tries = 0 + while endpoint is None: + describe_resp = es.describe_elasticsearch_domain(DomainName=new) + endpoint = describe_resp['DomainStatus'].get('Endpoint') + if max_tries is not None and tries >= max_tries: + break + if endpoint is None: + print(".") + tries += 1 + time.sleep(10) + + # aws uses port 80 for es connection, lets be specific + if endpoint: + endpoint += ":80" + print('Found ES endpoint for %s: %s' % (new, endpoint)) + return endpoint + + +######################################################################### +# Functions meant to be used locally to clone or remove a beanstalk ENV # +# A lot of these functions use command line tools. # +# Sort of janky, but could end up being helpful someday ... # +######################################################################### + + +def add_es(new, force_new=False, kill_indices=False): + """ + Either gets information on an existing Elasticsearch instance + or, if force_new is True, will create a new instance. + + If not force_new, attempt to delete all indices if kill_indices=True + + Args: + new (str): Fourfront EB environment name used for ES instance + force_new (bool): if True, make a new ES. Default False + kill_indices(bool): if True, delete all indices. Default False + + Returns: + str: AWS ARN of the ES instance + """ + es = boto3.client('es', region_name=REGION) + if force_new: + # fallback is a new ES env to use if cannot create one with + # `new` environment name + fallback = new + if new.endswith("-a"): + fallback = fallback.replace("-a", "-b") + elif new.endswith("-b"): + fallback = fallback.replace("-b", "-a") + else: + fallback += "-a" + try: + resp = create_new_es(new) + except: # noqa: E722 + resp = create_new_es(fallback) + else: + try: + resp = es.describe_elasticsearch_domain(DomainName=new) + except ClientError: # its not there + resp = create_new_es(new) + else: + # now kill all the indexes + if kill_indices: + base = 'https://' + resp['DomainStatus']['Endpoint'] + url = base + '/_all' + requests.delete(url) + + return resp['DomainStatus']['ARN'] + + +def delete_es_domain(env_name): + """ + Given an Elasticsearch domain name, delete the domain + + Args: + env_name (str): Fourfront EB environment name used for ES instance + + Returns: + None + """ + # get the status of this bad boy + es = boto3.client('es') + try: + res = es.delete_elasticsearch_domain(DomainName=env_name) + print(res) except: # noqa: E722 - raise Exception(res.text) + print("es domain %s not found, skipping" % env_name) + + +def clone_bs_env_cli(old, new, load_prod, db_endpoint, es_url): + """ + Use the eb command line client to clone an ElasticBeanstalk environment + with some extra options. + + Args: + old (str): existing EB environment name + new (str): new EB environment name + load_prod (bool): determins LOAD_FUNCTION EB env var + db_endpoint (str): determines RDS_HOSTNAME EB env var + es_url (str): determines ES_URL EB env var + + Returns: + None + """ + env = 'RDS_HOSTNAME=%s,ENV_NAME=%s,ES_URL=%s' % (db_endpoint, new, es_url) + if load_prod is True: + env += ",LOAD_FUNCTION=load_prod_data" + subprocess.check_call(['eb', 'clone', old, '-n', new, + '--envvars', env, + '--exact', '--nohang']) + + +def delete_bs_env_cli(env_name): + """ + Use the eb command line client to remove an ElasticBeanstalk environment + with some extra options. + + Args: + env_name (str): EB environment name + + Returns: + None + """ + subprocess.check_call(['eb', 'terminate', env_name, '-nh', '--force']) def create_s3_buckets(new): + """ + Given an ElasticBeanstalk env name, create the following s3 buckets that + are standard for any of our EB environments. + + Args: + new (str): EB environment name + + Returns: + None + """ new_buckets = [ 'elasticbeanstalk-%s-blobs' % new, 'elasticbeanstalk-%s-files' % new, @@ -594,44 +1004,88 @@ def create_s3_buckets(new): s3.create_bucket(Bucket=bucket) -def copy_s3_buckets(new, old): - # each env needs the following buckets - new_buckets = [ - 'elasticbeanstalk-%s-blobs' % new, - 'elasticbeanstalk-%s-files' % new, - 'elasticbeanstalk-%s-wfoutput' % new, - 'elasticbeanstalk-%s-system' % new, - ] - old_buckets = [ - 'elasticbeanstalk-%s-blobs' % old, - 'elasticbeanstalk-%s-files' % old, - 'elasticbeanstalk-%s-wfoutput' % old, +def delete_s3_buckets(env_name): + """ + Given an ElasticBeanstalk env name, remove the following s3 buckets that + are standard for any of our EB environments. + + Args: + env_name (str): EB environment name + + Returns: + None + """ + buckets = [ + 'elasticbeanstalk-%s-blobs' % env_name, + 'elasticbeanstalk-%s-files' % env_name, + 'elasticbeanstalk-%s-wfoutput' % env_name, + 'elasticbeanstalk-%s-system' % env_name, ] - s3 = boto3.client('s3', region_name=REGION) - for bucket in new_buckets: + + s3 = boto3.resource('s3') + for bucket in buckets: + print("deleting content for " + bucket) try: - s3.create_bucket(Bucket=bucket) + s3.Bucket(bucket).objects.delete() + s3.Bucket(bucket).delete() except: # noqa: E722 + print(bucket + " not found skipping...") - print("bucket already created....") - # now copy them - # aws s3 sync s3://mybucket s3://backup-mybucket - # get rid of system bucket - new_buckets.pop() - for old, new in zip(old_buckets, new_buckets): - oldb = "s3://%s" % old - newb = "s3://%s" % new - print("copying data from old %s to new %s" % (oldb, newb)) - subprocess.call(['aws', 's3', 'sync', oldb, newb]) +def snapshot_and_clone_db(db_identifier, snapshot_name): + """ + Given a RDS instance identifier and snapshot name, will create a snapshot + with that name and then spin up a new RDS instance named after the snapshot + + Args: + db_identifier (str): original RDS identifier of DB to snapshot + snapshot_name (str): identifier of snapshot AND new instance + + Returns: + str: address of the new instance + """ + client = boto3.client('rds', region_name=REGION) + snap_res = create_db_snapshot(db_identifier, snapshot_name) + if snap_res == 'Deleting': + snap_res = client.create_db_snapshot( + DBSnapshotIdentifier=snapshot_name, + DBInstanceIdentifier=db_identifier + ) + print("Response from create db snapshot: %s" % snap_res) + print("Waiting for snapshot to create...") + waiter = client.get_waiter('db_snapshot_completed') + waiter.wait(DBSnapshotIdentifier=snapshot_name) + print("Done waiting, creating a new database with name %s" % snapshot_name) + db_res = create_db_from_snapshot(snapshot_name, snapshot_name, False) + if db_res == 'Error': + raise Exception('Could not create DB %s; already exists' % snapshot_name) + print("Waiting for DB to be created from snapshot...") + endpoint = '' + while not endpoint: + resp = client.describe_db_instances(DBInstanceIdentifier=snapshot_name) + endpoint = resp['DBInstances'][0].get('Endpoint') + if endpoint and endpoint.get('Address'): + print("We got an endpoint:", endpoint['Address']) + return endpoint['Address'] + print(".") + time.sleep(10) def add_to_auth0_client(new): - # first get the url of the newly created beanstalk environment + """ + Given an ElasticBeanstalk env name, find the url and use it to update the + callback URLs for Auth0 + + Args: + new (str): EB environment name + + Returns: + None + """ client = boto3.client('elasticbeanstalk', region_name=REGION) env = describe_beanstalk_environments(client, EnvironmentNames=[new]) url = None - print("waiting for beanstalk to be up, this make take some time...") + print("Getting beanstalk URL for env %s..." % new) while url is None: url = env['Environments'][0].get('CNAME') if url is None: @@ -639,16 +1093,44 @@ def add_to_auth0_client(new): time.sleep(10) auth0_client_update(url) - # TODO: need to also update ES permissions policy with ip addresses of elasticbeanstalk - # or configure application to use AWS IAM stuff + +def remove_from_auth0_client(env_name): + """ + Given an ElasticBeanstalk env name, find the url and remove it from the + callback urls for Auth0 + + Args: + env_name (str): EB environment name + + Returns: + None + """ + eb = boto3.client('elasticbeanstalk') + env = eb.describe_environments(EnvironmentNames=[env_name]) + url = None + print("Getting beanstalk URL for env %s..." % env_name) + while url is None: + url = env['Environments'][0].get('CNAME') + if url is None: + print(".") + time.sleep(10) + auth0_client_remove(url) def auth0_client_update(url): - # Auth0 stuff - # generate a jwt to validate future requests - client = os.environ.get("Auth0Client") - secret = os.environ.get("Auth0Secret") + """ + Get a JWT for programmatic access to Auth0 using Client/Secret env vars. + Then add the given `url` to the Auth0 callbacks list. + Args: + url (str): url to add to callbacks + + Returns: + None + """ + # generate a jwt to validate future requests + client = os.environ["Auth0Client"] + secret = os.environ["Auth0Secret"] payload = {"grant_type": "client_credentials", "client_id": client, "client_secret": secret, @@ -657,170 +1139,205 @@ def auth0_client_update(url): res = requests.post("https://hms-dbmi.auth0.com/oauth/token", data=json.dumps(payload), headers=headers) - - print(res.json()) jwt = res.json()['access_token'] + client_url = "https://hms-dbmi.auth0.com/api/v2/clients/%s" % client headers['authorization'] = 'Bearer %s' % jwt get_res = requests.get(client_url + '?fields=callbacks', headers=headers) - callbacks = get_res.json()['callbacks'] callbacks.append("http://" + url) client_data = {'callbacks': callbacks} update_res = requests.patch(client_url, data=json.dumps(client_data), headers=headers) - print(update_res.json().get('callbacks')) + print('auth0 callback urls are: %s' % update_res.json().get('callbacks')) -def sizeup_es(new): - es = boto3.client('es', region_name=REGION) - resp = es.update_elasticsearch_domain_config( - DomainName=new, - ElasticsearchClusterConfig={ - 'InstanceType': 'm3.xlarge.elasticsearch', - 'InstanceCount': 4, - 'DedicatedMasterEnabled': True, - } - ) - - print(resp) - - -def add_es(new, force_new=False, kill_indices=False): - es = boto3.client('es', region_name=REGION) - if force_new: - fallback = new - if new.endswith("-a"): - fallback = fallback.replace("-a", "-b") - elif new.endswith("-b"): - fallback = fallback.replace("-b", "-a") - else: - fallback += "-a" - try: - resp = create_new_es(new) - except: # noqa: E722 - resp = create_new_es(fallback) - else: - try: - resp = es.describe_elasticsearch_domain(DomainName=new) - except ClientError: # its not there - resp = create_new_es(new) - else: - # now kill all the indexes - if kill_indices: - base = 'https://' + resp['DomainStatus']['Endpoint'] - url = base + '/_all' - requests.delete(url) +def auth0_client_remove(url): + """ + Get a JWT for programmatic access to Auth0 using Client/Secret env vars. + Then use that to remove the given `url` from the Auth0 callbacks list. - return resp['DomainStatus']['ARN'] + Args: + url (str): url to remove from callbacks + Returns: + None + """ + # generate a jwt to validate future requests + client = os.environ["Auth0Client"] + secret = os.environ["Auth0Secret"] + payload = {"grant_type": "client_credentials", + "client_id": client, + "client_secret": secret, + "audience": "https://hms-dbmi.auth0.com/api/v2/"} + headers = {'content-type': "application/json"} + res = requests.post("https://hms-dbmi.auth0.com/oauth/token", + data=json.dumps(payload), + headers=headers) + jwt = res.json()['access_token'] + client_url = "https://hms-dbmi.auth0.com/api/v2/clients/%s" % client + headers['authorization'] = 'Bearer %s' % jwt -def create_new_es(new): - es = boto3.client('es', region_name=REGION) - resp = es.create_elasticsearch_domain( - DomainName=new, - ElasticsearchVersion='5.3', - ElasticsearchClusterConfig={ - 'InstanceType': 'm4.large.elasticsearch', - 'InstanceCount': 3, - 'DedicatedMasterEnabled': False, - }, - EBSOptions={"EBSEnabled": True, "VolumeType": "standard", "VolumeSize": 10}, - AccessPolicies=json.dumps({ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::643366669028:role/Developer" - }, - "Action": [ - "es:*" - ], - "Condition": { - "IpAddress": { - "aws:SourceIp": [ - "0.0.0.0/0", - "134.174.140.197/32", - "134.174.140.208/32", - "172.31.16.84/32", - "172.31.73.1/24", - "172.31.77.1/24" - ] - } - }, - } - ] - }) - ) - print(resp) - return resp + get_res = requests.get(client_url + '?fields=callbacks', headers=headers) + callbacks = get_res.json()['callbacks'] + full_url = 'http://' + url + try: + idx = callbacks.index(full_url) + except ValueError: + print(full_url + " Not in auth0 auth, doesn't need to be removed") + return + if idx: + callbacks.pop(idx) + client_data = {'callbacks': callbacks} + update_res = requests.patch(client_url, data=json.dumps(client_data), headers=headers) + print('auth0 callback urls are: %s' % update_res.json().get('callbacks')) -def get_es_build_status(new): - # get the status of this bad boy - es = boto3.client('es', region_name=REGION) - endpoint = None - while endpoint is None: - describe_resp = es.describe_elasticsearch_domain(DomainName=new) - endpoint = describe_resp['DomainStatus'].get('Endpoint') - if endpoint is None: - print(".") - time.sleep(10) - print(endpoint) +def copy_s3_buckets(new, old): + """ + Given a new ElasticBeanstalk environment name and existing "old" one, + create the given buckets and copy contents from the corresponding + existing ones - # aws uses port 80 for es connection, lets be specific - return endpoint + ":80" + Args: + new (str): new EB environment name + old (str): existing EB environment name + Returns: + None + """ + # each env needs the following buckets + new_buckets = [ + 'elasticbeanstalk-%s-blobs' % new, + 'elasticbeanstalk-%s-files' % new, + 'elasticbeanstalk-%s-wfoutput' % new, + 'elasticbeanstalk-%s-system' % new, + ] + old_buckets = [ + 'elasticbeanstalk-%s-blobs' % old, + 'elasticbeanstalk-%s-files' % old, + 'elasticbeanstalk-%s-wfoutput' % old, + ] + s3 = boto3.client('s3', region_name=REGION) + for bucket in new_buckets: + try: + s3.create_bucket(Bucket=bucket) + except: # noqa: E722 + print("bucket %s already created..." % bucket) -def eb_deploy(new): - subprocess.check_call(['eb', 'deploy', new]) + # now copy them + # aws s3 sync s3://mybucket s3://backup-mybucket + # get rid of system bucket + new_buckets.pop() + for old, new in zip(old_buckets, new_buckets): + oldb = "s3://%s" % old + newb = "s3://%s" % new + print("copying data from old %s to new %s" % (oldb, newb)) + subprocess.call(['aws', 's3', 'sync', oldb, newb]) -def main(): - parser = argparse.ArgumentParser( - description="Clone a beanstalk env into a new one", - ) - parser.add_argument('--old') - parser.add_argument('--new') - parser.add_argument('--prod', action='store_true', default=False, help='load prod data on new env?') - parser.add_argument('--deploy_current', action='store_true', help='deploy current branch') - parser.add_argument('--skips3', action='store_true', default=False, - help='skip copying files from s3') - - parser.add_argument('--onlys3', action='store_true', default=False, - help='skip copying files from s3') - - args = parser.parse_args() - if args.onlys3: - print("### only copy contents of s3") - copy_s3_buckets(args.new, args.old) +def clone_beanstalk_command_line(old, new, prod=False, copy_s3=False): + """ + Maybe useful command to clone an existing ElasticBeanstalk environment to + a new one. Will create an Elasticsearch instance, s3 buckets, clone the + existing RDS of the environment, and optionally copy s3 contents. + Also adds the new EB url to Auth0 callback urls. + Should be run exclusively via command line, as it requires manual input + and subprocess calls of AWS command line tools. + + Note: + The eb cli tool sets up a configuration file in the directory of the + project respository. As such, this command MUST be called from that + directory. Will exit if not called from an eb initialized directory. + + Args: + old (str): environment name of existing ElasticBeanstalk + new (str): new ElasticBeanstalk environment name + prod (bool): set to True if this is a prod environment. Default False + copy_s3 (bool): set to True to copy s3 contents. Default False + + Returns: + None + """ + if 'Auth0Client' not in os.environ or "Auth0Secret" not in os.environ: + print('Must set Auth0Client and Auth0Secret env variables! Exiting...') + return + print('### eb status (START)') + eb_ret = subprocess.call(['eb', 'status']) + if eb_ret != 0: + print('This command must be called from an eb initialized repo! Exiting...') + return + print('### eb status (END)') + name = use_input("This will create an environment named %s, cloned from %s." + " This includes s3, ES, RDS, and Auth0 callbacks. If you " + "are sure, type the new env name to confirm: " % (new, old)) + if str(name) != new: + print("Could not confirm env. Exiting...") return - print("### start build ES service") - add_es(args.new) + add_es(new) print("### create the s3 buckets") - create_s3_buckets(args.new) - print("### copy database") - db_endpoint = snapshot_db(args.old, args.new) + create_s3_buckets(new) + print("### create snapshot and copy database") + db_endpoint = snapshot_and_clone_db(old, new) print("### waiting for ES service") - es_endpoint = get_es_build_status(args.new) + es_endpoint = get_es_build_status(new) print("### clone elasticbeanstalk envrionment") # TODO, can we pass in github commit id here? - clone_bs_env(args.old, args.new, args.prod, db_endpoint, es_endpoint) + clone_bs_env_cli(old, new, prod, db_endpoint, es_endpoint) print("### allow auth-0 requests") - add_to_auth0_client(args.new) - if not args.skips3: + add_to_auth0_client(new) + if copy_s3 is True: print("### copy contents of s3") - copy_s3_buckets(args.new, args.old) - if args.deploy_current: - print("### deploying local code to new eb environment") - eb_deploy(args.new) - - print("all set, it may take some time for the beanstalk env to finish starting up") + copy_s3_buckets(new, old) + print("### All done! It may take some time for the beanstalk env to finish" + " initialization. You may want to deploy the most current FF branch.") -if __name__ == "__main__": - main() +def delete_beanstalk_command_line(env): + """ + Maybe useful command to delete an existing ElasticBeanstalk environment, + including associated ES, s3, and RDS resources. Will also remove the + associated callback url from Auth0. + Should be run exclusively via command line, as it requires manual input + and subprocess calls of AWS command line tools. + + Note: + The eb cli tool sets up a configuration file in the directory of the + project respository. As such, this command MUST be called from that + directory. Will exit if not called from an eb initialized directory. + + Args: + env (str): EB environment name to delete + + Returns: + None + """ + if 'Auth0Client' not in os.environ or "Auth0Secret" not in os.environ: + print('Must set Auth0Client and Auth0Secret env variables! Exiting...') + return + print('### eb status (START)') + eb_ret = subprocess.call(['eb', 'status']) + if eb_ret != 0: + print('This command must be called from an eb initialized repo! Exiting...') + return + print('### eb status (END)') + name = use_input("This will totally blow away the environment, including s3," + "ES, RDS, and Auth0 callbacks. If you are sure, type the " + "env name to confirm: ") + if str(name) != env: + print("Could not confirm env. Exiting...") + return + print("### Removing access to auth0") + remove_from_auth0_client(env) + print("### Deleting beanstalk enviornment") + delete_bs_env_cli(env) + print("### Delete contents of s3") + delete_s3_buckets(env) + print("### Delete es domain") + delete_es_domain(env) + print("### Delete database") + delete_db(env) + print('### All done!') diff --git a/dcicutils/s3_utils.py b/dcicutils/s3_utils.py index 673a4ade3..4a1f19be4 100644 --- a/dcicutils/s3_utils.py +++ b/dcicutils/s3_utils.py @@ -11,7 +11,8 @@ ########################### # Config ########################### -LOG = logging.getLogger(__name__) +logging.basicConfig() +logger = logging.getLogger(__name__) class s3Utils(object): @@ -83,7 +84,7 @@ def get_key(self, keyfile_name='illnevertell'): def read_s3(self, filename): response = self.s3.get_object(Bucket=self.outfile_bucket, Key=filename) - LOG.info(str(response)) + logger.info(str(response)) return response['Body'].read() def does_key_exist(self, key, bucket=None, print_error=True):