diff --git a/dcicutils/beanstalk_utils.py b/dcicutils/beanstalk_utils.py index cbf860132..86a105aaf 100755 --- a/dcicutils/beanstalk_utils.py +++ b/dcicutils/beanstalk_utils.py @@ -14,6 +14,7 @@ from . import ff_utils from botocore.exceptions import ClientError from .misc_utils import PRINT +from .env_utils import is_cgap_env, is_stg_or_prd_env, public_url_mappings, blue_green_mirror_env logging.basicConfig() logger = logging.getLogger('logger') @@ -35,9 +36,22 @@ FOURSIGHT_URL = 'https://foursight.4dnucleome.org/' -# magic CNAME corresponds to data.4dnucleome -MAGIC_CNAME = 'fourfront-green.us-east-1.elasticbeanstalk.com' -GOLDEN_DB = 'fourfront-production.co3gwj7b7tpq.us-east-1.rds.amazonaws.com' + +# FF_MAGIC_CNAME corresponds to data.4dnucleome.org +FF_MAGIC_CNAME = 'fourfront-green.us-east-1.elasticbeanstalk.com' +# CGAP_MAGIC_CNAME corresponds to cgap.hms.harvard.edu +CGAP_MAGIC_CNAME = 'fourfront-cgap.9wzadzju3p.us-east-1.elasticbeanstalk.com' +# The legacy name MAGIC_CNAME is deprecated (retained for backward compatibility until a major release boundary). +MAGIC_CNAME = FF_MAGIC_CNAME + +# FF_GOLDEN_DB is the database behind data.4dnucleome.org (and shared by staging.4dnucleome.org) +FF_GOLDEN_DB = 'fourfront-production.co3gwj7b7tpq.us-east-1.rds.amazonaws.com' +# CGAP_GOLDEN_DB is the database behind cgap.hms.harvard.edu +CGAP_GOLDEN_DB = 'fourfront-cgap.co3gwj7b7tpq.us-east-1.rds.amazonaws.com' +# The name GOLDEN_DB is deprecated (retained for backward compatibility until a major release boundary). +# Although not visibly used in this repository, this variable is imported by Torb. +GOLDEN_DB = FF_GOLDEN_DB + REGION = 'us-east-1' @@ -206,7 +220,7 @@ def swap_cname(src, dest): client.restart_app_server(EnvironmentName=dest) -def whodaman(): +def _compute_prd_env_for_project(project): ''' Determines which ElasticBeanstalk environment is currently hosting data.4dnucleome.org. Requires IAM permissions for EB! @@ -214,16 +228,28 @@ def whodaman(): Returns: str: EB environment name hosting data.4dnucleome ''' + magic_cname = CGAP_MAGIC_CNAME if project == 'cgap' else FF_MAGIC_CNAME client = boto3.client('elasticbeanstalk', region_name=REGION) res = describe_beanstalk_environments(client, ApplicationName="4dn-web") logger.info(res) for env in res['Environments']: logger.info(env) - if env.get('CNAME') == MAGIC_CNAME: + if env.get('CNAME') == magic_cname: # we found data return env.get('EnvironmentName') +def compute_ff_prd_env(): + return _compute_prd_env_for_project('ff') + + +whodaman = compute_ff_prd_env # This naming is deprecated but retained for compatibility. + + +def compute_cgap_prd_env(): + return _compute_prd_env_for_project('cgap') + + def beanstalk_info(env): """ Describe a ElasticBeanstalk environment given an environment name @@ -242,7 +268,7 @@ def beanstalk_info(env): def get_beanstalk_real_url(env): """ Return the real url for the elasticbeanstalk with given environment name. - Name can be 'data', 'staging', or an actual environment. + Name can be 'cgap', 'data', 'staging', or an actual environment. Args: env (str): ElasticBeanstalk environment name @@ -251,25 +277,24 @@ def get_beanstalk_real_url(env): str: url of the ElasticBeanstalk environment """ url = '' - urls = {'staging': 'http://staging.4dnucleome.org', - 'data': 'https://data.4dnucleome.org'} + urls = public_url_mappings(env) - if env in urls: + if env in urls: # Special case handling of 'cgap', 'data', or 'staging' as an argument. return urls[env] - # TODO (C4-91): Reconsider environment names. - # This code is too fragile. - if 'webprod' in env or 'blue' in env or 'green' in env: - data_env = whodaman() - - if data_env == env: - url = urls['data'] - else: - url = urls['staging'] - else: - bs_info = beanstalk_info(env) - url = "http://" + bs_info['CNAME'] - + if is_stg_or_prd_env(env): + # What counts as staging/prod depends on whether we're in the CGAP or Fourfront space. + data_env = compute_cgap_prd_env() if is_cgap_env(env) else compute_ff_prd_env() + # There is only one production environment. Everything else is staging, but everything + # else is not staging.4dnucleome.org. Only one is that. + if env == data_env: + return urls['data'] + elif env == blue_green_mirror_env(data_env): + # Mirror env might be None, in which case this clause will not be entered + return urls['staging'] + + bs_info = beanstalk_info(env) + url = "http://" + bs_info['CNAME'] return url diff --git a/dcicutils/env_utils.py b/dcicutils/env_utils.py index 53eb50a28..f02c3d20d 100644 --- a/dcicutils/env_utils.py +++ b/dcicutils/env_utils.py @@ -34,6 +34,10 @@ # CGAP_ENV_WEBPROD2_NEW is meaningless here. See CGAP_ENV_STAGING_NEW. CGAP_ENV_WOLF_NEW = 'cgap-wolf' # Maybe not used +# The bucket names were allocated originally and needn't change. + +FF_PROD_BUCKET_ENV = FF_ENV_WEBPROD +CGAP_PROD_BUCKET_ENV = CGAP_ENV_WEBPROD # Done this way to get maximally compatible behavior. FOURFRONT_STG_OR_PRD_TOKENS = ['webprod', 'blue', 'green'] @@ -41,10 +45,41 @@ # Done this way because it's safer going forward. CGAP_STG_OR_PRD_TOKENS = [] -CGAP_STG_OR_PRD_NAMES = [CGAP_ENV_WEBPROD, CGAP_ENV_PRODUCTION_GREEN, CGAP_ENV_PRODUCTION_BLUE] +CGAP_STG_OR_PRD_NAMES = [CGAP_ENV_WEBPROD, CGAP_ENV_PRODUCTION_GREEN, CGAP_ENV_PRODUCTION_BLUE, 'cgap'] + + +FF_PUBLIC_URL_STG = 'http://staging.4dnucleome.org' +FF_PUBLIC_URL_PRD = 'https://data.4dnucleome.org' +FF_PUBLIC_URLS = { + 'staging': FF_PUBLIC_URL_STG, + 'data': FF_PUBLIC_URL_PRD, +} + +CGAP_PUBLIC_URL_STG = 'https://staging.cgap.hms.harvard.edu' # This is a stopgap for testing and may have to change +CGAP_PUBLIC_URL_PRD = 'https://cgap.hms.harvard.edu' + +CGAP_PUBLIC_URLS = { + 'cgap': CGAP_PUBLIC_URL_PRD, + 'data': CGAP_PUBLIC_URL_PRD, + 'staging': CGAP_PUBLIC_URL_STG, +} + +BEANSTALK_PROD_BUCKET_ENVS = { + 'staging': FF_PROD_BUCKET_ENV, + 'data': FF_PROD_BUCKET_ENV, + FF_ENV_WEBPROD: FF_PROD_BUCKET_ENV, + FF_ENV_WEBPROD2: FF_PROD_BUCKET_ENV, + FF_ENV_PRODUCTION_BLUE: FF_PROD_BUCKET_ENV, + FF_ENV_PRODUCTION_GREEN: FF_PROD_BUCKET_ENV, + 'cgap': CGAP_PROD_BUCKET_ENV, + CGAP_ENV_PRODUCTION_BLUE: CGAP_PROD_BUCKET_ENV, + CGAP_ENV_PRODUCTION_GREEN: CGAP_PROD_BUCKET_ENV, + CGAP_ENV_WEBPROD: CGAP_PROD_BUCKET_ENV, + CGAP_ENV_PRODUCTION_BLUE_NEW: CGAP_PROD_BUCKET_ENV, + CGAP_ENV_PRODUCTION_GREEN_NEW: CGAP_PROD_BUCKET_ENV, +} -# These operate as pairs. Don't add extras. BEANSTALK_PROD_MIRRORS = { FF_ENV_PRODUCTION_BLUE: FF_ENV_PRODUCTION_GREEN, @@ -61,7 +96,6 @@ } - BEANSTALK_TEST_ENVS = [ FF_ENV_HOTSEAT, @@ -95,6 +129,34 @@ def blue_green_mirror_env(envname): return None +def prod_bucket_env(envname): + """ + Given a production-class envname returns the envname of the associated production bucket. + For other envnames that aren't production envs, this returns None. + + The envname is something that is either a staging or production env, in particular something + that is_stg_or_prd_env returns True for. + + This is intended for use when configuring a beanstalk. This functionality is agnostic + about whether we're asking on behalf of CGAP or Fourfront, and whether we're using an old or new + naming scheme. Just give the current envname as an argument, and it will know (by declaration, + see the BEANSTALK_PROD_ENV_BUCKET_TOKENS table) what the appropriate production bucket name token is for + that ecosystem. + """ + return BEANSTALK_PROD_BUCKET_ENVS.get(envname) + + +def public_url_mappings(envname): + """ + Returns a table of the public URLs we use for the ecosystem in which the envname resides. + For example, if envname is a CGAP URL, this returns a set table of CGAP public URLs, + and otherwise it returns a set of Fourfront URLs. + + The envname may be 'cgap', 'data', 'staging', or an environment name. + """ + return CGAP_PUBLIC_URLS if is_cgap_env(envname) else FF_PUBLIC_URLS + + def is_cgap_env(envname): """ Returns True of the given string looks like a CGAP elasticbeanstalk environment name. diff --git a/dcicutils/ff_utils.py b/dcicutils/ff_utils.py index 8aa49c236..7cceefde5 100644 --- a/dcicutils/ff_utils.py +++ b/dcicutils/ff_utils.py @@ -7,7 +7,8 @@ import boto3 from . import ( s3_utils, - es_utils + es_utils, + env_utils, ) from .misc_utils import PRINT import requests @@ -20,11 +21,13 @@ from urllib.parse import urlencode +# TODO (C4-92, C4-102): Probably to centralize this information in env_utils. Also figure out relation to CGAP. HIGLASS_BUCKETS = ['elasticbeanstalk-fourfront-webprod-wfoutput', 'elasticbeanstalk-fourfront-webdev-wfoutput'] # TODO (C4-92): Centralize this information, it is repeated in other repos +# TODO (C4-102): Does this need to include CGAP envs? As part of the same list, or as a separate list? PRODUCTION_ENVS = ['fourfront-blue', 'fourfront-green'] @@ -1067,11 +1070,8 @@ def unified_authentication(auth=None, ff_env=None): """ # first see if key should be obtained from using ff_env if not auth and ff_env: - # webprod, webprod2 and blue/green all use the fourfront-webprod bucket for keys - if 'webprod' in ff_env or ff_env in PRODUCTION_ENVS: - use_env = 'fourfront-webprod' - else: - use_env = ff_env + # TODO: The ff_env argument is mis-named, something we should fix sometime. It can be a cgap env, too. + use_env = env_utils.prod_bucket_env(ff_env) if env_utils.is_stg_or_prd_env(ff_env) else ff_env auth = s3_utils.s3Utils(env=use_env).get_access_keys() # see if auth is directly from get_access_keys() use_auth = None diff --git a/dcicutils/qa_utils.py b/dcicutils/qa_utils.py new file mode 100644 index 000000000..debf61f11 --- /dev/null +++ b/dcicutils/qa_utils.py @@ -0,0 +1,20 @@ +""" +qa_utils: Tools for use in quality assurance testing. +""" + +from .misc_utils import PRINT + + +def mock_not_called(name): + """ + This can be used in mocking to mock a function that should not be called. + Called with the name of a function, it returns a function that if called + will raise an AssertionError complaining that such a name was called. + """ + def mocked_function(*args, **kwargs): + # It's OK to print here because we're expected to be called in a testing context, and + # we're just about to fail a test. The person invoking the tests may want this data. + PRINT("args=", args) + PRINT("kwargs=", kwargs) + raise AssertionError("%s was called where not expected." % name) + return mocked_function diff --git a/dcicutils/s3_utils.py b/dcicutils/s3_utils.py index 63644fa63..adf0a03c0 100644 --- a/dcicutils/s3_utils.py +++ b/dcicutils/s3_utils.py @@ -6,7 +6,7 @@ from zipfile import ZipFile from io import BytesIO import logging -from .env_utils import is_stg_or_prd_env +from .env_utils import is_stg_or_prd_env, prod_bucket_env from .misc_utils import PRINT @@ -33,7 +33,7 @@ def __init__(self, outfile_bucket=None, sys_bucket=None, raw_file_bucket=None, if env: if is_stg_or_prd_env(env): self.url = get_beanstalk_real_url(env) - env = 'fourfront-webprod' + env = prod_bucket_env(env) # we use standardized naming schema, so s3 buckets always have same prefix sys_bucket = "elasticbeanstalk-%s-system" % env outfile_bucket = "elasticbeanstalk-%s-wfoutput" % env diff --git a/pyproject.toml b/pyproject.toml index ae69342ae..0aef11532 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "0.12.1" +version = "0.13.0" description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["William Ronchetti "] license = "MIT" diff --git a/test/test_beanstalk_utils.py b/test/test_beanstalk_utils.py index e6d6935b5..d92ff3b5d 100644 --- a/test/test_beanstalk_utils.py +++ b/test/test_beanstalk_utils.py @@ -1,34 +1,71 @@ +import boto3 import io import os -from dcicutils import beanstalk_utils as bs, source_beanstalk_env_vars +from dcicutils import beanstalk_utils as bs, env_utils, source_beanstalk_env_vars +from dcicutils.qa_utils import mock_not_called from unittest import mock -def test_get_beanstalk_prod_url(): - with mock.patch('dcicutils.beanstalk_utils.whodaman') as man_not_hot: - man_not_hot.return_value = 'webprod-1' - url = bs.get_beanstalk_real_url('webprod-1') - assert url == 'https://data.4dnucleome.org' +def _mocked_beanstalk_info(env): + return {'CNAME': 'blah-%s.blahblah.us-east-1.elasticbeanstalk.com' % env} -def test_get_beanstalk_staging_url(): - with mock.patch('dcicutils.beanstalk_utils.whodaman') as man_not_hot: - man_not_hot.return_value = 'webprod-2' - url = bs.get_beanstalk_real_url('webprod-1') - assert url == 'http://staging.4dnucleome.org' +def test_get_beanstalk_real_url_blue_data(): + with mock.patch('dcicutils.beanstalk_utils._compute_prd_env_for_project') as mock_whodaman: + with mock.patch('dcicutils.beanstalk_utils.beanstalk_info') as mock_beanstalk_info: + mock_whodaman.return_value = 'fourfront-blue' + mock_beanstalk_info.side_effect = _mocked_beanstalk_info # mock_not_called('dcicutils.beanstalk_utils.beanstalk_info') + url = bs.get_beanstalk_real_url('fourfront-blue') + assert url == 'https://data.4dnucleome.org' -def test_get_beanstalk_normal_url(): - with mock.patch('dcicutils.beanstalk_utils.beanstalk_info') as man_not_hot: - man_not_hot.return_value = {'CNAME': 'take-of-your-jacket'} - url = bs.get_beanstalk_real_url('take-of-your-jacket') - assert url == 'http://take-of-your-jacket' +def test_get_beanstalk_real_url_green_data(): + with mock.patch('dcicutils.beanstalk_utils._compute_prd_env_for_project') as mock_whodaman: + with mock.patch('dcicutils.beanstalk_utils.beanstalk_info') as mock_beanstalk_info: + mock_whodaman.return_value = 'fourfront-green' + mock_beanstalk_info.side_effect = _mocked_beanstalk_info # mock_not_called('dcicutils.beanstalk_utils.beanstalk_info') + url = bs.get_beanstalk_real_url('fourfront-green') + assert url == 'https://data.4dnucleome.org' -def _mock_not_called(name): - def mock_not_called(*args, **kwargs): - raise AssertionError("%s was called where not expected." % name) - return mock_not_called +def test_get_beanstalk_real_url_blue_staging(): + with mock.patch('dcicutils.beanstalk_utils._compute_prd_env_for_project') as mock_whodaman: + with mock.patch('dcicutils.beanstalk_utils.beanstalk_info') as mock_beanstalk_info: + mock_whodaman.return_value = 'fourfront-green' + mock_beanstalk_info.side_effect = _mocked_beanstalk_info # mock_not_called('dcicutils.beanstalk_utils.beanstalk_info') + url = bs.get_beanstalk_real_url('fourfront-blue') + assert url == 'http://staging.4dnucleome.org' + + +def test_get_beanstalk_real_url_green_staging(): + with mock.patch('dcicutils.beanstalk_utils._compute_prd_env_for_project') as mock_whodaman: + with mock.patch('dcicutils.beanstalk_utils.beanstalk_info') as mock_beanstalk_info: + mock_whodaman.return_value = 'fourfront-blue' + mock_beanstalk_info.side_effect = _mocked_beanstalk_info # mock_not_called('dcicutils.beanstalk_utils.beanstalk_info') + url = bs.get_beanstalk_real_url('fourfront-green') + assert url == 'http://staging.4dnucleome.org' + + +# Non-production environments will do this: + +def test_get_beanstalk_real_url_other(): + with mock.patch('dcicutils.beanstalk_utils._compute_prd_env_for_project') as mock_whodaman: + with mock.patch('dcicutils.beanstalk_utils.beanstalk_info') as mock_beanstalk_info: + mock_whodaman.side_effect = mock_not_called('dcicutils.beanstalk_utils._compute_prd_env_for_project') + mock_beanstalk_info.side_effect = _mocked_beanstalk_info + url = bs.get_beanstalk_real_url('beanstalk-name') + assert url == 'http://blah-beanstalk-name.blahblah.us-east-1.elasticbeanstalk.com' + + +# These will fail in CGAP, which Soo reported (C4-101): + +def test_get_beanstalk_real_url_cgap(): + with mock.patch('dcicutils.beanstalk_utils._compute_prd_env_for_project') as mock_whodaman: + with mock.patch('dcicutils.beanstalk_utils.beanstalk_info') as mock_beanstalk_info: + mock_whodaman.return_value = 'fourfront-cgap' + mock_beanstalk_info.side_effect = _mocked_beanstalk_info # mock_not_called('dcicutils.beanstalk_utils.beanstalk_info') + url = bs.get_beanstalk_real_url('fourfront-cgap') + assert url == 'https://cgap.hms.harvard.edu' def test_source_beanstalk_env_vars_no_config_file(): @@ -38,7 +75,7 @@ def test_source_beanstalk_env_vars_no_config_file(): with mock.patch.object(os, "environ", {}): with mock.patch("subprocess.Popen") as mock_popen: mock_exists.return_value = False - mock_popen = _mock_not_called("subprocess.Popen") + mock_popen.side_effect = mock_not_called("subprocess.Popen") source_beanstalk_env_vars() @@ -49,7 +86,7 @@ def test_source_beanstalk_env_vars_aws_access_key_id(): with mock.patch.object(os, "environ", {"AWS_ACCESS_KEY_ID": "something"}): with mock.patch("subprocess.Popen") as mock_popen: mock_exists.return_value = True - mock_popen.side_effect = _mock_not_called("subprocess.Popen") + mock_popen.side_effect = mock_not_called("subprocess.Popen") source_beanstalk_env_vars() @@ -76,3 +113,90 @@ def communicate(self): 'AWS_ACCESS_KEY_ID': '12345', 'AWS_FAKE_SECRET': 'amazon' } + +def test_magic_cnames(): + + # These tests are highly specific and will have to change if we make something else be magic. + # But such is magic. The values should not be casually changed, and such overhead is appropriate. + # It's good to have tests that will catch unwanted tinkering, typos, etc. + + # TODO: Think about whether we should use code like this to initialize these values rather than to test them. + # The need to edit sources just because we have to rebuild some stacks may be overly brittle. + # We could find other ways to test this if we did that. -kmp 31-Mar-2020 + + ff_magic_env = env_utils.FF_ENV_PRODUCTION_GREEN + cgap_magic_env = env_utils.CGAP_ENV_WEBPROD + + assert ff_magic_env == 'fourfront-green' + assert cgap_magic_env == 'fourfront-cgap' + + client = boto3.client('elasticbeanstalk', region_name=bs.REGION) + res = bs.describe_beanstalk_environments(client, ApplicationName='4dn-web') + envs = res['Environments'] + for env in envs: + env_name = env.get('EnvironmentName') + if env_name == ff_magic_env: + assert bs.FF_MAGIC_CNAME == env.get('CNAME') + elif env_name == cgap_magic_env: + assert bs.CGAP_MAGIC_CNAME == env.get('CNAME') + + +def test_deprecated_whodaman(): + # This just makes sure that the old name is properly retained, since it's used in a lot of other repos. + assert bs.whodaman is bs.compute_ff_prd_env + + +def test_compute_ff_pred_env(): + with mock.patch("dcicutils.beanstalk_utils._compute_prd_env_for_project") as mock_compute: + def mocked_compute_prd_env_for_project(project): + return project + "-prod-env" + mock_compute.side_effect = mocked_compute_prd_env_for_project + assert bs.compute_ff_prd_env() == 'ff-prod-env' + + +def test_compute_cgap_pred_env(): + with mock.patch("dcicutils.beanstalk_utils._compute_prd_env_for_project") as mock_compute: + def mocked_compute_prd_env_for_project(project): + return project + "-prod-env" + mock_compute.side_effect = mocked_compute_prd_env_for_project + assert bs.compute_cgap_prd_env() == 'cgap-prod-env' + + +def _mocked_describe_beanstalk_environments(*args, **kwargs): + print("Ignoring args=", args, "kwargs=", kwargs) + return { + 'Environments': [ + { + "CNAME": "not." + bs.CGAP_MAGIC_CNAME, + "EnvironmentName": "cgap-env-1" + }, + { + "CNAME": bs.CGAP_MAGIC_CNAME, + "EnvironmentName": "cgap-env-2" + }, + { + "CNAME": "also-not." + bs.CGAP_MAGIC_CNAME, + "EnvironmentName": "cgap-env-3" + }, + { + "CNAME": "not." + bs.FF_MAGIC_CNAME, + "EnvironmentName": "ff-env-1" + }, + { + "CNAME": bs.FF_MAGIC_CNAME, + "EnvironmentName": "ff-env-2" + }, + { + "CNAME": "also-not." + bs.FF_MAGIC_CNAME, + "EnvironmentName": "ff-env-3" + }, + ] + } + + +def test_compute_prd_env_for_project(): + with mock.patch("boto3.client"): + with mock.patch("dcicutils.beanstalk_utils.describe_beanstalk_environments") as mock_describer: + mock_describer.side_effect = _mocked_describe_beanstalk_environments + assert bs._compute_prd_env_for_project('cgap') == 'cgap-env-2' + assert bs._compute_prd_env_for_project('ff') == 'ff-env-2' diff --git a/test/test_env_utils.py b/test/test_env_utils.py index 28c21321f..300a4425b 100644 --- a/test/test_env_utils.py +++ b/test/test_env_utils.py @@ -10,10 +10,49 @@ CGAP_ENV_PRODUCTION_BLUE_NEW, CGAP_ENV_PRODUCTION_GREEN_NEW, CGAP_ENV_WEBPROD_NEW, CGAP_ENV_MASTERTEST_NEW, CGAP_ENV_HOTSEAT_NEW, CGAP_ENV_STAGING_NEW, CGAP_ENV_WEBDEV_NEW, CGAP_ENV_WOLF_NEW, get_mirror_env_from_context, is_test_env, is_hotseat_env, guess_mirror_env, + prod_bucket_env, public_url_mappings, CGAP_PUBLIC_URLS, FF_PUBLIC_URLS, FF_PROD_BUCKET_ENV, CGAP_PROD_BUCKET_ENV, ) from unittest import mock +def test_prod_bucket_env(): + + # Fourfront tests + + assert prod_bucket_env('fourfront-webprod') == FF_PROD_BUCKET_ENV + assert prod_bucket_env('fourfront-webprod2') == FF_PROD_BUCKET_ENV + + assert prod_bucket_env('fourfront-mastertest') is None + assert prod_bucket_env('fourfront-webdev') is None + + assert prod_bucket_env('fourfront-blue') == FF_PROD_BUCKET_ENV + assert prod_bucket_env('fourfront-green') == FF_PROD_BUCKET_ENV + + # CGAP tests + + assert prod_bucket_env('fourfront-cgap') == CGAP_PROD_BUCKET_ENV + + assert prod_bucket_env('fourfront-cgap-blue') == CGAP_PROD_BUCKET_ENV + assert prod_bucket_env('fourfront-cgap-green') == CGAP_PROD_BUCKET_ENV + + assert prod_bucket_env('fourfront-cgapdev') is None + assert prod_bucket_env('fourfront-cgapwolf') is None + + +def test_public_url_mappings(): + + assert public_url_mappings('fourfront-webprod') == FF_PUBLIC_URLS + assert public_url_mappings('fourfront-webprod2') == FF_PUBLIC_URLS + assert public_url_mappings('fourfront-blue') == FF_PUBLIC_URLS + assert public_url_mappings('fourfront-green') == FF_PUBLIC_URLS + + assert public_url_mappings('fourfront-cgap') == CGAP_PUBLIC_URLS + assert public_url_mappings('fourfront-cgap-blue') == CGAP_PUBLIC_URLS + assert public_url_mappings('fourfront-cgap-green') == CGAP_PUBLIC_URLS + assert public_url_mappings('cgap-blue') == CGAP_PUBLIC_URLS + assert public_url_mappings('cgap-green') == CGAP_PUBLIC_URLS + + def test_blue_green_mirror_env(): # Should work for basic fourfront diff --git a/test/test_qa_utils.py b/test/test_qa_utils.py new file mode 100644 index 000000000..60ac70d90 --- /dev/null +++ b/test/test_qa_utils.py @@ -0,0 +1,14 @@ +import re +from dcicutils.qa_utils import mock_not_called + + +def test_mock_not_called(): + name = "foo" + mocked_foo = mock_not_called(name) + try: + mocked_foo(1, 2, three=3) + except AssertionError as e: + m = re.match("%s.*called" % re.escape(name), str(e)) + assert m, "Expected assertion text did not appear." + else: + raise AssertionError("An AssertionError was not raised.") diff --git a/test/test_s3_utils.py b/test/test_s3_utils.py index 500e6a488..82c00667a 100644 --- a/test/test_s3_utils.py +++ b/test/test_s3_utils.py @@ -2,25 +2,67 @@ import pytest -def test_s3Utils_creation(): - util = s3Utils(env='fourfront-mastertest') - assert util.sys_bucket == 'elasticbeanstalk-fourfront-mastertest-system' - - -def test_s3Utils_creation_staging(): - util = s3Utils(env='staging') - assert util.sys_bucket == 'elasticbeanstalk-fourfront-webprod-system' - assert util.outfile_bucket == 'elasticbeanstalk-fourfront-webprod-wfoutput' - assert util.raw_file_bucket == 'elasticbeanstalk-fourfront-webprod-files' - assert util.url == 'http://staging.4dnucleome.org' - - -def test_s3Utils_creation_data(): - util = s3Utils(env='data') - assert util.sys_bucket == 'elasticbeanstalk-fourfront-webprod-system' - assert util.outfile_bucket == 'elasticbeanstalk-fourfront-webprod-wfoutput' - assert util.raw_file_bucket == 'elasticbeanstalk-fourfront-webprod-files' - assert util.url == 'https://data.4dnucleome.org' +@pytest.mark.parametrize('ff_ordinary_envname', ['fourfront-mastertest', 'fourfront-webdev', 'fourfront-hotseat']) +def test_s3Utils_creation(ff_ordinary_envname): + util = s3Utils(env=ff_ordinary_envname) + assert util.sys_bucket == 'elasticbeanstalk-%s-system' % ff_ordinary_envname + + +@pytest.mark.parametrize('ff_staging_envname', ['staging', 'fourfront-blue']) # fourfront-webprod2 no longer exists +def test_s3Utils_creation_staging(ff_staging_envname): + util = s3Utils(env=ff_staging_envname) + actual_props = { + 'sys_bucket': util.sys_bucket, + 'outfile_bucket': util.outfile_bucket, + 'raw_file_bucket': util.raw_file_bucket, + 'url': util.url, + } + assert actual_props == { + 'sys_bucket': 'elasticbeanstalk-fourfront-webprod-system', + 'outfile_bucket': 'elasticbeanstalk-fourfront-webprod-wfoutput', + 'raw_file_bucket': 'elasticbeanstalk-fourfront-webprod-files', + 'url': 'http://staging.4dnucleome.org', + } + + +@pytest.mark.parametrize('ff_production_envname', ['data', 'fourfront-green']) # fourfront-webprod no longer exists +def test_s3Utils_creation_data(ff_production_envname): + util = s3Utils(env=ff_production_envname) + actual_props = { + 'sys_bucket': util.sys_bucket, + 'outfile_bucket': util.outfile_bucket, + 'raw_file_bucket': util.raw_file_bucket, + 'url': util.url, + } + assert actual_props == { + 'sys_bucket': 'elasticbeanstalk-fourfront-webprod-system', + 'outfile_bucket': 'elasticbeanstalk-fourfront-webprod-wfoutput', + 'raw_file_bucket': 'elasticbeanstalk-fourfront-webprod-files', + 'url': 'https://data.4dnucleome.org', + } + + +@pytest.mark.parametrize('cgap_production_envname', ['cgap', 'fourfront-cgap']) +def test_s3Utils_creation_cgap(cgap_production_envname): + util = s3Utils(env=cgap_production_envname) + actual_props = { + 'sys_bucket': util.sys_bucket, + 'outfile_bucket': util.outfile_bucket, + 'raw_file_bucket': util.raw_file_bucket, + 'url': util.url, + } + assert actual_props == { + 'sys_bucket': 'elasticbeanstalk-fourfront-cgap-system', + 'outfile_bucket': 'elasticbeanstalk-fourfront-cgap-wfoutput', + 'raw_file_bucket': 'elasticbeanstalk-fourfront-cgap-files', + 'url': 'https://cgap.hms.harvard.edu', + } + + +@pytest.mark.parametrize('cgap_ordinary_envname', ['fourfront-cgaptest', 'fourfront-cgapdev', 'fourfront-cgapwolf']) +def test_s3Utils_creation(cgap_ordinary_envname): + util = s3Utils(env=cgap_ordinary_envname) + assert util.sys_bucket == 'elasticbeanstalk-%s-system' % cgap_ordinary_envname def test_s3Utils_get_keys_for_data():