Merge pull request #3 from dbmi-bgm/core

Core

.coveragerc

@@ -7,7 +7,6 @@ omit =
     ./tests/*
     conftest.py
     app.py
-    ./chalicelib/abstract_connection.py
     ./chalicelib/checks/helpers/*
     ./chalicelib/*_checks.py
     ./chalicelib/templates/*

.gitignore

@@ -5,12 +5,14 @@
 *.pyc
 __pycache__
 *.egg-info/
+dist/
 .coverage
 *.DS_Store
 htmlcov/
 coverage.xml
 *.ipynb
 !*LocalTest.ipynb
+requirements.txt
 
 # Virtual environments
 *env/

app.py

@@ -1,15 +1,16 @@
-from __future__ import print_function, unicode_literals
 from chalice import Chalice, Cron, Rate, Response
 import json
 import os
 import requests
 import datetime
-from chalicelib.app_utils import *
+from chalicelib.app_utils import AppUtils
 
 app = Chalice(app_name='foursight-cgap')
 app.debug = True
 STAGE = os.environ.get('chalice_stage', 'dev')
 DEFAULT_ENV = 'cgap'
+app_utils_obj = AppUtils()
+
 
 '''######### SCHEDULED FXNS #########'''
 
@@ -59,54 +60,54 @@ def end_of_day_on_weekdays():
 
 @app.schedule(foursight_cron_by_schedule[STAGE]['ten_min_checks'])
 def ten_min_checks(event):
-    queue_scheduled_checks('all', 'ten_min_checks')
+    app_utils_obj.queue_scheduled_checks('all', 'ten_min_checks')
 
 
 @app.schedule(foursight_cron_by_schedule[STAGE]['thirty_min_checks'])
 def thirty_min_checks(event):
-    queue_scheduled_checks('all', 'thirty_min_checks')
+    app_utils_obj.queue_scheduled_checks('all', 'thirty_min_checks')
 
 
 @app.schedule(foursight_cron_by_schedule[STAGE]['hourly_checks'])
 def hourly_checks(event):
-    queue_scheduled_checks('all', 'hourly_checks')
+    app_utils_obj.queue_scheduled_checks('all', 'hourly_checks')
 
 
 @app.schedule(foursight_cron_by_schedule[STAGE]['hourly_checks_2'])
 def hourly_checks_2(event):
-    queue_scheduled_checks('all', 'hourly_checks_2')
+    app_utils_obj.queue_scheduled_checks('all', 'hourly_checks_2')
 
 
 @app.schedule(foursight_cron_by_schedule[STAGE]['early_morning_checks'])
 def early_morning_checks(event):
-    queue_scheduled_checks('all', 'early_morning_checks')
+    app_utils_obj.queue_scheduled_checks('all', 'early_morning_checks')
 
 
 @app.schedule(foursight_cron_by_schedule[STAGE]['morning_checks'])
 def morning_checks(event):
-    queue_scheduled_checks('all', 'morning_checks')
+    app_utils_obj.queue_scheduled_checks('all', 'morning_checks')
 
 
 @app.schedule(foursight_cron_by_schedule[STAGE]['morning_checks_2'])
 def morning_checks_2(event):
-    queue_scheduled_checks('all', 'morning_checks_2')
+    app_utils_obj.queue_scheduled_checks('all', 'morning_checks_2')
 
 
 @app.schedule(foursight_cron_by_schedule[STAGE]['monday_checks'])
 def monday_checks(event):
-    queue_scheduled_checks('all', 'monday_checks')
+    app_utils_obj.queue_scheduled_checks('all', 'monday_checks')
 
 
 @app.schedule(foursight_cron_by_schedule[STAGE]['monthly_checks'])
 def monthly_checks(event):
-    queue_scheduled_checks('all', 'monthly_checks')
+    app_utils_obj.queue_scheduled_checks('all', 'monthly_checks')
 
 
 @app.schedule(foursight_cron_by_schedule[STAGE]['deployment_checks'])
 def deployment_checks(event):
     if STAGE == 'dev':
         return  # do not schedule the deployment checks on dev
-    queue_scheduled_checks('all', 'deployment_checks')
+    app_utils_obj.queue_scheduled_checks('all', 'deployment_checks')
 
 
 '''######### END SCHEDULED FXNS #########'''
@@ -120,7 +121,7 @@ def auth0_callback():
     """
     request = app.current_request
     req_dict = request.to_dict()
-    domain, context = get_domain_and_context(req_dict)
+    domain, context = app_utils_obj.get_domain_and_context(req_dict)
     # extract redir cookie
     cookies = req_dict.get('headers', {}).get('cookie')
     redir_url = context + 'view/' + DEFAULT_ENV
@@ -131,7 +132,7 @@ def auth0_callback():
     resp_headers = {'Location': redir_url}
     params = req_dict.get('query_params')
     if not params:
-        return forbidden_response()
+        return app_utils_obj.forbidden_response()
     auth0_code = params.get('code', None)
     auth0_client = os.environ.get('CLIENT_ID', None)
     auth0_secret = os.environ.get('CLIENT_SECRET', None)
@@ -165,7 +166,7 @@ def index():
     Redirect with 302 to view page of DEFAULT_ENV
     Non-protected route
     """
-    domain, context = get_domain_and_context(app.current_request.to_dict())
+    domain, context = app_utils_obj.get_domain_and_context(app.current_request.to_dict())
     resp_headers = {'Location': context + 'view/' + DEFAULT_ENV}
     return Response(status_code=302, body=json.dumps(resp_headers),
                     headers=resp_headers)
@@ -176,11 +177,11 @@ def introspect(environ):
     """
     Test route
     """
-    auth = check_authorization(app.current_request.to_dict(), environ)
+    auth = app_utils_obj.check_authorization(app.current_request.to_dict(), environ)
     if auth:
         return Response(status_code=200, body=json.dumps(app.current_request.to_dict()))
     else:
-        return forbidden_response()
+        return app_utils_obj.forbidden_response()
 
 
 @app.route('/view_run/{environ}/{check}/{method}', methods=['GET'])
@@ -189,15 +190,15 @@ def view_run_route(environ, check, method):
     Protected route
     """
     req_dict = app.current_request.to_dict()
-    domain, context = get_domain_and_context(req_dict)
+    domain, context = app_utils_obj.get_domain_and_context(req_dict)
     query_params = req_dict.get('query_params', {})
-    if check_authorization(req_dict, environ):
+    if app_utils_obj.check_authorization(req_dict, environ):
         if method == 'action':
-            return view_run_action(environ, check, query_params, context)
+            return app_utils_obj.view_run_action(environ, check, query_params, context)
         else:
-            return view_run_check(environ, check, query_params, context)
+            return app_utils_obj.view_run_check(environ, check, query_params, context)
     else:
-        return forbidden_response(context)
+        return app_utils_obj.forbidden_response(context)
 
 
 @app.route('/view/{environ}', methods=['GET'])
@@ -206,8 +207,8 @@ def view_route(environ):
     Non-protected route
     """
     req_dict = app.current_request.to_dict()
-    domain, context = get_domain_and_context(req_dict)
-    return view_foursight(environ, check_authorization(req_dict, environ), domain, context)
+    domain, context = app_utils_obj.get_domain_and_context(req_dict)
+    return app_utils_obj.view_foursight(environ, app_utils_obj.check_authorization(req_dict, environ), domain, context)
 
 
 @app.route('/view/{environ}/{check}/{uuid}', methods=['GET'])
@@ -216,11 +217,11 @@ def view_check_route(environ, check, uuid):
     Protected route
     """
     req_dict = app.current_request.to_dict()
-    domain, context = get_domain_and_context(req_dict)
-    if check_authorization(req_dict, environ):
-        return view_foursight_check(environ, check, uuid, True, domain, context)
+    domain, context = app_utils_obj.get_domain_and_context(req_dict)
+    if app_utils_obj.check_authorization(req_dict, environ):
+        return app_utils_obj.view_foursight_check(environ, check, uuid, True, domain, context)
     else:
-        return forbidden_response()
+        return app_utils_obj.forbidden_response()
 
 
 @app.route('/history/{environ}/{check}', methods=['GET'])
@@ -233,31 +234,31 @@ def history_route(environ, check):
     query_params = req_dict.get('query_params')
     start = int(query_params.get('start', '0')) if query_params else 0
     limit = int(query_params.get('limit', '25')) if query_params else 25
-    domain, context = get_domain_and_context(req_dict)
-    return view_foursight_history(environ, check, start, limit,
-                                  check_authorization(req_dict, environ), domain, context)
+    domain, context = app_utils_obj.get_domain_and_context(req_dict)
+    return app_utils_obj.view_foursight_history(environ, check, start, limit,
+                                  app_utils_obj.check_authorization(req_dict, environ), domain, context)
 
 
 @app.route('/checks/{environ}/{check}/{uuid}', methods=['GET'])
 def get_check_with_uuid_route(environ, check, uuid):
     """
     Protected route
     """
-    if check_authorization(app.current_request.to_dict(), environ):
-        return run_get_check(environ, check, uuid)
+    if app_utils_obj.check_authorization(app.current_request.to_dict(), environ):
+        return app_utils_obj.run_get_check(environ, check, uuid)
     else:
-        return forbidden_response()
+        return app_utils_obj.forbidden_response()
 
 
 @app.route('/checks/{environ}/{check}', methods=['GET'])
 def get_check_route(environ, check):
     """
     Protected route
     """
-    if check_authorization(app.current_request.to_dict(), environ):
-        return run_get_check(environ, check, None)
+    if app_utils_obj.check_authorization(app.current_request.to_dict(), environ):
+        return app_utils_obj.run_get_check(environ, check, None)
     else:
-        return forbidden_response()
+        return app_utils_obj.forbidden_response()
 
 
 @app.route('/checks/{environ}/{check}', methods=['PUT'])
@@ -272,11 +273,11 @@ def put_check_route(environ, check):
     Protected route
     """
     request = app.current_request
-    if check_authorization(request.to_dict(), environ):
+    if app_utils_obj.check_authorization(request.to_dict(), environ):
         put_data = request.json_body
-        return run_put_check(environ, check, put_data)
+        return app_utils_obj.run_put_check(environ, check, put_data)
     else:
-        return forbidden_response()
+        return app_utils_obj.forbidden_response()
 
 
 @app.route('/environments/{environ}', methods=['PUT'])
@@ -290,22 +291,22 @@ def put_environment(environ):
     Protected route
     """
     request = app.current_request
-    if check_authorization(request.to_dict(), environ):
+    if app_utils_obj.check_authorization(request.to_dict(), environ):
         env_data = request.json_body
-        return run_put_environment(environ, env_data)
+        return app_utils_obj.run_put_environment(environ, env_data)
     else:
-        return forbidden_response()
+        return app_utils_obj.forbidden_response()
 
 
 @app.route('/environments/{environ}', methods=['GET'])
 def get_environment_route(environ):
     """
     Protected route
     """
-    if check_authorization(app.current_request.to_dict(), environ):
-        return run_get_environment(environ)
+    if app_utils_obj.check_authorization(app.current_request.to_dict(), environ):
+        return app_utils_obj.run_get_environment(environ)
     else:
-        return forbidden_response()
+        return app_utils_obj.forbidden_response()
 
 
 @app.route('/environments/{environ}/delete', methods=['DELETE'])
@@ -317,10 +318,10 @@ def delete_environment(environ):
 
     Protected route
     """
-    if check_authorization(app.current_request.to_dict(), environ):  # TODO (C4-138) Centralize authorization check
-        return run_delete_environment(environ)
+    if app_utils_obj.check_authorization(app.current_request.to_dict(), environ):  # TODO (C4-138) Centralize authorization check
+        return app_utils_obj.run_delete_environment(environ)
     else:
-        return forbidden_response()
+        return app_utils_obj.forbidden_response()
 
 
 ######### PURE LAMBDA FUNCTIONS #########
@@ -334,23 +335,17 @@ def check_runner(event, context):
     """
     if not event:
         return
-    run_check_runner(event)
+    app_utils_obj.run_check_runner(event)
 
 ######### MISC UTILITY FUNCTIONS #########
 
 
 def set_stage(stage):
-    from deploy import CONFIG_BASE
-    if stage != 'test' and stage not in CONFIG_BASE['stages']:
-        print('ERROR! Input stage is not valid. Must be one of: %s' % str(list(CONFIG_BASE['stages'].keys()).extend('test')))
+    from deploy import Deploy
+    if stage != 'test' and stage not in Deploy.CONFIG_BASE['stages']:
+        print('ERROR! Input stage is not valid. Must be one of: %s' % str(list(Deploy.CONFIG_BASE['stages'].keys()).extend('test')))
     os.environ['chalice_stage'] = stage
 
 
 def set_timeout(timeout):
-    from chalicelib import utils
-    try:
-        timeout = int(timeout)
-    except ValueError:
-        print('ERROR! Timeout must be an integer. You gave: %s' % timeout)
-    else:
-        utils.CHECK_TIMEOUT = timeout
+    app_utils_obj.set_timeout(timeout)

buckets.py

@@ -1,32 +1,13 @@
 import boto3
 import json
-from chalicelib.vars import FOURSIGHT_PREFIX
+from .chalicelib.vars import FOURSIGHT_PREFIX
+from foursight_core.buckets import Buckets as Buckets_from_core
 
-class Buckets(object):
+class Buckets(Buckets_from_core):
     """create and configure buckets for foursight"""
 
     prefix = FOURSIGHT_PREFIX
     envs = ['cgap', 'cgapdev', 'cgaptest', 'cgapwolf']
-    default_acl = 'private'
-    region = 'us-east-1'
-
-    def __init__(self):
-        pass
-
-    @property
-    def bucket_names(self):
-        dev_suffices = ['dev-' + env for env in self.envs]
-        prod_suffices = ['prod-' + env for env in self.envs]
-        test_suffices = ['test-s3', 'unit-test-envs']
-        suffices = dev_suffices + prod_suffices + test_suffices + ['runs', 'envs']
-        return [self.prefix + '-' + suffix for suffix in suffices]
-
-    @property
-    def env_bucket(self):
-        return self.prefix + '-envs'
-
-    def ff_env(self, env):
-        return 'fourfront-%s' % env
 
     def ff_url(self, env):
         if env == 'cgap':
@@ -40,28 +21,6 @@ def es_url(self, env):
         else:
             return "https://search-cgap-testing-6-8-vo4mdkmkshvmyddc65ux7dtaou.us-east-1.es.amazonaws.com"
 
-    def create_buckets(self):
-        s3 = boto3.client('s3')
-        for bucket in self.bucket_names:
-            param = {'Bucket': bucket, 'ACL': self.default_acl}
-            if self.region != 'us-east-1':
-                param.update({'CreateBucketConfiguration': {'LocationConstraint': self.region}})
-            s3.create_bucket(**param)
-
-    def configure_env_bucket(self):
-        s3 = boto3.client('s3')
-        try:
-            s3.head_bucket(self.env_bucket)  # check if bucket exists
-        except Exception as e:
-            if 'NoSuchBucket' in str(e):
-                print("first create buckets! %s" % str(e))
-        for env in self.envs:
-            content = {"fourfront": self.ff_url(env),
-                       "es": self.es_url(env),
-                       "ff_env": self.ff_env(env)}
-            body = json.dumps(content).encode('utf-8')
-            s3.put_object(Bucket=self.env_bucket, Key=env, Body=body)
-
 
 def main():
     buckets = Buckets()