diff --git a/CHANGELOG.md b/CHANGELOG.md index 65764b3..e7ac5b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -1.6.0 - 1/24/2024 +1.6.2 Live-docs: https://tapis-project.github.io/live-docs/?service=GlobusProxy @@ -6,12 +6,7 @@ Breaking Changes: - auth flow has been reworked to allow for v5 endpoints - users will need to refresh their auth tokens New features: - - created functional tests - unit tests were inadequate for certain functions. These must be run from inside the container - - better handling of personal connect endpoints - - initial support for additional consent auth flow - - initial support for consent management - - more descriptive response codes for several endpoints + - support for GCP collections Bug fixes: - - catch consent required errors instead of returning a 500 code - - fixed imports of functions + - none diff --git a/config-local.json b/config-local.json index 550fa82..66b675e 100644 --- a/config-local.json +++ b/config-local.json @@ -4,5 +4,5 @@ "service_site_id": "tacc", "service_tenant_id": "admin", "log_level": "DEBUG", - "tenants": ["dev"] + "tenants": ["dev"], } diff --git a/docker-compose.yml b/docker-compose.yml index b45aa14..7e54186 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,6 @@ services: - ./configschema.json:/home/tapis/configschema.json - ./config-local.json:/home/tapis/config.json - ./service.log:/home/tapis/service.log - - ../gpsettings.json:/home/tapis/gpsettings.json container_name: globus-proxy networks: - tapis diff --git a/service/api.py b/service/api.py index f404cdc..20a36a2 100644 --- a/service/api.py +++ b/service/api.py @@ -25,7 +25,6 @@ def __init__(self, url_map, *items): api = TapisApi(app, errors=flask_errors_dict) app.url_map.converters['regex'] = RegexConverter - # Set up error handling api.handle_error = handle_error api.handle_exception = handle_error diff --git a/service/controllers/auth.py b/service/controllers/auth.py index 760c857..697c4af 100644 --- a/service/controllers/auth.py +++ b/service/controllers/auth.py @@ -21,6 +21,7 @@ def get(self, client_id, endpoint_id): # TODO: a call to get_identities needs to be authenticated # there might not be a way to figure out if the client_id # is valid or not without letting the user go to the url and find out + try: client = globus_sdk.NativeAppAuthClient(client_id) except Exception as e: @@ -28,8 +29,14 @@ def get(self, client_id, endpoint_id): logger.error(msg) raise InternalServerError(msg=msg) - DEPENDENT_SCOPE = f"https://auth.globus.org/scopes/{endpoint_id}/data_access" - SCOPE = f"urn:globus:auth:scope:transfer.api.globus.org:all[*{DEPENDENT_SCOPE}]" + # build scopes + SCOPE = None + if not is_gcp(endpoint_id): + logger.debug(f'endpoint is not a gcp') + DEPENDENT_SCOPE = f"https://auth.globus.org/scopes/{endpoint_id}/data_access" + SCOPE = f"urn:globus:auth:scope:transfer.api.globus.org:all[*{DEPENDENT_SCOPE}]" + else: + logger.debug(f'endpoint is a gcp') session_client = client.oauth2_start_flow(refresh_tokens=True, requested_scopes=SCOPE) @@ -38,7 +45,7 @@ def get(self, client_id, endpoint_id): logger.debug(f"successfully got auth url for client {client_id}") return utils.ok( result = {"url": authorize_url, "session_id": session_client.verifier}, - msg = f'Please go to the URL and login' + msg = f'Please go to the URL and login.' ) class TokensResource(Resource): diff --git a/service/errors.py b/service/errors.py index 0035a50..752c161 100644 --- a/service/errors.py +++ b/service/errors.py @@ -27,15 +27,15 @@ def __init__(self, msg="Given path is not found on the endpoint", code=404): class GlobusError(BaseTapisError): """General error with the Globus SDK""" - def __init__(self, msg="Uncaught Globus error", code=407): + def __init__(self, msg="Uncaught Globus error", code=400): super().__init__(msg, code) class GlobusConsentRequired(BaseTapisError): - def __init__(self, msg="Endpoint requires consent", code=407): + def __init__(self, msg="Endpoint requires consent", code=403): super().__init__(msg, code) class GlobusUnauthorized(BaseTapisError): - def __init__(self, msg="Permission denied", code=407): + def __init__(self, msg="Permission denied", code=401): msg=f"You do not have permission to perform that operation on this endpoint:: {msg}" super().__init__(msg, code) diff --git a/service/tests.py b/service/tests.py index 7493dbb..08fd844 100644 --- a/service/tests.py +++ b/service/tests.py @@ -12,6 +12,7 @@ # local from utils import * +from controllers.auth import * logger = get_logger(__name__) @@ -20,10 +21,16 @@ class Base: config = None cid = None gcp_eid = None + ls6_eid = None + frontera_eid = None + corral_eid = None source = None at = None rt = None + gcp_at = None + gcp_rt = None base_url = None + secret = None def __init__(self): self.config = self.load_config() @@ -36,14 +43,49 @@ def load_config(self): self.source_eid = "722751ce-1264-43b8-9160-a9272f746d78" # ESnet CERN DTN (Anonymous read-only testing) self.at = self.config["access_token"] self.rt = self.config["refresh_token"] + self.gcp_at = self.config["gcp_at"] + self.gcp_rt = self.config["gcp_rt"] self.base_url = self.config["base_url"] + self.secret = self.config['client_secret'] + self.ls6_eid = self.config['ls6_endpoint_id'] return self.config +# TODO: write setup / teardown functions that can be run by the tests indivudially so the success of one test isn't dependant on another +## setup / teardown +def make_dir(path): + os.mkdir(path) +def rm_dir(path): + os.rmdir(path) + +## utils def get_endpoint_test(epid): info = get_collection_type(epid) print(f'got ep info:: {info}') - + +def get_collection_id_test(client_id, client_secret, name): + info = get_collection_id(client_id, client_secret, name) + print(f'got ep id:: {info}') + +def is_gcp_test(endpoint_id): + res = is_gcp(endpoint_id) + print(res) + +# def get_gcp_auth_url_test(client_id, client_secret, endpoint_id=None) +# print(f'about to fetch url for {endpoint_id} using client {client_id}') + + +def get_transfer_client_with_secret_test(client_id, client_secret, endpoint_id=None, addl_scopes=None): + print(f'about to auth {client_id} with endpoint {endpoint_id} and scopes {addl_scopes}') + tc = get_transfer_client_with_secret(client_id, client_secret, endpoint_id=endpoint_id, addl_scopes=addl_scopes) + print(f'authd:: {tc}') + +def build_scopes_test(scopes:list, eid=None): + print(f'about to test building a scope') + s_list = [''] + scopes = build_scopes(s_list, collection_id=eid) + +## ops def ls_test(base, path): url = f'{base.base_url}/ops/{base.cid}/{base.gcp_eid}/{path}' query = {"access_token": base.at, @@ -75,6 +117,7 @@ def mkdir_test(base, path): if response.status_code != 200: raise Exception(f'{response.status_code}:: {response.text}') +## transfers def make_xfer_test(base): pass @@ -97,40 +140,75 @@ def rm_xfer_test(base): # print(e) # fails['base_test_1'] = e # ls tests + # try: + # ls_test(base, base_path) + # except Exception as e: + # print(e) + # fails['ls_test_1'] = e + # try: + # ls_test(base, f'{base_path}/test') + # except Exception as e: + # print(e) + # fails['ls_test_2'] = e + # try: + # ls_test(base, f'{base_path}/test.py') + # except Exception as e: + # print(e) + # fails['ls_test_3'] = e + # # mkdir tests + # try: + # mkdir_test(base, f'{base_path}/mkdirtest') + # except Exception as e: + # print(e) + # fails['mkdir_test_1'] = e + # # rename tests + # try: + # mv_test(base, f'{base_path}/mkdirtest', f'{base_path}/mkdirtest2') + # except Exception as e: + # print(e) + # fails['mv_test_1'] = e + # # rm tests + # try: + # rm_test(base, f'{base_path}/mkdirtest2') + # except Exception as e: + # print(e) + # fails['rm_test_1'] = e + try: - ls_test(base, base_path) - except Exception as e: - print(e) - fails['ls_test_1'] = e - try: - ls_test(base, f'{base_path}/test') - except Exception as e: - print(e) - fails['ls_test_2'] = e - try: - ls_test(base, f'{base_path}/test.py') - except Exception as e: - print(e) - fails['ls_test_3'] = e - # mkdir tests - try: - mkdir_test(base, f'{base_path}/mkdirtest') - except Exception as e: - print(e) - fails['mkdir_test_1'] = e - # rename tests - try: - mv_test(base, f'{base_path}/mkdirtest', f'{base_path}/mkdirtest2') + client_id = base.cid + client_secret = base.secret + get_transfer_client_with_secret_test(client_id, client_secret) except Exception as e: print(e) - fails['mv_test_1'] = e - # rm tests + fails['auth_test_1'] = e + try: - rm_test(base, f'{base_path}/mkdirtest2') + client_id = base.cid + client_secret = base.secret + endpoint_id = base.ls6_eid + get_transfer_client_with_secret_test(client_id, client_secret, endpoint_id=endpoint_id) except Exception as e: print(e) - fails['rm_test_1'] = e - + fails['auth_test_2'] = e + + # try: + # client_id = base.cid + # client_secret = base.secret + # endpoint_id = base.gcp_eid + # get_gcp_auth_url_test(client_id, client_secret, endpoint_id=endpoint_id) + # except Exception as e: + # print(e) + # fails['gcp_test_1'] = e + + # try: + # client_id = base.cid + # client_secret = base.secret + # endpoint_id = base.ls6_eid + # addl_scopes = 'manage_projects' + # get_transfer_client_with_secret_test(client_id, client_secret, endpoint_id=endpoint_id, addl_scopes=addl_scopes) + # except Exception as e: + # print(e) + # fails['auth_test_3'] = e except Exception as e: print(f'Unknown error running tests:: {e}') diff --git a/service/utils.py b/service/utils.py index da83668..a65906b 100644 --- a/service/utils.py +++ b/service/utils.py @@ -6,10 +6,12 @@ ## globus import globus_sdk -from globus_sdk.scopes import TransferScopes +from globus_sdk.scopes import ScopeBuilder, GCSCollectionScopeBuilder, TransferScopes, AuthScopes +from globus_sdk.services.transfer.errors import TransferAPIError ## tapis from tapisservice.logs import get_logger +from tapisservice.config import conf as config from tapisservice.tapisflask import utils from tapisservice.errors import AuthenticationError @@ -18,28 +20,6 @@ logger = get_logger(__name__) -def get_collection_type(endpoint_id): # TODO - ''' - Given endpoint id, return type of collection - Requires that we have a client ID and client secret in the /globus-proxy/env file - ''' - # _user = os.environ.get("") - # _pass = os.environ.get("") - client_id = '700bc50b-241c-4805-a4fe-6bd72e50062e' - client_secret = 'eHmt/LxxbQqua73tyyQX7G0zJpDXOMQ0oBP/ld+SrS0=' - auth_client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) - token_response = auth_client.oauth2_client_credentials_tokens().by_resource_server - logger.debug(f'in get_collection_type, got token resp: {token_response}') - - scopes = "urn:globus:auth:scope:transfer.api.globus.org:all" - at = token_response["transfer.api.globus.org"]["access_token"] - logger.debug(f'got at: {at}') - cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(auth_client, scopes) - tk_authorizer = globus_sdk.AccessTokenAuthorizer(at) - # create a new client - transfer_client = globus_sdk.TransferClient(authorizer=tk_authorizer) - transfer_client.get_endpoint(endpoint_id) - def activate_endpoint(tc, ep_id, username, password): ''' @@ -136,7 +116,17 @@ def format_path(path, default_dir=None): return f"/{path.rstrip('/').lstrip('/')}" + +def get_collection_id(client_id, client_secret, name): + client = get_transfer_client_with_secret(client_id, client_secret) + result = client.endpoint_search(filter_fulltext=name, filter_non_functional=False, limit=1) + # print(f'have result:: {result["DATA"][0]["id"]}') + return result["DATA"][0]["id"] + + def get_transfer_client(client_id, refresh_token, access_token): + logger.debug(f'Attempting auth for client {client_id} using token') + print(f'Attempting auth for client {client_id} using token') client = globus_sdk.NativeAppAuthClient(client_id) # check_token(client_id, refresh_token, access_token) tomorrow = datetime.today() + timedelta(days=1) @@ -151,14 +141,36 @@ def get_transfer_client(client_id, refresh_token, access_token): transfer_client = globus_sdk.TransferClient(authorizer=authorizer) return transfer_client -def get_token_introspect(client_id, refresh_token): - logger.debug(f'authed {client_id} with ') - CLIENT_ID = '0ffd2a38-27e0-48c5-a870-bcb964237439' - CLIENT_SECRET = '+P3dXBG0BE26dLui8HiLQEj8VH+kcbQ/7GyVJzxsOco=' - ac = globus_sdk.ConfidentialAppAuthClient(CLIENT_ID, CLIENT_SECRET) - data = ac.oauth2_token_introspect(refresh_token, include="identity_set") - for identity in data["identity_set"]: - logger.debug(f'token authenticates for "{identity}"') + +def get_transfer_client_with_secret(client_id, client_secret, endpoint_id=None, addl_scopes=None): + logger.debug(f'Attempting auth for client {client_id} using secret') + try: + auth_client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) + except Exception as e: + msg = f'exception:: {e}' + print(msg) + logger.critical(msg) + raise handle_transfer_error(e) + scopes = f'urn:globus:auth:scope:transfer.api.globus.org:all' + if endpoint_id: + access_scope = f'[*https://auth.globus.org/scopes/{endpoint_id}/data_access]' + scopes = scopes + access_scope + + logger.debug(scopes) + try: + cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(auth_client, scopes) + except Exception as e: + logger.critical(f'failure instantiating cc_auth:: {e}') + raise handle_transfer_error(e) + # create a new client + try: + transfer_client = globus_sdk.TransferClient(authorizer=cc_authorizer) + except Exception as e: + logger.critical(f'failure getting transfer client:: {e}') + raise handle_transfer_error(e) + logger.debug(f'got client: {transfer_client} with endpoint: {endpoint_id} and scopes: {scopes}') + return transfer_client + def get_valid_token(client_id, refresh_token): ''' @@ -197,7 +209,8 @@ def handle_transfer_error(exception, endpoint_id=None, msg=None): error = GlobusUnauthorized(msg=e.http_reason) if exception.code == 'EndpointError': if exception.http_reason == 'Bad Gateway': - return + error = InternalServerError(msg="Bad Gateway", code=502) + logger.error(error) return error def is_endpoint_activated(tc, ep): @@ -223,6 +236,33 @@ def is_endpoint_connected(transfer_client, endpoint_id): # return connected return True +def is_gcp(endpoint_id): + ''' + Given endpoint id, return type of collection + Requires that we have a client ID and client secret in the config-local.json file + ''' + logger.error(f'is in gcp with eid: {endpoint_id}') + tapisconf = config.get_config_from_file() + client_id = tapisconf['client_id'] + client_secret = tapisconf['client_secret'] + res = {} + + client = get_transfer_client_with_secret(client_id, client_secret) + try: + res = client.get_endpoint(endpoint_id) + except TransferAPIError as e: + # assume it's a gcp + logger.error(f'got error checking collection type: {e}') + res['is_globus_connect'] = 'true' + except: + logger.error(f'got error checking collection type: {e}') + raise handle_transfer_error(e) + + + gcp = True if res["is_globus_connect"] == 'true' else False + logger.debug(f'Is collection {endpoint_id} a gcp? : {gcp}') + return gcp + def ls_endpoint(tc, ep_id, path="~"): ls = tc.operation_ls(ep_id, path=path) return ls @@ -257,6 +297,7 @@ def precheck(client_id, endpoints, access_token, refresh_token): transfer_client = None try: transfer_client = get_transfer_client(client_id, refresh_token, access_token) + # transfer_client = get)get_transfer_client_with_secret(client_id, ) except Exception as e: logger.error(f'unable to get transfer client or client {client_id}: {e}') raise GlobusError(msg='Exception while generating authorization. Please check your request syntax and try again')