From da33760286701ccbd4d54e02527c75dfbfaf04cc Mon Sep 17 00:00:00 2001 From: kevinpricethesecond Date: Fri, 12 Apr 2024 13:36:41 -0500 Subject: [PATCH 1/6] trying client creds --- config-local.json | 4 +- docker-compose.yml | 2 +- service/controllers/auth.py | 13 +++- service/errors.py | 6 +- service/tests.py | 118 +++++++++++++++++++++++++++--------- service/utils.py | 100 ++++++++++++++++++++---------- 6 files changed, 173 insertions(+), 70 deletions(-) diff --git a/config-local.json b/config-local.json index 550fa82..2038f46 100644 --- a/config-local.json +++ b/config-local.json @@ -4,5 +4,7 @@ "service_site_id": "tacc", "service_tenant_id": "admin", "log_level": "DEBUG", - "tenants": ["dev"] + "tenants": ["dev"], + "client_id": "0ffd2a38-27e0-48c5-a870-bcb964237439", + "client_secret": "+P3dXBG0BE26dLui8HiLQEj8VH+kcbQ/7GyVJzxsOco=" } diff --git a/docker-compose.yml b/docker-compose.yml index ed816f6..1a6c760 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ version: "3.5" services: globus-proxy: - # image: tapis/globus-proxy + image: tapis/globus-proxy build: . volumes: - ./configschema.json:/home/tapis/configschema.json diff --git a/service/controllers/auth.py b/service/controllers/auth.py index 760c857..768172a 100644 --- a/service/controllers/auth.py +++ b/service/controllers/auth.py @@ -21,6 +21,7 @@ def get(self, client_id, endpoint_id): # TODO: a call to get_identities needs to be authenticated # there might not be a way to figure out if the client_id # is valid or not without letting the user go to the url and find out + try: client = globus_sdk.NativeAppAuthClient(client_id) except Exception as e: @@ -28,8 +29,14 @@ def get(self, client_id, endpoint_id): logger.error(msg) raise InternalServerError(msg=msg) - DEPENDENT_SCOPE = f"https://auth.globus.org/scopes/{endpoint_id}/data_access" - SCOPE = f"urn:globus:auth:scope:transfer.api.globus.org:all[*{DEPENDENT_SCOPE}]" + # build scopes + SCOPE = None + if not is_gcp(endpoint_id): + logger.debug(f'endpoint is not a gcp') + DEPENDENT_SCOPE = f"https://auth.globus.org/scopes/{endpoint_id}/data_access" + SCOPE = f"urn:globus:auth:scope:transfer.api.globus.org:all[*{DEPENDENT_SCOPE}]" + else: + logger.debug(f'logger is a gcp') session_client = client.oauth2_start_flow(refresh_tokens=True, requested_scopes=SCOPE) @@ -38,7 +45,7 @@ def get(self, client_id, endpoint_id): logger.debug(f"successfully got auth url for client {client_id}") return utils.ok( result = {"url": authorize_url, "session_id": session_client.verifier}, - msg = f'Please go to the URL and login' + msg = f'Please go to the URL and login. this is a test' ) class TokensResource(Resource): diff --git a/service/errors.py b/service/errors.py index 0035a50..752c161 100644 --- a/service/errors.py +++ b/service/errors.py @@ -27,15 +27,15 @@ def __init__(self, msg="Given path is not found on the endpoint", code=404): class GlobusError(BaseTapisError): """General error with the Globus SDK""" - def __init__(self, msg="Uncaught Globus error", code=407): + def __init__(self, msg="Uncaught Globus error", code=400): super().__init__(msg, code) class GlobusConsentRequired(BaseTapisError): - def __init__(self, msg="Endpoint requires consent", code=407): + def __init__(self, msg="Endpoint requires consent", code=403): super().__init__(msg, code) class GlobusUnauthorized(BaseTapisError): - def __init__(self, msg="Permission denied", code=407): + def __init__(self, msg="Permission denied", code=401): msg=f"You do not have permission to perform that operation on this endpoint:: {msg}" super().__init__(msg, code) diff --git a/service/tests.py b/service/tests.py index 7493dbb..731bf2c 100644 --- a/service/tests.py +++ b/service/tests.py @@ -20,10 +20,14 @@ class Base: config = None cid = None gcp_eid = None + ls6_eid = None + frontera_eid = None + corral_eid = None source = None at = None rt = None base_url = None + secret = None def __init__(self): self.config = self.load_config() @@ -37,13 +41,42 @@ def load_config(self): self.at = self.config["access_token"] self.rt = self.config["refresh_token"] self.base_url = self.config["base_url"] + self.secret = self.config['client_secret'] + self.ls6_eid = self.config['ls6_endpoint_id'] return self.config +# TODO: write setup / teardown functions that can be run by the tests indivudially so the success of one test isn't dependant on another +## setup / teardown +def make_dir(path): + os.mkdir(path) +def rm_dir(path): + os.rmdir(path) + +## utils def get_endpoint_test(epid): info = get_collection_type(epid) print(f'got ep info:: {info}') - + +def get_collection_id_test(client_id, client_secret, name): + info = get_collection_id(client_id, client_secret, name) + print(f'got ep id:: {info}') + +def is_gcp_test(endpoint_id): + res = is_gcp(endpoint_id) + print(res) + +def get_transfer_client_with_secret_test(client_id, client_secret, endpoint_id=None, addl_scopes=None): + print(f'about to auth {client_id} with endpoint {endpoint_id} and scopes {addl_scopes}') + tc = get_transfer_client_with_secret(client_id, client_secret, endpoint_id=endpoint_id, addl_scopes=addl_scopes) + print(f'authd:: {tc}') + +def build_scopes_test(scopes:list, eid=None): + print(f'about to test building a scope') + s_list = [''] + scopes = build_scopes(s_list, collection_id=eid) + +## ops def ls_test(base, path): url = f'{base.base_url}/ops/{base.cid}/{base.gcp_eid}/{path}' query = {"access_token": base.at, @@ -75,6 +108,7 @@ def mkdir_test(base, path): if response.status_code != 200: raise Exception(f'{response.status_code}:: {response.text}') +## transfers def make_xfer_test(base): pass @@ -97,40 +131,66 @@ def rm_xfer_test(base): # print(e) # fails['base_test_1'] = e # ls tests + # try: + # ls_test(base, base_path) + # except Exception as e: + # print(e) + # fails['ls_test_1'] = e + # try: + # ls_test(base, f'{base_path}/test') + # except Exception as e: + # print(e) + # fails['ls_test_2'] = e + # try: + # ls_test(base, f'{base_path}/test.py') + # except Exception as e: + # print(e) + # fails['ls_test_3'] = e + # # mkdir tests + # try: + # mkdir_test(base, f'{base_path}/mkdirtest') + # except Exception as e: + # print(e) + # fails['mkdir_test_1'] = e + # # rename tests + # try: + # mv_test(base, f'{base_path}/mkdirtest', f'{base_path}/mkdirtest2') + # except Exception as e: + # print(e) + # fails['mv_test_1'] = e + # # rm tests + # try: + # rm_test(base, f'{base_path}/mkdirtest2') + # except Exception as e: + # print(e) + # fails['rm_test_1'] = e + try: - ls_test(base, base_path) - except Exception as e: - print(e) - fails['ls_test_1'] = e - try: - ls_test(base, f'{base_path}/test') - except Exception as e: - print(e) - fails['ls_test_2'] = e - try: - ls_test(base, f'{base_path}/test.py') - except Exception as e: - print(e) - fails['ls_test_3'] = e - # mkdir tests - try: - mkdir_test(base, f'{base_path}/mkdirtest') - except Exception as e: - print(e) - fails['mkdir_test_1'] = e - # rename tests - try: - mv_test(base, f'{base_path}/mkdirtest', f'{base_path}/mkdirtest2') + client_id = base.cid + client_secret = base.secret + get_transfer_client_with_secret_test(client_id, client_secret) except Exception as e: print(e) - fails['mv_test_1'] = e - # rm tests + fails['auth_test_1'] = e + try: - rm_test(base, f'{base_path}/mkdirtest2') + client_id = base.cid + client_secret = base.secret + endpoint_id = base.ls6_eid + get_transfer_client_with_secret_test(client_id, client_secret, endpoint_id=endpoint_id) except Exception as e: print(e) - fails['rm_test_1'] = e - + fails['auth_test_2'] = e + + # try: + # client_id = base.cid + # client_secret = base.secret + # endpoint_id = base.ls6_eid + # addl_scopes = 'manage_projects' + # get_transfer_client_with_secret_test(client_id, client_secret, endpoint_id=endpoint_id, addl_scopes=addl_scopes) + # except Exception as e: + # print(e) + # fails['auth_test_3'] = e except Exception as e: print(f'Unknown error running tests:: {e}') diff --git a/service/utils.py b/service/utils.py index d5712ee..d00367b 100644 --- a/service/utils.py +++ b/service/utils.py @@ -6,10 +6,11 @@ ## globus import globus_sdk -from globus_sdk.scopes import TransferScopes +from globus_sdk.scopes import ScopeBuilder, GCSCollectionScopeBuilder, TransferScopes, AuthScopes ## tapis from tapisservice.logs import get_logger +from tapisservice.config import conf as config from tapisservice.tapisflask import utils from tapisservice.errors import AuthenticationError @@ -18,28 +19,6 @@ logger = get_logger(__name__) -def get_collection_type(endpoint_id): # TODO - ''' - Given endpoint id, return type of collection - Requires that we have a client ID and client secret in the /globus-proxy/env file - ''' - # _user = os.environ.get("") - # _pass = os.environ.get("") - client_id = '700bc50b-241c-4805-a4fe-6bd72e50062e' - client_secret = 'eHmt/LxxbQqua73tyyQX7G0zJpDXOMQ0oBP/ld+SrS0=' - auth_client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) - token_response = auth_client.oauth2_client_credentials_tokens().by_resource_server - logger.debug(f'in get_collection_type, got token resp: {token_response}') - - scopes = "urn:globus:auth:scope:transfer.api.globus.org:all" - at = token_response["transfer.api.globus.org"]["access_token"] - logger.debug(f'got at: {at}') - cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(auth_client, scopes) - tk_authorizer = globus_sdk.AccessTokenAuthorizer(at) - # create a new client - transfer_client = globus_sdk.TransferClient(authorizer=tk_authorizer) - transfer_client.get_endpoint(endpoint_id) - def activate_endpoint(tc, ep_id, username, password): ''' @@ -127,7 +106,24 @@ def check_tokens(client_id, refresh_token, access_token): return access_token, refresh_token +def format_path(path, default_dir=None): + ''' + Force absoulte paths for now, due to Globus not ahndling /~/ the same way on all systems + if a user provides a relative path, it will instead be returned as an INCORRECT abs path. + ''' + logger.info(f'building path with path {path} and default {default_dir} ') + + return f"/{path.rstrip('/').lstrip('/')}" + +def get_collection_id(client_id, client_secret, name): + client = get_transfer_client_with_secret(client_id, client_secret) + result = client.endpoint_search(filter_fulltext=name, filter_non_functional=False, limit=1) + # print(f'have result:: {result["DATA"][0]["id"]}') + return result["DATA"][0]["id"] + def get_transfer_client(client_id, refresh_token, access_token): + logger.debug(f'Attempting auth for client {client_id} using token') + print(f'Attempting auth for client {client_id} using token') client = globus_sdk.NativeAppAuthClient(client_id) # check_token(client_id, refresh_token, access_token) tomorrow = datetime.today() + timedelta(days=1) @@ -141,6 +137,35 @@ def get_transfer_client(client_id, refresh_token, access_token): transfer_client = globus_sdk.TransferClient(authorizer=authorizer) return transfer_client +def get_transfer_client_with_secret(client_id, client_secret, endpoint_id=None, addl_scopes=None): + logger.debug(f'Attempting auth for client {client_id} using secret') + try: + auth_client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) + except Exception as e: + msg = f'exception:: {e}' + print(msg) + logger.critical(msg) + raise handle_transfer_error(e) + scopes = f'urn:globus:auth:scope:transfer.api.globus.org:all' + if endpoint_id: + access_scope = f'[*https://auth.globus.org/scopes/{endpoint_id}/data_access]' + scopes = scopes + access_scope + + logger.debug(scopes) + try: + cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(auth_client, scopes) + except Exception as e: + logger.critical(f'failure instantiating cc_auth:: {e}') + raise handle_transfer_error(e) + # create a new client + try: + transfer_client = globus_sdk.TransferClient(authorizer=cc_authorizer) + except Exception as e: + logger.critical(f'failure getting transfer client:: {e}') + raise handle_transfer_error(e) + logger.debug(f'got client: {transfer_client} with endpoint: {endpoint_id} and scopes: {scopes}') + return transfer_client + def get_valid_token(client_id, refresh_token): ''' Utility function that takes a client id and refresh token @@ -155,15 +180,6 @@ def get_valid_token(client_id, refresh_token): response = client.oauth2_refresh_token(refresh_token) return response['transfer.api.globus.org']['access_token'] -def format_path(path, default_dir=None): - ''' - Force absoulte paths for now, due to Globus not ahndling /~/ the same way on all systems - if a user provides a relative path, it will instead be returned as an INCORRECT abs path. - ''' - logger.info(f'building path with path {path} and default {default_dir} ') - - return f"/{path.rstrip('/').lstrip('/')}" - def handle_transfer_error(exception, endpoint_id=None, msg=None): '''Tanslates transfer api errors into the configured basetapiserrors in ./errors.py''' logger.critical(f'\nhandling transfer API error:: {exception.code}:: with message {exception.message}\n') @@ -187,7 +203,8 @@ def handle_transfer_error(exception, endpoint_id=None, msg=None): error = GlobusUnauthorized(msg=e.http_reason) if exception.code == 'EndpointError': if exception.http_reason == 'Bad Gateway': - return + error = InternalServerError(msg="Bad Gateway", code=502) + logger.error(error) return error def is_endpoint_activated(tc, ep): @@ -213,6 +230,22 @@ def is_endpoint_connected(transfer_client, endpoint_id): # return connected return True +def is_gcp(endpoint_id): + ''' + Given endpoint id, return type of collection + Requires that we have a client ID and client secret in the config-local.json file + ''' + logger.error(f'is in gcp with eid: {endpoint_id}') + tapisconf = config.get_config_from_file() + client_id = tapisconf['client_id'] + client_secret = tapisconf['client_secret'] + + client = get_transfer_client_with_secret(client_id, client_secret) + res = client.get_endpoint(endpoint_id) + gcp = True if res["is_globus_connect"] == 'true' else False + logger.debug(f'collection_type: {res} with id {_id}') + return gcp + def ls_endpoint(tc, ep_id, path="~"): ls = tc.operation_ls(ep_id, path=path) return ls @@ -247,6 +280,7 @@ def precheck(client_id, endpoints, access_token, refresh_token): transfer_client = None try: transfer_client = get_transfer_client(client_id, refresh_token, access_token) + # transfer_client = get)get_transfer_client_with_secret(client_id, ) except Exception as e: logger.error(f'unable to get transfer client or client {client_id}: {e}') raise GlobusError(msg='Exception while generating authorization. Please check your request syntax and try again') From ecd4e920b0e0c768c388eaed977ab5e4d56a8d18 Mon Sep 17 00:00:00 2001 From: kevinpricethesecond Date: Mon, 10 Jun 2024 15:17:05 -0500 Subject: [PATCH 2/6] changes to gcp auth --- docker-compose.yml | 1 - service/api.py | 1 - service/controllers/auth.py | 2 +- service/tests.py | 18 ++++++++++++++++++ service/utils.py | 18 +++++++++++++++--- 5 files changed, 34 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1a6c760..ece3724 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,6 @@ services: - ./configschema.json:/home/tapis/configschema.json - ./config-local.json:/home/tapis/config.json - ./service.log:/home/tapis/service.log - - ../gpsettings.json:/home/tapis/gpsettings.json container_name: globus-proxy networks: - tapis diff --git a/service/api.py b/service/api.py index 524916a..15839a7 100644 --- a/service/api.py +++ b/service/api.py @@ -25,7 +25,6 @@ def __init__(self, url_map, *items): api = TapisApi(app, errors=flask_errors_dict) app.url_map.converters['regex'] = RegexConverter - # Set up error handling api.handle_error = handle_error api.handle_exception = handle_error diff --git a/service/controllers/auth.py b/service/controllers/auth.py index 768172a..1261a72 100644 --- a/service/controllers/auth.py +++ b/service/controllers/auth.py @@ -36,7 +36,7 @@ def get(self, client_id, endpoint_id): DEPENDENT_SCOPE = f"https://auth.globus.org/scopes/{endpoint_id}/data_access" SCOPE = f"urn:globus:auth:scope:transfer.api.globus.org:all[*{DEPENDENT_SCOPE}]" else: - logger.debug(f'logger is a gcp') + logger.debug(f'endpoint is a gcp') session_client = client.oauth2_start_flow(refresh_tokens=True, requested_scopes=SCOPE) diff --git a/service/tests.py b/service/tests.py index 731bf2c..08fd844 100644 --- a/service/tests.py +++ b/service/tests.py @@ -12,6 +12,7 @@ # local from utils import * +from controllers.auth import * logger = get_logger(__name__) @@ -26,6 +27,8 @@ class Base: source = None at = None rt = None + gcp_at = None + gcp_rt = None base_url = None secret = None @@ -40,6 +43,8 @@ def load_config(self): self.source_eid = "722751ce-1264-43b8-9160-a9272f746d78" # ESnet CERN DTN (Anonymous read-only testing) self.at = self.config["access_token"] self.rt = self.config["refresh_token"] + self.gcp_at = self.config["gcp_at"] + self.gcp_rt = self.config["gcp_rt"] self.base_url = self.config["base_url"] self.secret = self.config['client_secret'] self.ls6_eid = self.config['ls6_endpoint_id'] @@ -66,6 +71,10 @@ def is_gcp_test(endpoint_id): res = is_gcp(endpoint_id) print(res) +# def get_gcp_auth_url_test(client_id, client_secret, endpoint_id=None) +# print(f'about to fetch url for {endpoint_id} using client {client_id}') + + def get_transfer_client_with_secret_test(client_id, client_secret, endpoint_id=None, addl_scopes=None): print(f'about to auth {client_id} with endpoint {endpoint_id} and scopes {addl_scopes}') tc = get_transfer_client_with_secret(client_id, client_secret, endpoint_id=endpoint_id, addl_scopes=addl_scopes) @@ -182,6 +191,15 @@ def rm_xfer_test(base): print(e) fails['auth_test_2'] = e + # try: + # client_id = base.cid + # client_secret = base.secret + # endpoint_id = base.gcp_eid + # get_gcp_auth_url_test(client_id, client_secret, endpoint_id=endpoint_id) + # except Exception as e: + # print(e) + # fails['gcp_test_1'] = e + # try: # client_id = base.cid # client_secret = base.secret diff --git a/service/utils.py b/service/utils.py index d00367b..5275b53 100644 --- a/service/utils.py +++ b/service/utils.py @@ -7,6 +7,7 @@ ## globus import globus_sdk from globus_sdk.scopes import ScopeBuilder, GCSCollectionScopeBuilder, TransferScopes, AuthScopes +from globus_sdk.services.transfer.errors import TransferAPIError ## tapis from tapisservice.logs import get_logger @@ -239,11 +240,22 @@ def is_gcp(endpoint_id): tapisconf = config.get_config_from_file() client_id = tapisconf['client_id'] client_secret = tapisconf['client_secret'] - + res = {} + client = get_transfer_client_with_secret(client_id, client_secret) - res = client.get_endpoint(endpoint_id) + try: + res = client.get_endpoint(endpoint_id) + except TransferAPIError as e: + # assume it's a gcp + logger.error(f'got error checking collection type: {e}') + res['is_globus_connect'] = 'true' + except: + logger.error(f'got error checking collection type: {e}') + raise handle_transfer_error(e) + + gcp = True if res["is_globus_connect"] == 'true' else False - logger.debug(f'collection_type: {res} with id {_id}') + logger.debug(f'Is collection {endpoint_id} a gcp? : {gcp}') return gcp def ls_endpoint(tc, ep_id, path="~"): From 55216211aa45287cdc4bfb034fb5cb31b80f4030 Mon Sep 17 00:00:00 2001 From: Kevin Price <39307688+kevinpricethesecond@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:26:22 -0500 Subject: [PATCH 3/6] Update config-local.json --- config-local.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/config-local.json b/config-local.json index 2038f46..66b675e 100644 --- a/config-local.json +++ b/config-local.json @@ -5,6 +5,4 @@ "service_tenant_id": "admin", "log_level": "DEBUG", "tenants": ["dev"], - "client_id": "0ffd2a38-27e0-48c5-a870-bcb964237439", - "client_secret": "+P3dXBG0BE26dLui8HiLQEj8VH+kcbQ/7GyVJzxsOco=" } From e7bd106e774beeb41cead9defa518c305401a23e Mon Sep 17 00:00:00 2001 From: kevinpricethesecond Date: Fri, 14 Jun 2024 15:29:39 -0500 Subject: [PATCH 4/6] update changelog --- CHANGELOG.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65764b3..e7ac5b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -1.6.0 - 1/24/2024 +1.6.2 Live-docs: https://tapis-project.github.io/live-docs/?service=GlobusProxy @@ -6,12 +6,7 @@ Breaking Changes: - auth flow has been reworked to allow for v5 endpoints - users will need to refresh their auth tokens New features: - - created functional tests - unit tests were inadequate for certain functions. These must be run from inside the container - - better handling of personal connect endpoints - - initial support for additional consent auth flow - - initial support for consent management - - more descriptive response codes for several endpoints + - support for GCP collections Bug fixes: - - catch consent required errors instead of returning a 500 code - - fixed imports of functions + - none From 64cf5979b3109193bd8ded4ef5b7a3fce4021170 Mon Sep 17 00:00:00 2001 From: kevinpricethesecond Date: Fri, 14 Jun 2024 15:38:02 -0500 Subject: [PATCH 5/6] remove build from compose --- docker-compose.yml | 1 - service/controllers/auth.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ece3724..923b0ea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,6 @@ version: "3.5" services: globus-proxy: image: tapis/globus-proxy - build: . volumes: - ./configschema.json:/home/tapis/configschema.json - ./config-local.json:/home/tapis/config.json diff --git a/service/controllers/auth.py b/service/controllers/auth.py index 1261a72..697c4af 100644 --- a/service/controllers/auth.py +++ b/service/controllers/auth.py @@ -45,7 +45,7 @@ def get(self, client_id, endpoint_id): logger.debug(f"successfully got auth url for client {client_id}") return utils.ok( result = {"url": authorize_url, "session_id": session_client.verifier}, - msg = f'Please go to the URL and login. this is a test' + msg = f'Please go to the URL and login.' ) class TokensResource(Resource): From 57941ddbede524c8afc6efb71b472336117db412 Mon Sep 17 00:00:00 2001 From: kevinpricethesecond Date: Fri, 14 Jun 2024 15:39:46 -0500 Subject: [PATCH 6/6] change image version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 923b0ea..e7526d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ version: "3.5" services: globus-proxy: - image: tapis/globus-proxy + image: tapis/globus-proxy:1.6.2 volumes: - ./configschema.json:/home/tapis/configschema.json - ./config-local.json:/home/tapis/config.json