Merge pull request #36 from tapis-project/v5updates

V5updates

CHANGELOG.md

@@ -1,17 +1,12 @@
-1.6.0 - 1/24/2024
+1.6.2
 
 Live-docs: https://tapis-project.github.io/live-docs/?service=GlobusProxy
 
 Breaking Changes:
  - auth flow has been reworked to allow for v5 endpoints - users will need to refresh their auth tokens
 
 New features:
- - created functional tests - unit tests were inadequate for certain functions. These must be run from inside the container
- - better handling of personal connect endpoints
- - initial support for additional consent auth flow
- - initial support for consent management
- - more descriptive response codes for several endpoints
+ - support for GCP collections
 
 Bug fixes:
- - catch consent required errors instead of returning a 500 code
- - fixed imports of functions
+ - none

config-local.json

@@ -4,5 +4,5 @@
   "service_site_id": "tacc",
   "service_tenant_id": "admin",
   "log_level": "DEBUG",
-  "tenants": ["dev"]
+  "tenants": ["dev"],
 }

docker-compose.yml

@@ -9,7 +9,6 @@ services:
             - ./configschema.json:/home/tapis/configschema.json
             - ./config-local.json:/home/tapis/config.json
             - ./service.log:/home/tapis/service.log
-            - ../gpsettings.json:/home/tapis/gpsettings.json
         container_name: globus-proxy
         networks: 
             - tapis

service/api.py

@@ -25,7 +25,6 @@ def __init__(self, url_map, *items):
 api = TapisApi(app, errors=flask_errors_dict)
 app.url_map.converters['regex'] = RegexConverter
 
-
 # Set up error handling
 api.handle_error = handle_error
 api.handle_exception = handle_error

service/controllers/auth.py

@@ -21,15 +21,22 @@ def get(self, client_id, endpoint_id):
         # TODO: a call to get_identities needs to be authenticated
         # there might not be a way to figure out if the client_id 
         # is valid or not without letting the user go to the url and find out
+
         try:
             client = globus_sdk.NativeAppAuthClient(client_id)
         except Exception as e:
             msg = f'Encountered exception while initializing globus_sdk for client {client_id}\n\t{e}'
             logger.error(msg)
             raise InternalServerError(msg=msg)
 
-        DEPENDENT_SCOPE = f"https://auth.globus.org/scopes/{endpoint_id}/data_access"
-        SCOPE = f"urn:globus:auth:scope:transfer.api.globus.org:all[*{DEPENDENT_SCOPE}]"
+        # build scopes
+        SCOPE = None
+        if not is_gcp(endpoint_id):
+            logger.debug(f'endpoint is not a gcp')
+            DEPENDENT_SCOPE = f"https://auth.globus.org/scopes/{endpoint_id}/data_access"
+            SCOPE = f"urn:globus:auth:scope:transfer.api.globus.org:all[*{DEPENDENT_SCOPE}]"
+        else:
+            logger.debug(f'endpoint is a gcp')
 
         session_client = client.oauth2_start_flow(refresh_tokens=True, requested_scopes=SCOPE)
 
@@ -38,7 +45,7 @@ def get(self, client_id, endpoint_id):
         logger.debug(f"successfully got auth url for client {client_id}")
         return utils.ok(
                 result = {"url": authorize_url, "session_id": session_client.verifier}, 
-                msg = f'Please go to the URL and login'
+                msg = f'Please go to the URL and login.'
             )
 
 class TokensResource(Resource):

service/errors.py

@@ -27,15 +27,15 @@ def __init__(self, msg="Given path is not found on the endpoint", code=404):
 
 class GlobusError(BaseTapisError):
     """General error with the Globus SDK"""
-    def __init__(self, msg="Uncaught Globus error", code=407):
+    def __init__(self, msg="Uncaught Globus error", code=400):
         super().__init__(msg, code)
 
 class GlobusConsentRequired(BaseTapisError):
-    def __init__(self, msg="Endpoint requires consent", code=407):
+    def __init__(self, msg="Endpoint requires consent", code=403):
         super().__init__(msg, code)
 
 class GlobusUnauthorized(BaseTapisError):
-    def __init__(self, msg="Permission denied", code=407):
+    def __init__(self, msg="Permission denied", code=401):
         msg=f"You do not have permission to perform that operation on this endpoint:: {msg}"
         super().__init__(msg, code)
 

service/tests.py

@@ -12,6 +12,7 @@
 
 # local
 from utils import *
+from controllers.auth import *
 
 
 logger = get_logger(__name__)
@@ -20,10 +21,16 @@ class Base:
     config = None 
     cid = None 
     gcp_eid = None 
+    ls6_eid = None
+    frontera_eid = None 
+    corral_eid = None
     source = None 
     at = None 
     rt = None 
+    gcp_at = None
+    gcp_rt = None
     base_url = None
+    secret = None
 
     def __init__(self):
         self.config = self.load_config()
@@ -36,14 +43,49 @@ def load_config(self):
             self.source_eid = "722751ce-1264-43b8-9160-a9272f746d78" # ESnet CERN DTN (Anonymous read-only testing)
             self.at = self.config["access_token"]
             self.rt = self.config["refresh_token"]
+            self.gcp_at = self.config["gcp_at"]
+            self.gcp_rt = self.config["gcp_rt"]
             self.base_url = self.config["base_url"]
+            self.secret = self.config['client_secret']
+            self.ls6_eid = self.config['ls6_endpoint_id']
             return self.config
 
+# TODO: write setup / teardown functions that can be run by the tests indivudially so the success of one test isn't dependant on another
+## setup / teardown
+def make_dir(path):
+    os.mkdir(path)
 
+def rm_dir(path):
+    os.rmdir(path)
+
+## utils
 def get_endpoint_test(epid):
     info = get_collection_type(epid)
     print(f'got ep info:: {info}')
-
+
+def get_collection_id_test(client_id, client_secret, name):
+    info = get_collection_id(client_id, client_secret, name)
+    print(f'got ep id:: {info}')
+
+def is_gcp_test(endpoint_id):
+    res = is_gcp(endpoint_id)
+    print(res)
+
+# def get_gcp_auth_url_test(client_id, client_secret, endpoint_id=None)
+#     print(f'about to fetch url for {endpoint_id} using client {client_id}')
+
+
+def get_transfer_client_with_secret_test(client_id, client_secret, endpoint_id=None, addl_scopes=None):
+    print(f'about to auth {client_id} with endpoint {endpoint_id} and scopes {addl_scopes}')
+    tc = get_transfer_client_with_secret(client_id, client_secret, endpoint_id=endpoint_id, addl_scopes=addl_scopes)
+    print(f'authd:: {tc}')
+
+def build_scopes_test(scopes:list, eid=None):
+    print(f'about to test building a scope')
+    s_list = ['']
+    scopes = build_scopes(s_list, collection_id=eid)
+
+## ops    
 def ls_test(base, path):
     url = f'{base.base_url}/ops/{base.cid}/{base.gcp_eid}/{path}'
     query = {"access_token": base.at,
@@ -75,6 +117,7 @@ def mkdir_test(base, path):
     if response.status_code != 200:
         raise Exception(f'{response.status_code}:: {response.text}')
 
+## transfers
 def make_xfer_test(base):
     pass
 
@@ -97,40 +140,75 @@ def rm_xfer_test(base):
         #     print(e)
         #     fails['base_test_1'] = e
         # ls tests
+        # try:
+        #     ls_test(base, base_path)
+        # except Exception as e:
+        #     print(e)
+        #     fails['ls_test_1'] = e
+        # try:
+        #     ls_test(base, f'{base_path}/test')
+        # except Exception as e:
+        #     print(e)
+        #     fails['ls_test_2'] = e
+        # try:
+        #     ls_test(base, f'{base_path}/test.py')  
+        # except Exception as e:
+        #     print(e)
+        #     fails['ls_test_3'] = e
+        # # mkdir tests
+        # try:
+        #     mkdir_test(base, f'{base_path}/mkdirtest')
+        # except Exception as e:
+        #     print(e)
+        #     fails['mkdir_test_1'] = e
+        # # rename tests
+        # try:
+        #     mv_test(base, f'{base_path}/mkdirtest', f'{base_path}/mkdirtest2')
+        # except Exception as e:
+        #     print(e)
+        #     fails['mv_test_1'] = e
+        # # rm tests
+        # try:
+        #     rm_test(base, f'{base_path}/mkdirtest2')
+        # except Exception as e:
+        #     print(e)
+        #     fails['rm_test_1'] = e
+
         try:
-            ls_test(base, base_path)
-        except Exception as e:
-            print(e)
-            fails['ls_test_1'] = e
-        try:
-            ls_test(base, f'{base_path}/test')
-        except Exception as e:
-            print(e)
-            fails['ls_test_2'] = e
-        try:
-            ls_test(base, f'{base_path}/test.py')  
-        except Exception as e:
-            print(e)
-            fails['ls_test_3'] = e
-        # mkdir tests
-        try:
-            mkdir_test(base, f'{base_path}/mkdirtest')
-        except Exception as e:
-            print(e)
-            fails['mkdir_test_1'] = e
-        # rename tests
-        try:
-            mv_test(base, f'{base_path}/mkdirtest', f'{base_path}/mkdirtest2')
+            client_id = base.cid
+            client_secret = base.secret
+            get_transfer_client_with_secret_test(client_id, client_secret)
         except Exception as e:
             print(e)
-            fails['mv_test_1'] = e
-        # rm tests
+            fails['auth_test_1'] = e 
+
         try:
-            rm_test(base, f'{base_path}/mkdirtest2')
+            client_id = base.cid
+            client_secret = base.secret
+            endpoint_id = base.ls6_eid
+            get_transfer_client_with_secret_test(client_id, client_secret, endpoint_id=endpoint_id)
         except Exception as e:
             print(e)
-            fails['rm_test_1'] = e
-
+            fails['auth_test_2'] = e 
+
+        # try:
+        #     client_id = base.cid
+        #     client_secret = base.secret
+        #     endpoint_id = base.gcp_eid
+        #     get_gcp_auth_url_test(client_id, client_secret, endpoint_id=endpoint_id)
+        # except Exception as e:
+        #     print(e)
+        #     fails['gcp_test_1'] = e 
+
+        # try:
+        #     client_id = base.cid
+        #     client_secret = base.secret
+        #     endpoint_id = base.ls6_eid
+        #     addl_scopes = 'manage_projects'
+        #     get_transfer_client_with_secret_test(client_id, client_secret, endpoint_id=endpoint_id, addl_scopes=addl_scopes)
+        # except Exception as e:
+        #     print(e)
+        #     fails['auth_test_3'] = e         
 
     except Exception as e:
         print(f'Unknown error running tests:: {e}')