Automated copy feature/OIDC oauth groups rev2

.gitignore

@@ -108,3 +108,6 @@ tests/resources/keys/*.pem
 .DS_Store
 .vscode
 .idea
+
+# snyk
+.dccache

fence/__init__.py

@@ -470,6 +470,7 @@ def _setup_oidc_clients(app):
                 logger=logger,
                 HTTP_PROXY=config.get("HTTP_PROXY"),
                 idp=settings.get("name") or idp.title(),
+                arborist=app.arborist,
             )
             clean_idp = idp.lower().replace(" ", "")
             setattr(app, f"{clean_idp}_client", client)

fence/blueprints/login/base.py

@@ -1,14 +1,19 @@
+import time
+import base64
+import json
+from urllib.parse import urlparse, urlencode, parse_qsl
+import jwt
+from flask import current_app
 import flask
 from cdislogging import get_logger
 from flask_restful import Resource
-from urllib.parse import urlparse, urlencode, parse_qsl
-
 from fence.auth import login_user
 from fence.blueprints.login.redirect import validate_redirect
 from fence.config import config
 from fence.errors import UserError
 from fence.metrics import metrics
 
+
 logger = get_logger(__name__)
 
 
@@ -20,7 +25,7 @@ def __init__(self, idp_name, client):
         Args:
             idp_name (str): name for the identity provider
             client (fence.resources.openid.idp_oauth2.Oauth2ClientBase):
-                Some instaniation of this base client class or a child class
+                Some instantiation of this base client class or a child class
         """
         self.idp_name = idp_name
         self.client = client
@@ -67,6 +72,9 @@ def __init__(
         username_field="email",
         email_field="email",
         id_from_idp_field="sub",
+        firstname_claim_field="given_name",
+        lastname_claim_field="family_name",
+        organization_claim_field="org",
         app=flask.current_app,
     ):
         """
@@ -92,8 +100,25 @@ def __init__(
         self.is_mfa_enabled = "multifactor_auth_claim_info" in config[
             "OPENID_CONNECT"
         ].get(self.idp_name, {})
+
+        # Config option to explicitly persist refresh tokens
+        self.persist_refresh_token = False
+
+        self.read_authz_groups_from_tokens = False
+
         self.app = app
 
+        self.persist_refresh_token = (
+            config["OPENID_CONNECT"].get(self.idp_name, {}).get("persist_refresh_token")
+        )
+
+        if "is_authz_groups_sync_enabled" in config["OPENID_CONNECT"].get(
+            self.idp_name, {}
+        ):
+            self.read_authz_groups_from_tokens = config["OPENID_CONNECT"][
+                self.idp_name
+            ]["is_authz_groups_sync_enabled"]
+
     def get(self):
         # Check if user granted access
         if flask.request.args.get("error"):
@@ -119,7 +144,11 @@ def get(self):
 
         code = flask.request.args.get("code")
         result = self.client.get_auth_info(code)
+
+        refresh_token = result.get("refresh_token")
+
         username = result.get(self.username_field)
+
         if not username:
             raise UserError(
                 f"OAuth2 callback error: no '{self.username_field}' in {result}"
@@ -128,12 +157,114 @@ def get(self):
         email = result.get(self.email_field)
         id_from_idp = result.get(self.id_from_idp_field)
 
-        resp = _login(username, self.idp_name, email=email, id_from_idp=id_from_idp)
-        self.post_login(user=flask.g.user, token_result=result, id_from_idp=id_from_idp)
+        resp = _login(
+            username,
+            self.idp_name,
+            email=email,
+            id_from_idp=id_from_idp,
+            token_result=result,
+        )
+
+        if not flask.g.user:
+            raise UserError("Authentication failed: flask.g.user is missing.")
+
+        expires = self.extract_exp(refresh_token)
+
+        # if the access token is not a JWT, or does not carry exp,
+        # default to now + REFRESH_TOKEN_EXPIRES_IN
+        if expires is None:
+            expires = int(time.time()) + config["REFRESH_TOKEN_EXPIRES_IN"]
+            logger.info(self, f"Refresh token not in JWT, using default: {expires}")
+
+        # Store refresh token in db
+        should_persist_token = (
+            self.persist_refresh_token or self.read_authz_groups_from_tokens
+        )
+        if should_persist_token:
+            # Ensure flask.g.user exists to avoid a potential AttributeError
+            if getattr(flask.g, "user", None):
+                self.client.store_refresh_token(flask.g.user, refresh_token, expires)
+            else:
+                logger.error(
+                    "User information is missing from flask.g; cannot store refresh token."
+                )
+
+        self.post_login(
+            user=flask.g.user,
+            token_result=result,
+            id_from_idp=id_from_idp,
+        )
+
         return resp
 
+    def extract_exp(self, refresh_token):
+        """
+        Extract the expiration time (`exp`) from a refresh token.
+
+        This function attempts to retrieve the expiration time from the provided
+        refresh token using three methods:
+
+        1. Using PyJWT to decode the token (without signature verification).
+        2. Introspecting the token (if supported by the identity provider).
+        3. Manually base64 decoding the token's payload (if it's a JWT).
+
+        **Disclaimer:** This function assumes that the refresh token is valid and
+        does not perform any JWT validation. For JWTs from an OpenID Connect (OIDC)
+        provider, validation should be done using the public keys provided by the
+        identity provider (from the JWKS endpoint) before using this function to
+        extract the expiration time. Without validation, the token's integrity and
+        authenticity cannot be guaranteed, which may expose your system to security
+        risks. Ensure validation is handled prior to calling this function,
+        especially in any public or production-facing contexts.
+
+        Args:
+            refresh_token (str): The JWT refresh token from which to extract the expiration.
+
+        Returns:
+            int or None: The expiration time (`exp`) in seconds since the epoch,
+            or None if extraction fails.
+        """
+
+        # Method 1: PyJWT
+        try:
+            # Skipping keys since we're not verifying the signature
+            decoded_refresh_token = jwt.decode(
+                refresh_token,
+                options={
+                    "verify_aud": False,
+                    "verify_at_hash": False,
+                    "verify_signature": False,
+                },
+                algorithms=["RS256", "HS512"],
+            )
+            exp = decoded_refresh_token.get("exp")
+
+            if exp is not None:
+                return exp
+        except Exception as e:
+            logger.info(f"Refresh token expiry: Method (PyJWT) failed: {e}")
+
+        # Method 2: Manual base64 decoding
+        try:
+            # Assuming the token is a JWT (header.payload.signature)
+            payload_encoded = refresh_token.split(".")[1]
+            # Add necessary padding for base64 decoding
+            payload_encoded += "=" * (4 - len(payload_encoded) % 4)
+            payload_decoded = base64.urlsafe_b64decode(payload_encoded)
+            payload_json = json.loads(payload_decoded)
+            exp = payload_json.get("exp")
+
+            if exp is not None:
+                return exp
+        except Exception as e:
+            logger.info(f"Method 3 (Manual decoding) failed: {e}")
+
+        # If all methods fail, return None
+        return None
+
     def post_login(self, user=None, token_result=None, **kwargs):
         prepare_login_log(self.idp_name)
+
         metrics.add_login_event(
             user_sub=flask.g.user.id,
             idp=self.idp_name,
@@ -142,6 +273,11 @@ def post_login(self, user=None, token_result=None, **kwargs):
             client_id=flask.session.get("client_id"),
         )
 
+        if self.read_authz_groups_from_tokens:
+            self.client.update_user_authorization(
+                user=user, pkey_cache=None, db_session=None, idp_name=self.idp_name
+            )
+
         if token_result:
             username = token_result.get(self.username_field)
             if self.is_mfa_enabled:
@@ -171,19 +307,76 @@ def prepare_login_log(idp_name):
     }
 
 
-def _login(username, idp_name, email=None, id_from_idp=None):
+def _login(
+    username,
+    idp_name,
+    email=None,
+    id_from_idp=None,
+    token_result=None,
+):
     """
-    Login user with given username, then redirect if session has a saved
-    redirect.
+    Login user with given username, then automatically register if needed,
+    and finally redirect if session has a saved redirect.
     """
     login_user(username, idp_name, email=email, id_from_idp=id_from_idp)
 
+    register_idp_users = (
+        config["OPENID_CONNECT"]
+        .get(idp_name, {})
+        .get("enable_idp_users_registration", False)
+    )
+
     if config["REGISTER_USERS_ON"]:
-        if not flask.g.user.additional_info.get("registration_info"):
-            return flask.redirect(
-                config["BASE_URL"] + flask.url_for("register.register_user")
-            )
+        user = flask.g.user
+        if not user.additional_info.get("registration_info"):
+            # If enabled, automatically register user from Idp
+            if register_idp_users:
+                firstname = token_result.get("firstname")
+                lastname = token_result.get("lastname")
+                organization = token_result.get("org")
+                email = token_result.get("email")
+                if email is None:
+                    raise UserError("OAuth2 id token is missing email claim")
+                # Log warnings and set defaults if needed
+                if not firstname or not lastname:
+                    logger.warning(
+                        f"User {username} missing name fields. Proceeding with minimal info."
+                    )
+                    firstname = firstname or "Unknown"
+                    lastname = lastname or "User"
+
+                if not organization:
+                    organization = None
+                    logger.info(
+                        f"User {username} missing organization. Defaulting to None."
+                    )
+
+                # Store registration info
+                registration_info = {
+                    "firstname": firstname,
+                    "lastname": lastname,
+                    "org": organization,
+                    "email": email,
+                }
+                user.additional_info = user.additional_info or {}
+                user.additional_info["registration_info"] = registration_info
+
+                # Persist to database
+                current_app.scoped_session().add(user)
+                current_app.scoped_session().commit()
+
+                # Ensure user exists in Arborist and assign to group
+                with current_app.arborist.context():
+                    current_app.arborist.create_user(dict(name=username))
+                    current_app.arborist.add_user_to_group(
+                        username=username,
+                        group_name=config["REGISTERED_USERS_GROUP"],
+                    )
+            else:
+                return flask.redirect(
+                    config["BASE_URL"] + flask.url_for("register.register_user")
+                )
 
     if flask.session.get("redirect"):
         return flask.redirect(flask.session.get("redirect"))
-    return flask.jsonify({"username": username})
+    return flask.jsonify({"username": username, "registered": True})

fence/config-default.yaml

@@ -45,7 +45,7 @@ ENCRYPTION_KEY: ''
 # //////////////////////////////////////////////////////////////////////////////////////
 # flask's debug setting
 # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only)
-DEBUG: true
+DEBUG: false
 # if true, will automatically login a user with username "test"
 # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only)
 MOCK_AUTH: false
@@ -94,6 +94,7 @@ DB_MIGRATION_POSTGRES_LOCK_KEY: 100
 #   - WARNING: Be careful changing the *_ALLOWED_SCOPES as you can break basic
 #              and optional functionality
 # //////////////////////////////////////////////////////////////////////////////////////
+
 OPENID_CONNECT:
   # any OIDC IDP that does not differ from the generic implementation can be
   # configured without code changes
@@ -115,6 +116,40 @@ OPENID_CONNECT:
     multifactor_auth_claim_info: # optional, include if you're using arborist to enforce mfa on a per-file level
       claim: '' # claims field that indicates mfa, either the acr or acm claim.
       values: [ "" ] # possible values that indicate mfa was used. At least one value configured here is required to be in the token
+    # When true, it allows refresh tokens to be stored even if is_authz_groups_sync_enabled is set false.
+    # When false, the system will only store refresh tokens if is_authz_groups_sync_enabled is enabled
+    persist_refresh_token: false
+    # is_authz_groups_sync_enabled: A configuration flag that determines whether the application should
+    # verify and synchronize user group memberships between the identity provider (IdP)
+    # and the local authorization system (Arborist). When enabled, the refresh token is stored, the system retrieves
+    # the user's group information from their token issued by the IdP and compares it against
+    # the groups defined in the local system. Based on the comparison, the user is added to
+    # or removed from relevant groups in the local system to ensure their group memberships
+    # remain up-to-date. If this flag is disabled, no group synchronization occurs
+    is_authz_groups_sync_enabled: true
+    # Key used to retrieve group information from the token
+    group_claim_field: "groups"
+    # IdP group membership expiration (seconds).
+    group_membership_expiration_duration: 604800
+    authz_groups_sync:
+      # This defines the prefix used to identify authorization groups.
+      group_prefix: "some_prefix"
+    # This flag indicates whether the audience (aud) claim in the JWT should be verified during token validation.
+    verify_aud: true
+    # This specifies the expected audience (aud) value for the JWT, ensuring that the token is intended for use with the 'fence' service.
+    audience: fence
+    # default refresh token expiration duration
+    default_refresh_token_exp: 3600
+    # firstname claim field from idp
+    firstname_claim_field: 'firstName'
+    # lastname claim field from idp
+    lastname_claim_field: 'lastName'
+    # organization claim field from idp
+    organization_claim_field: 'org'
+    # default organization
+    default_organization: 'AU BioCommons'
+    # automatically register users from Idp
+    enable_idp_users_registration: true
   # These Google values must be obtained from Google's Cloud Console
   # Follow: https://developers.google.com/identity/protocols/OpenIDConnect
   #

fence/config.py

@@ -140,12 +140,23 @@ def post_process(self):
             )
 
         for idp_id, idp in self._configs.get("OPENID_CONNECT", {}).items():
+            if not isinstance(idp, dict):
+                raise TypeError(
+                    "Expected 'OPENID_CONNECT' configuration to be a dictionary."
+                )
             mfa_info = idp.get("multifactor_auth_claim_info")
             if mfa_info and mfa_info["claim"] not in ["amr", "acr"]:
                 logger.warning(
                     f"IdP '{idp_id}' is using multifactor_auth_claim_info '{mfa_info['claim']}', which is neither AMR or ACR. Unable to determine if a user used MFA. Fence will continue and assume they have not used MFA."
                 )
 
+            groups_sync_enabled = idp.get("is_authz_groups_sync_enabled", False)
+            # when is_authz_groups_sync_enabled, then you must provide authz_groups_sync, with group prefix
+            if groups_sync_enabled and not idp.get("authz_groups_sync"):
+                error = f"Error: is_authz_groups_sync_enabled is enabled, required values not configured, for idp: {idp_id}"
+                logger.error(error)
+                raise Exception(error)
+
         self._validate_parent_child_studies(self._configs["dbGaP"])
 
     @staticmethod