From 69f51a8f6c5a67e723baa2b84f6d201c2549e3e3 Mon Sep 17 00:00:00 2001 From: James Kunstle Date: Mon, 28 Aug 2023 19:28:59 -0400 Subject: [PATCH] add oauth w/ flask routes, serverside user groups adds /login, /logout, /authorize routes to backend api. moves querying and processing of user's groups purely serverside. removes group querying from background_callback and page-reload group refreshes. uses Flask-Login as login handler, session token cookie added. Signed-off-by: James Kunstle --- app.py | 296 +++++++++++++++++++++++++----- docker-compose.yml | 18 ++ pages/index/index_callbacks.py | 323 ++++++++++++++------------------- pages/index/index_layout.py | 41 ++++- queries/user_groups_query.py | 156 ++++++++++++++++ requirements.txt | 1 + 6 files changed, 588 insertions(+), 247 deletions(-) create mode 100644 queries/user_groups_query.py diff --git a/app.py b/app.py index 3a041305..e004fb43 100644 --- a/app.py +++ b/app.py @@ -11,20 +11,37 @@ Having laid out the HTML-like organization of this page, we write the callbacks for this page in the neighbor 'app_callbacks.py' file. """ -from db_manager.augur_manager import AugurManager -import dash -import dash_bootstrap_components as dbc -from dash_bootstrap_templates import load_figure_template -from sqlalchemy.exc import SQLAlchemyError +import os import sys import logging +import secrets +import uuid +from urllib.parse import urlencode +import requests +import json +import dash +from redis import StrictRedis +from sqlalchemy.exc import SQLAlchemyError import plotly.io as plt_io from celery import Celery from dash import CeleryManager, Input, Output +import dash_bootstrap_components as dbc +from flask import url_for, redirect, abort, session, request, flash, current_app +from flask_login import ( + current_user, + LoginManager, + logout_user, + login_user, + UserMixin, + login_required, +) +from dash_bootstrap_templates import load_figure_template +from db_manager.augur_manager import AugurManager import worker_settings -import os -logging.basicConfig(format="%(asctime)s %(levelname)-8s %(message)s", level=logging.INFO) +logging.basicConfig( + format="%(asctime)s %(levelname)-8s %(message)s", level=logging.INFO +) """CREATE CELERY TASK QUEUE AND MANAGER""" celery_app = Celery( @@ -33,7 +50,9 @@ backend=worker_settings.REDIS_URL, ) -celery_app.conf.update(task_time_limit=84600, task_acks_late=True, task_track_started=True) +celery_app.conf.update( + task_time_limit=84600, task_acks_late=True, task_track_started=True +) celery_manager = CeleryManager(celery_app=celery_app) @@ -89,61 +108,240 @@ background_callback_manager=celery_manager, ) -# expose the application object's server variable so that the wsgi server can use it. +"""CONFIGURE FLASK-LOGIN STUFF""" server = app.server +server.config["SECRET_KEY"] = os.environ.get("SECRET_KEY") +server.config["OAUTH2_PROVIDERS"] = { + os.environ.get("OAUTH_CLIENT_NAME"): { + "client_id": os.environ.get("OAUTH_CLIENT_ID"), + "client_secret": os.environ.get("OAUTH_CLIENT_SECRET"), + "authorize_url": os.environ.get("OAUTH_AUTHORIZE_URL"), + "token_url": os.environ.get("OAUTH_TOKEN_URL"), + "redirect_uri": os.environ.get("OAUTH_REDIRECT_URI"), + } +} + +# CREATE FLASK-LOGIN OBJECT +login = LoginManager(server) +login.login_view = "index" + +"""DASH PAGES LAYOUT""" # layout of the app stored in the app_layout file, must be imported after the app is initiated from pages.index.index_layout import layout app.layout = layout +"""CLIENTSIDE CALLBACK FOR LOGOUT + REFRESH""" # I know what you're thinking- "This callback shouldn't be here!" # well, circular imports are a hassle, and the 'app' object from this # file can't be imported into index_callbacks.py file where it should be. # This callback handles logging a user out of their preferences. -app.clientside_callback( - """ - function(logout, refresh) { - - // gets the string representing the component_id and component_prop that triggered the callback. - const triggered = window.dash_clientside.callback_context.triggered.map(t => t.prop_id)[0] - console.log(triggered) - - if(triggered == "logout-button.n_clicks"){ - // clear user's localStorage, - // pattern-match key's suffix. - const keys = Object.keys(localStorage) - for (let key of keys) { - if (String(key).includes('_dash_persistence')) { - localStorage.removeItem(key) - } - } - - // clear user's sessionStorage, - // pattern-match key's suffix. - const sesh = Object.keys(sessionStorage) - for (let key of sesh) { - if (String(key).includes('_dash_persistence')) { - sessionStorage.removeItem(key) - } - } - } - else{ - // trigger user preferences redownload - sessionStorage["is-client-startup"] = true +# app.clientside_callback( +# """ +# function(logout, refresh) { + +# // gets the string representing the component_id and component_prop that triggered the callback. +# const triggered = window.dash_clientside.callback_context.triggered.map(t => t.prop_id)[0] +# console.log(triggered) + +# if(triggered == "logout-button.n_clicks"){ +# // clear user's localStorage, +# // pattern-match key's suffix. +# const keys = Object.keys(localStorage) +# for (let key of keys) { +# if (String(key).includes('_dash_persistence')) { +# localStorage.removeItem(key) +# } +# } + +# // clear user's sessionStorage, +# // pattern-match key's suffix. +# const sesh = Object.keys(sessionStorage) +# for (let key of sesh) { +# if (String(key).includes('_dash_persistence')) { +# sessionStorage.removeItem(key) +# } +# } +# } +# else{ +# // trigger user preferences redownload +# sessionStorage["is-client-startup"] = true +# } + +# // reload the page, +# // redirect to index. +# window.location.reload() +# return "/" +# } +# """, +# Output("url", "pathname"), +# Input("logout-button", "n_clicks"), +# prevent_initial_call=True, +# ) + + +"""FLASK-LOGIN ROUTES + UTILITIES""" + + +class User(UserMixin): + def __init__(self, id): + self.id = id + + +@login.user_loader +def load_user(id): + users_cache = StrictRedis( + host="redis-users", + port=6379, + password=os.getenv("REDIS_PASSWORD", ""), + ) + + # return the JSON of a user that was set in the Redis instance + if users_cache.exists(id): + usn = json.loads(users_cache.get(id))["username"] + return User(id) + return None + + +@server.route("/logout/") +def logout(): + users_cache = StrictRedis( + host="redis-users", + port=6379, + password=os.getenv("REDIS_PASSWORD", ""), + ) + + if current_user.is_authenticated: + c_id = current_user.get_id() + users_cache.delete(c_id) + logout_user() + logging.warning(f"USER {c_id} LOGGED OUT") + else: + logging.warning("TRIED TO LOG OUT") + return redirect("/") + + +@server.route("/login/") +def oauth2_authorize(): + users_cache = StrictRedis( + host="redis-users", + port=6379, + password=os.getenv("REDIS_PASSWORD", ""), + ) + + provider = os.environ.get("OAUTH_CLIENT_NAME") + + if not current_user.is_anonymous: + return redirect(url_for("index")) + + provider_data = current_app.config["OAUTH2_PROVIDERS"].get(provider) + if provider_data is None: + abort(404) + + # generate a random string for the state parameter + session["oauth2_state"] = secrets.token_urlsafe(16) + + # create a query string with all the OAuth2 parameters + qs = urlencode( + { + "client_id": provider_data["client_id"], + # "redirect_uri": url_for("oauth2_callback", _external=True), + "response_type": "code", + # "state": session["oauth2_state"], } + ) + + # redirect the user to the OAuth2 provider authorization URL + return redirect(provider_data["authorize_url"] + "?" + qs) + + +@server.route("/authorize/") +def oauth2_callback(): + users_cache = StrictRedis( + host="redis-users", + port=6379, + password=os.getenv("REDIS_PASSWORD", ""), + ) + + provider = os.environ.get("OAUTH_CLIENT_NAME") - // reload the page, - // redirect to index. - window.location.reload() - return "/" + if not current_user.is_anonymous: + return redirect(url_for("index")) + + provider_data = current_app.config["OAUTH2_PROVIDERS"].get(provider) + if provider_data is None: + abort(404) + + # if there was an authentication error, flash the error messages and exit + if "error" in request.args: + for k, v in request.args.items(): + if k.startswith("error"): + flash(f"{k}: {v}") + return redirect(url_for("index")) + + # make sure that the state parameter matches the one we created in the + # authorization request + # if request.args["state"] != session.get("oauth2_state"): + # abort(401) + + # make sure that the authorization code is present + if "code" not in request.args: + abort(401) + + # exchange the authorization code for an access token + response = requests.post( + provider_data["token_url"], + data={ + "client_id": provider_data["client_id"], + "client_secret": provider_data["client_secret"], + "code": request.args["code"], + "grant_type": "code", + "redirect_uri": url_for("oauth2_callback", _external=True), + }, + headers={ + "Accept": "application/json", + "Authorization": f"Client {provider_data['client_secret']}", + }, + ) + logging.warning("Received response from authorize endpoint") + + # check whether login worked + if response.status_code != 200: + abort(401) + + # if login worked, get the token + resp = response.json() + oauth2_token = resp.get("access_token") + if not oauth2_token: + abort(401) + logging.debug("Got token from authorize endpoint") + + # get remaining credentials + username = resp.get("username") + oauth2_refresh = resp.get("refresh_token") + oauth2_token_expires = resp.get("expires") + + # random ID used to identify user. + id_number = str(uuid.uuid1()) + + logging.warning("Creating new user") + serverside_user_data = { + "username": username, + "access_token": oauth2_token, + "refresh_token": oauth2_refresh, + "expiration": oauth2_token_expires, } - """, - Output("url", "pathname"), - Input("logout-button", "n_clicks"), - Input("refresh-button", "n_clicks"), - prevent_initial_call=True, -) + users_cache.set(id_number, json.dumps(serverside_user_data)) + + login_user(User(id_number)) + logging.warning("User logged in") + + # forward-slash redirect because dash's index route has another name + return redirect("/") + + +"""DASH STARTUP PARAMETERS""" if os.getenv("8KNOT_DEBUG", "False") == "True": app.enable_dev_tools(dev_tools_ui=True, dev_tools_hot_reload=False) diff --git a/docker-compose.yml b/docker-compose.yml index 057aef5a..acaa02b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,7 @@ services: - worker-callback - worker-query - redis-cache + - redis-users env_file: - ./env.list restart: always @@ -33,6 +34,7 @@ services: [ "celery", "-A", "app:celery_app", "worker", "--loglevel=INFO" ] depends_on: - redis-cache + - redis-users env_file: - ./env.list restart: always @@ -49,6 +51,7 @@ services: - ./env.list restart: always + # for data blob caching redis-cache: image: docker.io/library/redis:6 command: @@ -63,6 +66,21 @@ services: - ./env.list restart: always + # for user session storage + redis-users: + image: docker.io/library/redis:6 + command: + - /bin/sh + - -c + # - Double dollars, so that the variable is not expanded by Docker Compose + # - Surround by quotes, so that the shell does not split the password + # - The ${variable:?message} syntax causes shell to exit with a non-zero + # code and print a message, when the variable is not set or empty + - redis-server --requirepass "$${REDIS_PASSWORD:?REDIS_PASSWORD variable is not set}" + env_file: + - ./env.list + restart: always + # flower: # build: # context: . diff --git a/pages/index/index_callbacks.py b/pages/index/index_callbacks.py index e3cbd08e..c337dffd 100644 --- a/pages/index/index_callbacks.py +++ b/pages/index/index_callbacks.py @@ -3,25 +3,24 @@ import os import time import logging +import json from celery.result import AsyncResult import dash_bootstrap_components as dbc import dash -from dash import html, callback +from dash import callback from dash.dependencies import Input, Output, State from app import augur +from flask_login import current_user from cache_manager.cache_manager import CacheManager as cm from queries.issues_query import issues_query as iq from queries.commits_query import commits_query as cq from queries.contributors_query import contributors_query as cnq from queries.prs_query import prs_query as prq from queries.company_query import company_query as cmq - -# DONE: imported other functions -from pages.index.login_help import ( - verify_previous_login_credentials, - get_user_groups, - get_admin_groups, -) +from queries.user_groups_query import user_groups_query as ugq +import redis +from redis import StrictRedis +import flask # list of queries to be run @@ -31,6 +30,48 @@ login_enabled = os.getenv("AUGUR_LOGIN_ENABLED", "False") == "True" +@callback( + [Output("user-group-loading-signal", "data")], + [Input("url", "href"), Input("refresh-button", "n_clicks")], +) +def kick_off_group_collection(url, n_clicks): + """Schedules a Celery task to collect user groups. + Sends a message via localStorage that will kick off a background callback + which waits for the Celery task to finish. + + if refresh-groups clicked, forces group reload. + + Args: + url (str): browser page URL + n_clicks (_type_): number 'refresh_groups' button has been clicked. + + Returns: + int: ID of Celery task that has started for group collection. + """ + if current_user.is_authenticated: + user_id = current_user.get_id() + users_cache = StrictRedis( + host="redis-users", + port=6379, + password=os.getenv("REDIS_PASSWORD", ""), + ) + + # TODO: check how old groups are. If they're pretty old (threshold tbd) then requery + + # check if groups are not already cached, or if the refresh-button was pressed + if not users_cache.exists(f"{user_id}_groups") or ( + dash.ctx.triggered_id == "refresh-button" + ): + # kick off celery task to collect groups + # on query worker queue, + return [ugq.apply_async(args=[user_id], queue="data").id] + else: + return dash.no_update + else: + # user anonymous + return dash.no_update + + @callback( [ Output("nav-login-container", "children"), @@ -39,10 +80,9 @@ Output("logout-button", "disabled"), Output("manage-group-button", "disabled"), ], - Input("augur_username_dash_persistence", "data"), - State("login-succeeded", "data"), + Input("url", "href"), ) -def login_username_button(username, login_succeeded): +def login_username_button(url): """Sets logged-in-status component in top left of page. If a non-null username is known then we're logged in so we provide @@ -59,29 +99,46 @@ def login_username_button(username, login_succeeded): _type_: _description_ """ - buttons_disabled = True + navlink = [ + dbc.NavLink( + "Augur log in/sign up", + href="/login/", + id="login-navlink", + active=True, + # communicating with the underlying Flask server + external_link=True, + ), + ] - if username: - navlink = [ - dbc.NavItem( - dbc.NavLink( - f"{username}", - href=augur.user_account_endpoint, - id="login-navlink", - disabled=True, + buttons_disabled = True + login_succeeded = True + + if current_user: + if current_user.is_authenticated: + logging.warning(f"LOGINBUTTON: USER LOGGED IN {current_user}") + # TODO: implement more permanent interface + users_cache = StrictRedis( + host="redis-users", + port=6379, + password=os.getenv("REDIS_PASSWORD", ""), + ) + + # TODO: Ping Redis, catch error and return no login + + user_id = current_user.get_id() + user_info = json.loads(users_cache.get(user_id)) + + navlink = [ + dbc.NavItem( + dbc.NavLink( + f"{user_info['username']}", + href=augur.user_account_endpoint, + id="login-navlink", + disabled=True, + ), ), - ), - ] - buttons_disabled = False - else: - navlink = [ - dbc.NavLink( - "Augur log in/sign up", - href=augur.user_auth_endpoint, - id="login-navlink", - active=True, - ), - ] + ] + buttons_disabled = False return ( navlink, @@ -92,167 +149,15 @@ def login_username_button(username, login_succeeded): ) -@callback( - [ - Output("augur_username_dash_persistence", "data"), - Output("augur_user_bearer_token_dash_persistence", "data"), - Output("augur_token_expiration_dash_persistence", "data"), - Output("augur_refresh_token_dash_persistence", "data"), - Output("augur_user_groups_dash_persistence", "data"), - Output("augur_user_group_options_dash_persistence", "data"), - Output("is-client-startup", "data"), - Output("url", "search"), - Output("login-succeeded", "data"), - ], - [ - Input("url", "href"), - State("url", "search"), - State("is-client-startup", "data"), - State("augur_username_dash_persistence", "data"), - State("augur_user_bearer_token_dash_persistence", "data"), - State("augur_token_expiration_dash_persistence", "data"), - State("augur_refresh_token_dash_persistence", "data"), - ], -) -def get_augur_user_preferences( - this_url, - search_val, - is_client_startup, - username, - bearer_token, - expiration, - refresh_token, -): - """Handles logging in when the user navigates to application. - - If the user is navigating to application with a fresh tab, the app - tries to log in with credentials (bearer token) if they're present and valid. - - If credentials are valid and user is logged in, user's groups are retrieved from - Augur front-end and stored in their session. - - This function will be invoked any time a page is switched in the app, including when - the application is accessed via redirect from Augur or on refresh. - - Args: - this_url (str): current full href - search_val (str): query strings to HREF - refresh_groups (bool): whether we should refresh user's preferences - username (str): stored username - bearer_token (str): stored bearer token - expiration (str): bearer token expiration date - refresh (str): refresh token for bearer token - - Raises: - dash.exceptions.PreventUpdate: if we're just switching between pages, don't update anything - - Returns: - augur_username_dash_persistence (str): username - augur_user_bearer_token_dash_persistence (str): bearer token - augur_token_expiration_dash_persistence (str): bearer token expiration - augur_refresh_token_dash_persistence (str): refresh token for bearer token - augur_user_groups_dash_persistence (str): user's groups - augur_user_group_options_dash_persistence (str): possible groups from source DB - refresh-groups (bool): whether we should refresh user's preferences - search_val (str): query strings to HREF- remove on login to fix refresh bug - login-succeeded (bool): - """ - - # used to extract auth from URL - code_pattern = re.compile(r"\?code=([0-9A-z]+)", re.IGNORECASE) - - # output values when login isn't possible - no_login = [ - "", # username - "", # bearer token - "", # bearer token expiration - "", # refresh token - {}, # user groups - [], # user group options - False, # fetch groups? - "", # search (code_val) removed once logged in - ] - # ^note about 'search' above- we're removing it when this function returns - # so that on refresh the logic below won't trigger another login try if the - # user tries to refresh while still on the page redirected-to from Augur authorization. - - # URL-triggered callback - auth_code_match = re.search(code_pattern, search_val) - - # always go through this path if login not enabled - if (not is_client_startup and not auth_code_match) or (not login_enabled): - logging.warning("LOGIN: Page Switch") - raise dash.exceptions.PreventUpdate - - if auth_code_match: - logging.warning("LOGIN: Redirect from Augur; Code pattern in href") - # the user has just redirected from Augur so we know - # that we need to get their new credentials. - - auth = auth_code_match.group(1) - - # use the auth token to get the bearer token - username, bearer_token, expiration, refresh_token = augur.auth_to_bearer_token(auth) - - # if we try to log in with the auth token we just get and the login fails, we - # tell the user with a popover and do nothing. - - if not all([username, bearer_token, expiration, refresh_token]): - return no_login + [False] - - # expiration should be a future time not a duration - expiration = datetime.now() + timedelta(seconds=expiration) - - elif is_client_startup: - logging.warning("LOGIN: STARTUP - GETTING ADMIN GROUPS") - # try to get admin groups - admin_groups, admin_group_options = get_admin_groups() - - no_login[4] = admin_groups - no_login[5] = admin_group_options - - logging.warning("LOGIN: STARTUP - ADMIN GROUPS SET") - - if expiration and bearer_token: - checked_bt, checked_rt = verify_previous_login_credentials(bearer_token, refresh_token, expiration) - if not all([checked_bt, checked_rt]): - return no_login + [True] - logging.warning("LOGIN: Warm startup; preexisting credentials available") - else: - logging.warning("LOGIN: Cold Startup; no credentials available") - return no_login + [True] - - # get groups for admin and user from front-end - user_groups, user_group_options = get_user_groups(username, bearer_token) - admin_groups, admin_group_options = get_admin_groups() - - # combine admin and user groups - user_groups.update(admin_groups) - user_group_options += admin_group_options - - logging.warning("LOGIN: Success") - return ( - username, - bearer_token, - expiration, - refresh_token, - user_groups, - user_group_options, - False, # refresh_groups - "", # reset search to empty post-login - True, # login succeeded - ) - - @callback( [Output("projects", "data")], [Input("projects", "searchValue")], [ State("projects", "value"), - State("augur_user_group_options_dash_persistence", "data"), + # State("augur_user_group_options_dash_persistence", "data"), ], ) -def dynamic_multiselect_options(user_in: str, selections, augur_groups): +def dynamic_multiselect_options(user_in: str, selections): """ Ref: https://dash.plotly.com/dash-core-components/dropdown#dynamic-options @@ -266,7 +171,26 @@ def dynamic_multiselect_options(user_in: str, selections, augur_groups): return dash.no_update options = augur.get_multiselect_options().copy() - options = options + augur_groups + + if current_user.is_authenticated: + logging.warning(f"LOGINBUTTON: USER LOGGED IN {current_user}") + # TODO: implement more permanent interface + users_cache = StrictRedis( + host="redis-users", + port=6379, + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + try: + if users_cache.exists(f"{current_user.get_id()}_group_options"): + options = options + json.loads( + users_cache.get(f"{current_user.get_id()}_group_options") + ) + except redis.exceptions.ConnectionError: + logging.error( + "Searchbar: couldn't connect to Redis for user group options." + ) # if the number of options changes then we're # adding AUGUR_ entries somewhere. @@ -296,10 +220,9 @@ def dynamic_multiselect_options(user_in: str, selections, augur_groups): [ Input("search", "n_clicks"), State("projects", "value"), - State("augur_user_groups_dash_persistence", "data"), ], ) -def multiselect_values_to_repo_ids(n_clicks, user_vals, user_groups): +def multiselect_values_to_repo_ids(n_clicks, user_vals): if not user_vals: logging.warning("NOTHING SELECTED IN SEARCH BAR") raise dash.exceptions.PreventUpdate @@ -316,6 +239,28 @@ def multiselect_values_to_repo_ids(n_clicks, user_vals, user_groups): org_repos = [v for l in org_repos for v in l] logging.warning(f"ORG_REPOS: {org_repos}") + user_groups = [] + if current_user.is_authenticated: + logging.warning(f"LOGINBUTTON: USER LOGGED IN {current_user}") + # TODO: implement more permanent interface + users_cache = StrictRedis( + host="redis-users", + port=6379, + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + try: + if users_cache.exists(f"{current_user.get_id()}_groups"): + user_groups = json.loads( + users_cache.get(f"{current_user.get_id()}_groups") + ) + logging.warning(f"USERS Groups: {type(user_groups)}, {user_groups}") + except redis.exceptions.ConnectionError: + logging.error( + "Searchbar: couldn't connect to Redis for user group options." + ) + group_repos = [user_groups[g] for g in names if not augur.is_org(g)] # flatten list repo_ids in orgs to 1D group_repos = [v for l in group_repos for v in l] diff --git a/pages/index/index_layout.py b/pages/index/index_layout.py index 68214126..c3b1a1a1 100644 --- a/pages/index/index_layout.py +++ b/pages/index/index_layout.py @@ -25,7 +25,9 @@ ] ), dbc.NavItem( - dbc.NavLink("Refresh Groups", id="refresh-button", disabled=True), + dbc.NavLink( + "Refresh Groups", id="refresh-button", disabled=True + ), ), dbc.NavItem( dbc.NavLink( @@ -38,7 +40,13 @@ ), ), dbc.NavItem( - dbc.NavLink("Log out", id="logout-button", disabled=True), + dbc.NavLink( + "Log out", + id="logout-button", + disabled=True, + href="/logout/", + external_link=True, + ), ), dbc.Popover( children="Login Failed", @@ -83,7 +91,9 @@ [ dbc.Nav( [ - dbc.NavLink(page["name"], href=page["path"], active="exact") + dbc.NavLink( + page["name"], href=page["path"], active="exact" + ) for page in dash.page_registry.values() if page["module"] != "pages.not_found_404" ], @@ -228,17 +238,26 @@ # components to store job-ids for the worker queue dcc.Store(id="job-ids", storage_type="session", data=[]), dcc.Store(id="is-client-startup", storage_type="session", data=True), - dcc.Store(id="augur_user_groups_dash_persistence", storage_type="session", data={}), + dcc.Store( + id="augur_user_groups_dash_persistence", storage_type="session", data=None + ), dcc.Store( id="augur_user_group_options_dash_persistence", storage_type="session", - data=[], + data=None, + ), + dcc.Store( + id="augur_user_bearer_token_dash_persistence", storage_type="local", data="" ), - dcc.Store(id="augur_user_bearer_token_dash_persistence", storage_type="local", data=""), dcc.Store(id="augur_username_dash_persistence", storage_type="local", data=""), - dcc.Store(id="augur_refresh_token_dash_persistence", storage_type="local", data=""), - dcc.Store(id="augur_token_expiration_dash_persistence", storage_type="local", data=""), + dcc.Store( + id="augur_refresh_token_dash_persistence", storage_type="local", data="" + ), + dcc.Store( + id="augur_token_expiration_dash_persistence", storage_type="local", data="" + ), dcc.Store(id="login-succeeded", data=True), + dcc.Store(id="user-group-loading-signal", data="", storage_type="memory"), dcc.Location(id="url"), navbar, dbc.Row( @@ -253,7 +272,11 @@ ), search_bar, dcc.Loading( - children=[html.Div(id="results-output-container", className="mb-4")], + children=[ + html.Div( + id="results-output-container", className="mb-4" + ) + ], color="#119DFF", type="dot", fullscreen=True, diff --git a/queries/user_groups_query.py b/queries/user_groups_query.py new file mode 100644 index 00000000..9be76dad --- /dev/null +++ b/queries/user_groups_query.py @@ -0,0 +1,156 @@ +import logging +from app import celery_app, augur +import pandas as pd +from cache_manager.cache_manager import CacheManager as cm +import io +import datetime as dt +from sqlalchemy.exc import SQLAlchemyError +from redis import StrictRedis +import json + +# DEBUGGING +import os + +QUERY_NAME = "USER_GROUPS_QUERY" + + +@celery_app.task( + bind=True, +) +def user_groups_query(self, user_id): + """ + (Worker Query) + Executes SQL query against Augur frontend for user's groups. + + Args: + ----- + user_id (int): which user's groups we want + + Returns: + -------- + bool: Success of getting groups + """ + logging.warning(f"{QUERY_NAME}_DATA_QUERY - START") + + users_cache = StrictRedis( + host="redis-users", port=6379, password=os.getenv("REDIS_PASSWORD", "") + ) + + # checks connection to Redis, raises redis.exceptions.ConnectionError if connection fails. + # returns True if connection succeeds. + users_cache.ping() + + # check if user is in sessions + if not users_cache.exists(user_id): + raise Exception("Expected user data under user_id not in cache.") + else: + user = json.loads(users_cache.get(user_id)) + + # query groups and options from Augur + users_groups, users_options = get_user_groups( + user["username"], user["access_token"] + ) + + # stores groups and options in cache + groups_set = users_cache.set( + name=f"{user_id}_groups", value=json.dumps(users_groups) + ) + options_set = users_cache.set( + name=f"{user_id}_group_options", value=json.dumps(users_options) + ) + + # returns success of operation + return bool(groups_set and options_set) + + +def get_user_groups(username, bearer_token): + """Requests all user-level groups from augur frontend. + + Args: + username (str): client's username + bearer_token (str): client's bearer token + + Returns: + dict{group_name: [repo_ids]}: dict of users groups + list[{group_name, group_label}]: list of dicts to translate group labels to their values. + """ + + # request to get user's groups + augur_users_groups = augur.make_user_request(access_token=bearer_token) + + # structure of the incoming data + # [{group_name: {favorited: False, repos: [{repo_git: asd;lfkj, repo_id=46555}, ...]}, ...] + # creates the group_name->repo_list mapping and the searchbar options for augur user groups + users_groups = {} + users_group_options = [] + g = augur_users_groups.get("data") + + # each of the augur user groups + for entry in g: + # group has one key- the name. + group_name: str = list(entry.keys())[0] + + # only one value per entry- {favorited: ..., repos: ...}, + # get the value component + repo_list = list(entry.values())[0]["repos"] + + ids = parse_repolist(repo_list) + + # don't accept empty groups + if len(ids) == 0: + continue + + # using lower_name for convenience later- no .lower() calls + lower_name = group_name.lower() + + # group_name->repo_list mapping + users_groups[lower_name] = ids + + # searchbar options + # user's groups are prefixed w/ username to guarantee uniqueness in searchbar + users_group_options.append( + {"value": lower_name, "label": f"{username}: {group_name}"} + ) + + return users_groups, users_group_options + + +def parse_repolist(repo_list, prepend_to_url=""): + """Converts repo_git URLs to + indexed repo_ids from startup for consumption + by group_name->list_of_repo_ids. + + Args: + repo_list ([{repo_info}]): list of repo metadata from augur frontend. + prepend_to_url (str): string to prepend, e.g. "https://" if known not available + + Returns: + [int]: list of repo ids identified. + """ + # structure of the incoming data + # [{repo_git: asd;lfkj, repo_id=46555, ...}, ...] + # creates the group_name->repo_list mapping and the searchbar options for augur user groups + + ids = [] + for repo in repo_list: + if "repo_git" in repo.keys(): + # user groups have URL under 'repo_git' key + repo_url = repo.get("repo_git") + elif "url" in repo.keys(): + # admin groups have URL under "url" key + repo_url = repo.get("url") + else: + # if neither present, skip + logging.error(f"PARSE_REPOLIST: NO REPO_URL IN OBJECT") + continue + + # translate that natural key to the repo's ID in the primary database + repo_id_translated = augur.repo_git_to_id(prepend_to_url + repo_url) + + # check if the translation worked. + if not repo_id_translated: + logging.error(f"PARSE_REPOLIST: {repo_url} NOT TRANSLATABLE TO REPO_ID") + + ids.append(repo_id_translated) + + return ids diff --git a/requirements.txt b/requirements.txt index a8028541..ee55665b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,3 +57,4 @@ dash-mantine-components pyarrow fuzzywuzzy python-Levenshtein +flask-login